FreeBSD kernel kern code
vfs_subr.c
Go to the documentation of this file.
1 /*-
2  * Copyright (c) 1989, 1993
3  * The Regents of the University of California. All rights reserved.
4  * (c) UNIX System Laboratories, Inc.
5  * All or some portions of this file are derived from material licensed
6  * to the University of California by American Telephone and Telegraph
7  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
8  * the permission of UNIX System Laboratories, Inc.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  * notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  * notice, this list of conditions and the following disclaimer in the
17  * documentation and/or other materials provided with the distribution.
18  * 4. Neither the name of the University nor the names of its contributors
19  * may be used to endorse or promote products derived from this software
20  * without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  * @(#)vfs_subr.c 8.31 (Berkeley) 5/26/95
35  */
36 
37 /*
38  * External virtual filesystem routines
39  */
40 
41 #include <sys/cdefs.h>
42 __FBSDID("$BSDSUniX$");
43 
44 #include "opt_compat.h"
45 #include "opt_ddb.h"
46 #include "opt_watchdog.h"
47 
48 #include <sys/param.h>
49 #include <sys/systm.h>
50 #include <sys/bio.h>
51 #include <sys/buf.h>
52 #include <sys/condvar.h>
53 #include <sys/conf.h>
54 #include <sys/dirent.h>
55 #include <sys/event.h>
56 #include <sys/eventhandler.h>
57 #include <sys/extattr.h>
58 #include <sys/file.h>
59 #include <sys/fcntl.h>
60 #include <sys/jail.h>
61 #include <sys/kdb.h>
62 #include <sys/kernel.h>
63 #include <sys/kthread.h>
64 #include <sys/lockf.h>
65 #include <sys/malloc.h>
66 #include <sys/mount.h>
67 #include <sys/namei.h>
68 #include <sys/priv.h>
69 #include <sys/reboot.h>
70 #include <sys/sched.h>
71 #include <sys/sleepqueue.h>
72 #include <sys/smp.h>
73 #include <sys/stat.h>
74 #include <sys/sysctl.h>
75 #include <sys/syslog.h>
76 #include <sys/vmmeter.h>
77 #include <sys/vnode.h>
78 #include <sys/watchdog.h>
79 
80 #include <machine/stdarg.h>
81 
82 #include <security/mac/mac_framework.h>
83 
84 #include <vm/vm.h>
85 #include <vm/vm_object.h>
86 #include <vm/vm_extern.h>
87 #include <vm/pmap.h>
88 #include <vm/vm_map.h>
89 #include <vm/vm_page.h>
90 #include <vm/vm_kern.h>
91 #include <vm/uma.h>
92 
93 #ifdef DDB
94 #include <ddb/ddb.h>
95 #endif
96 
97 #define WI_MPSAFEQ 0
98 #define WI_GIANTQ 1
99 
100 static void delmntque(struct vnode *vp);
101 static int flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo,
102  int slpflag, int slptimeo);
103 static void syncer_shutdown(void *arg, int howto);
104 static int vtryrecycle(struct vnode *vp);
105 static void v_incr_usecount(struct vnode *);
106 static void v_decr_usecount(struct vnode *);
107 static void v_decr_useonly(struct vnode *);
108 static void v_upgrade_usecount(struct vnode *);
109 static void vnlru_free(int);
110 static void vgonel(struct vnode *);
111 static void vfs_knllock(void *arg);
112 static void vfs_knlunlock(void *arg);
113 static void vfs_knl_assert_locked(void *arg);
114 static void vfs_knl_assert_unlocked(void *arg);
115 static void destroy_vpollinfo(struct vpollinfo *vi);
116 
117 /*
118  * Number of vnodes in existence. Increased whenever getnewvnode()
119  * allocates a new vnode, decreased in vdropl() for VI_DOOMED vnode.
120  */
121 static unsigned long numvnodes;
122 
123 SYSCTL_ULONG(_vfs, OID_AUTO, numvnodes, CTLFLAG_RD, &numvnodes, 0,
124  "Number of vnodes in existence");
125 
126 static u_long vnodes_created;
127 SYSCTL_ULONG(_vfs, OID_AUTO, vnodes_created, CTLFLAG_RD, &vnodes_created,
128  0, "Number of vnodes created by getnewvnode");
129 
130 /*
131  * Conversion tables for conversion from vnode types to inode formats
132  * and back.
133  */
134 enum vtype iftovt_tab[16] = {
135  VNON, VFIFO, VCHR, VNON, VDIR, VNON, VBLK, VNON,
136  VREG, VNON, VLNK, VNON, VSOCK, VNON, VNON, VBAD,
137 };
138 int vttoif_tab[10] = {
139  0, S_IFREG, S_IFDIR, S_IFBLK, S_IFCHR, S_IFLNK,
140  S_IFSOCK, S_IFIFO, S_IFMT, S_IFMT
141 };
142 
143 /*
144  * List of vnodes that are ready for recycling.
145  */
146 static TAILQ_HEAD(freelst, vnode) vnode_free_list;
147 
148 /*
149  * Free vnode target. Free vnodes may simply be files which have been stat'd
150  * but not read. This is somewhat common, and a small cache of such files
151  * should be kept to avoid recreation costs.
152  */
153 static u_long wantfreevnodes;
154 SYSCTL_ULONG(_vfs, OID_AUTO, wantfreevnodes, CTLFLAG_RW, &wantfreevnodes, 0, "");
155 /* Number of vnodes in the free list. */
156 static u_long freevnodes;
157 SYSCTL_ULONG(_vfs, OID_AUTO, freevnodes, CTLFLAG_RD, &freevnodes, 0,
158  "Number of vnodes in the free list");
159 
160 static int vlru_allow_cache_src;
161 SYSCTL_INT(_vfs, OID_AUTO, vlru_allow_cache_src, CTLFLAG_RW,
162  &vlru_allow_cache_src, 0, "Allow vlru to reclaim source vnode");
163 
164 static u_long recycles_count;
165 SYSCTL_ULONG(_vfs, OID_AUTO, recycles, CTLFLAG_RD, &recycles_count, 0,
166  "Number of vnodes recycled to avoid exceding kern.maxvnodes");
167 
168 /*
169  * Various variables used for debugging the new implementation of
170  * reassignbuf().
171  * XXX these are probably of (very) limited utility now.
172  */
173 static int reassignbufcalls;
174 SYSCTL_INT(_vfs, OID_AUTO, reassignbufcalls, CTLFLAG_RW, &reassignbufcalls, 0,
175  "Number of calls to reassignbuf");
176 
177 /*
178  * Cache for the mount type id assigned to NFS. This is used for
179  * special checks in nfs/nfs_nqlease.c and vm/vnode_pager.c.
180  */
181 int nfs_mount_type = -1;
182 
183 /* To keep more than one thread at a time from running vfs_getnewfsid */
184 static struct mtx mntid_mtx;
185 
186 /*
187  * Lock for any access to the following:
188  * vnode_free_list
189  * numvnodes
190  * freevnodes
191  */
192 static struct mtx vnode_free_list_mtx;
193 
194 /* Publicly exported FS */
195 struct nfs_public nfs_pub;
196 
197 /* Zone for allocation of new vnodes - used exclusively by getnewvnode() */
198 static uma_zone_t vnode_zone;
199 static uma_zone_t vnodepoll_zone;
200 
201 /*
202  * The workitem queue.
203  *
204  * It is useful to delay writes of file data and filesystem metadata
205  * for tens of seconds so that quickly created and deleted files need
206  * not waste disk bandwidth being created and removed. To realize this,
207  * we append vnodes to a "workitem" queue. When running with a soft
208  * updates implementation, most pending metadata dependencies should
209  * not wait for more than a few seconds. Thus, mounted on block devices
210  * are delayed only about a half the time that file data is delayed.
211  * Similarly, directory updates are more critical, so are only delayed
212  * about a third the time that file data is delayed. Thus, there are
213  * SYNCER_MAXDELAY queues that are processed round-robin at a rate of
214  * one each second (driven off the filesystem syncer process). The
215  * syncer_delayno variable indicates the next queue that is to be processed.
216  * Items that need to be processed soon are placed in this queue:
217  *
218  * syncer_workitem_pending[syncer_delayno]
219  *
220  * A delay of fifteen seconds is done by placing the request fifteen
221  * entries later in the queue:
222  *
223  * syncer_workitem_pending[(syncer_delayno + 15) & syncer_mask]
224  *
225  */
226 static int syncer_delayno;
227 static long syncer_mask;
228 LIST_HEAD(synclist, bufobj);
229 static struct synclist *syncer_workitem_pending[2];
230 /*
231  * The sync_mtx protects:
232  * bo->bo_synclist
233  * sync_vnode_count
234  * syncer_delayno
235  * syncer_state
236  * syncer_workitem_pending
237  * syncer_worklist_len
238  * rushjob
239  */
240 static struct mtx sync_mtx;
241 static struct cv sync_wakeup;
242 
243 #define SYNCER_MAXDELAY 32
244 static int syncer_maxdelay = SYNCER_MAXDELAY; /* maximum delay time */
245 static int syncdelay = 30; /* max time to delay syncing data */
246 static int filedelay = 30; /* time to delay syncing files */
247 SYSCTL_INT(_kern, OID_AUTO, filedelay, CTLFLAG_RW, &filedelay, 0,
248  "Time to delay syncing files (in seconds)");
249 static int dirdelay = 29; /* time to delay syncing directories */
250 SYSCTL_INT(_kern, OID_AUTO, dirdelay, CTLFLAG_RW, &dirdelay, 0,
251  "Time to delay syncing directories (in seconds)");
252 static int metadelay = 28; /* time to delay syncing metadata */
253 SYSCTL_INT(_kern, OID_AUTO, metadelay, CTLFLAG_RW, &metadelay, 0,
254  "Time to delay syncing metadata (in seconds)");
255 static int rushjob; /* number of slots to run ASAP */
256 static int stat_rush_requests; /* number of times I/O speeded up */
257 SYSCTL_INT(_debug, OID_AUTO, rush_requests, CTLFLAG_RW, &stat_rush_requests, 0,
258  "Number of times I/O speeded up (rush requests)");
259 
260 /*
261  * When shutting down the syncer, run it at four times normal speed.
262  */
263 #define SYNCER_SHUTDOWN_SPEEDUP 4
264 static int sync_vnode_count;
265 static int syncer_worklist_len;
266 static enum { SYNCER_RUNNING, SYNCER_SHUTTING_DOWN, SYNCER_FINAL_DELAY }
267  syncer_state;
268 
269 /*
270  * Number of vnodes we want to exist at any one time. This is mostly used
271  * to size hash tables in vnode-related code. It is normally not used in
272  * getnewvnode(), as wantfreevnodes is normally nonzero.)
273  *
274  * XXX desiredvnodes is historical cruft and should not exist.
275  */
276 int desiredvnodes;
277 SYSCTL_INT(_kern, KERN_MAXVNODES, maxvnodes, CTLFLAG_RW,
278  &desiredvnodes, 0, "Maximum number of vnodes");
279 SYSCTL_ULONG(_kern, OID_AUTO, minvnodes, CTLFLAG_RW,
280  &wantfreevnodes, 0, "Minimum number of vnodes (legacy)");
281 static int vnlru_nowhere;
282 SYSCTL_INT(_debug, OID_AUTO, vnlru_nowhere, CTLFLAG_RW,
283  &vnlru_nowhere, 0, "Number of times the vnlru process ran without success");
284 
285 /*
286  * Macros to control when a vnode is freed and recycled. All require
287  * the vnode interlock.
288  */
289 #define VCANRECYCLE(vp) (((vp)->v_iflag & VI_FREE) && !(vp)->v_holdcnt)
290 #define VSHOULDFREE(vp) (!((vp)->v_iflag & VI_FREE) && !(vp)->v_holdcnt)
291 #define VSHOULDBUSY(vp) (((vp)->v_iflag & VI_FREE) && (vp)->v_holdcnt)
292 
293 /* Shift count for (uintptr_t)vp to initialize vp->v_hash. */
294 static int vnsz2log;
295 
296 /*
297  * Initialize the vnode management data structures.
298  *
299  * Reevaluate the following cap on the number of vnodes after the physical
300  * memory size exceeds 512GB. In the limit, as the physical memory size
301  * grows, the ratio of physical pages to vnodes approaches sixteen to one.
302  */
303 #ifndef MAXVNODES_MAX
304 #define MAXVNODES_MAX (512 * (1024 * 1024 * 1024 / (int)PAGE_SIZE / 16))
305 #endif
306 static void
307 vntblinit(void *dummy __unused)
308 {
309  u_int i;
310  int physvnodes, virtvnodes;
311 
312  /*
313  * Desiredvnodes is a function of the physical memory size and the
314  * kernel's heap size. Generally speaking, it scales with the
315  * physical memory size. The ratio of desiredvnodes to physical pages
316  * is one to four until desiredvnodes exceeds 98,304. Thereafter, the
317  * marginal ratio of desiredvnodes to physical pages is one to
318  * sixteen. However, desiredvnodes is limited by the kernel's heap
319  * size. The memory required by desiredvnodes vnodes and vm objects
320  * may not exceed one seventh of the kernel's heap size.
321  */
322  physvnodes = maxproc + cnt.v_page_count / 16 + 3 * min(98304 * 4,
323  cnt.v_page_count) / 16;
324  virtvnodes = vm_kmem_size / (7 * (sizeof(struct vm_object) +
325  sizeof(struct vnode)));
326  desiredvnodes = min(physvnodes, virtvnodes);
327  if (desiredvnodes > MAXVNODES_MAX) {
328  if (bootverbose)
329  printf("Reducing kern.maxvnodes %d -> %d\n",
330  desiredvnodes, MAXVNODES_MAX);
331  desiredvnodes = MAXVNODES_MAX;
332  }
333  wantfreevnodes = desiredvnodes / 4;
334  mtx_init(&mntid_mtx, "mntid", NULL, MTX_DEF);
335  TAILQ_INIT(&vnode_free_list);
336  mtx_init(&vnode_free_list_mtx, "vnode_free_list", NULL, MTX_DEF);
337  vnode_zone = uma_zcreate("VNODE", sizeof (struct vnode), NULL, NULL,
338  NULL, NULL, UMA_ALIGN_PTR, 0);
339  vnodepoll_zone = uma_zcreate("VNODEPOLL", sizeof (struct vpollinfo),
340  NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
341  /*
342  * Initialize the filesystem syncer.
343  */
344  syncer_workitem_pending[WI_MPSAFEQ] = hashinit(syncer_maxdelay, M_VNODE,
345  &syncer_mask);
346  syncer_workitem_pending[WI_GIANTQ] = hashinit(syncer_maxdelay, M_VNODE,
347  &syncer_mask);
348  syncer_maxdelay = syncer_mask + 1;
349  mtx_init(&sync_mtx, "Syncer mtx", NULL, MTX_DEF);
350  cv_init(&sync_wakeup, "syncer");
351  for (i = 1; i <= sizeof(struct vnode); i <<= 1)
352  vnsz2log++;
353  vnsz2log--;
354 }
355 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_FIRST, vntblinit, NULL);
356 
357 
358 /*
359  * Mark a mount point as busy. Used to synchronize access and to delay
360  * unmounting. Eventually, mountlist_mtx is not released on failure.
361  *
362  * vfs_busy() is a custom lock, it can block the caller.
363  * vfs_busy() only sleeps if the unmount is active on the mount point.
364  * For a mountpoint mp, vfs_busy-enforced lock is before lock of any
365  * vnode belonging to mp.
366  *
367  * Lookup uses vfs_busy() to traverse mount points.
368  * root fs var fs
369  * / vnode lock A / vnode lock (/var) D
370  * /var vnode lock B /log vnode lock(/var/log) E
371  * vfs_busy lock C vfs_busy lock F
372  *
373  * Within each file system, the lock order is C->A->B and F->D->E.
374  *
375  * When traversing across mounts, the system follows that lock order:
376  *
377  * C->A->B
378  * |
379  * +->F->D->E
380  *
381  * The lookup() process for namei("/var") illustrates the process:
382  * VOP_LOOKUP() obtains B while A is held
383  * vfs_busy() obtains a shared lock on F while A and B are held
384  * vput() releases lock on B
385  * vput() releases lock on A
386  * VFS_ROOT() obtains lock on D while shared lock on F is held
387  * vfs_unbusy() releases shared lock on F
388  * vn_lock() obtains lock on deadfs vnode vp_crossmp instead of A.
389  * Attempt to lock A (instead of vp_crossmp) while D is held would
390  * violate the global order, causing deadlocks.
391  *
392  * dounmount() locks B while F is drained.
393  */
394 int
395 vfs_busy(struct mount *mp, int flags)
396 {
397 
398  MPASS((flags & ~MBF_MASK) == 0);
399  CTR3(KTR_VFS, "%s: mp %p with flags %d", __func__, mp, flags);
400 
401  MNT_ILOCK(mp);
402  MNT_REF(mp);
403  /*
404  * If mount point is currenly being unmounted, sleep until the
405  * mount point fate is decided. If thread doing the unmounting fails,
406  * it will clear MNTK_UNMOUNT flag before waking us up, indicating
407  * that this mount point has survived the unmount attempt and vfs_busy
408  * should retry. Otherwise the unmounter thread will set MNTK_REFEXPIRE
409  * flag in addition to MNTK_UNMOUNT, indicating that mount point is
410  * about to be really destroyed. vfs_busy needs to release its
411  * reference on the mount point in this case and return with ENOENT,
412  * telling the caller that mount mount it tried to busy is no longer
413  * valid.
414  */
415  while (mp->mnt_kern_flag & MNTK_UNMOUNT) {
416  if (flags & MBF_NOWAIT || mp->mnt_kern_flag & MNTK_REFEXPIRE) {
417  MNT_REL(mp);
418  MNT_IUNLOCK(mp);
419  CTR1(KTR_VFS, "%s: failed busying before sleeping",
420  __func__);
421  return (ENOENT);
422  }
423  if (flags & MBF_MNTLSTLOCK)
424  mtx_unlock(&mountlist_mtx);
425  mp->mnt_kern_flag |= MNTK_MWAIT;
426  msleep(mp, MNT_MTX(mp), PVFS | PDROP, "vfs_busy", 0);
427  if (flags & MBF_MNTLSTLOCK)
428  mtx_lock(&mountlist_mtx);
429  MNT_ILOCK(mp);
430  }
431  if (flags & MBF_MNTLSTLOCK)
432  mtx_unlock(&mountlist_mtx);
433  mp->mnt_lockref++;
434  MNT_IUNLOCK(mp);
435  return (0);
436 }
437 
438 /*
439  * Free a busy filesystem.
440  */
441 void
442 vfs_unbusy(struct mount *mp)
443 {
444 
445  CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
446  MNT_ILOCK(mp);
447  MNT_REL(mp);
448  KASSERT(mp->mnt_lockref > 0, ("negative mnt_lockref"));
449  mp->mnt_lockref--;
450  if (mp->mnt_lockref == 0 && (mp->mnt_kern_flag & MNTK_DRAINING) != 0) {
451  MPASS(mp->mnt_kern_flag & MNTK_UNMOUNT);
452  CTR1(KTR_VFS, "%s: waking up waiters", __func__);
453  mp->mnt_kern_flag &= ~MNTK_DRAINING;
454  wakeup(&mp->mnt_lockref);
455  }
456  MNT_IUNLOCK(mp);
457 }
458 
459 /*
460  * Lookup a mount point by filesystem identifier.
461  */
462 struct mount *
463 vfs_getvfs(fsid_t *fsid)
464 {
465  struct mount *mp;
466 
467  CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid);
468  mtx_lock(&mountlist_mtx);
469  TAILQ_FOREACH(mp, &mountlist, mnt_list) {
470  if (mp->mnt_stat.f_fsid.val[0] == fsid->val[0] &&
471  mp->mnt_stat.f_fsid.val[1] == fsid->val[1]) {
472  vfs_ref(mp);
473  mtx_unlock(&mountlist_mtx);
474  return (mp);
475  }
476  }
477  mtx_unlock(&mountlist_mtx);
478  CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid);
479  return ((struct mount *) 0);
480 }
481 
482 /*
483  * Lookup a mount point by filesystem identifier, busying it before
484  * returning.
485  *
486  * To avoid congestion on mountlist_mtx, implement simple direct-mapped
487  * cache for popular filesystem identifiers. The cache is lockess, using
488  * the fact that struct mount's are never freed. In worst case we may
489  * get pointer to unmounted or even different filesystem, so we have to
490  * check what we got, and go slow way if so.
491  */
492 struct mount *
493 vfs_busyfs(fsid_t *fsid)
494 {
495 #define FSID_CACHE_SIZE 256
496  typedef struct mount * volatile vmp_t;
497  static vmp_t cache[FSID_CACHE_SIZE];
498  struct mount *mp;
499  int error;
500  uint32_t hash;
501 
502  CTR2(KTR_VFS, "%s: fsid %p", __func__, fsid);
503  hash = fsid->val[0] ^ fsid->val[1];
504  hash = (hash >> 16 ^ hash) & (FSID_CACHE_SIZE - 1);
505  mp = cache[hash];
506  if (mp == NULL ||
507  mp->mnt_stat.f_fsid.val[0] != fsid->val[0] ||
508  mp->mnt_stat.f_fsid.val[1] != fsid->val[1])
509  goto slow;
510  if (vfs_busy(mp, 0) != 0) {
511  cache[hash] = NULL;
512  goto slow;
513  }
514  if (mp->mnt_stat.f_fsid.val[0] == fsid->val[0] &&
515  mp->mnt_stat.f_fsid.val[1] == fsid->val[1])
516  return (mp);
517  else
518  vfs_unbusy(mp);
519 
520 slow:
521  mtx_lock(&mountlist_mtx);
522  TAILQ_FOREACH(mp, &mountlist, mnt_list) {
523  if (mp->mnt_stat.f_fsid.val[0] == fsid->val[0] &&
524  mp->mnt_stat.f_fsid.val[1] == fsid->val[1]) {
525  error = vfs_busy(mp, MBF_MNTLSTLOCK);
526  if (error) {
527  cache[hash] = NULL;
528  mtx_unlock(&mountlist_mtx);
529  return (NULL);
530  }
531  cache[hash] = mp;
532  return (mp);
533  }
534  }
535  CTR2(KTR_VFS, "%s: lookup failed for %p id", __func__, fsid);
536  mtx_unlock(&mountlist_mtx);
537  return ((struct mount *) 0);
538 }
539 
540 /*
541  * Check if a user can access privileged mount options.
542  */
543 int
544 vfs_suser(struct mount *mp, struct thread *td)
545 {
546  int error;
547 
548  /*
549  * If the thread is jailed, but this is not a jail-friendly file
550  * system, deny immediately.
551  */
552  if (!(mp->mnt_vfc->vfc_flags & VFCF_JAIL) && jailed(td->td_ucred))
553  return (EPERM);
554 
555  /*
556  * If the file system was mounted outside the jail of the calling
557  * thread, deny immediately.
558  */
559  if (prison_check(td->td_ucred, mp->mnt_cred) != 0)
560  return (EPERM);
561 
562  /*
563  * If file system supports delegated administration, we don't check
564  * for the PRIV_VFS_MOUNT_OWNER privilege - it will be better verified
565  * by the file system itself.
566  * If this is not the user that did original mount, we check for
567  * the PRIV_VFS_MOUNT_OWNER privilege.
568  */
569  if (!(mp->mnt_vfc->vfc_flags & VFCF_DELEGADMIN) &&
570  mp->mnt_cred->cr_uid != td->td_ucred->cr_uid) {
571  if ((error = priv_check(td, PRIV_VFS_MOUNT_OWNER)) != 0)
572  return (error);
573  }
574  return (0);
575 }
576 
577 /*
578  * Get a new unique fsid. Try to make its val[0] unique, since this value
579  * will be used to create fake device numbers for stat(). Also try (but
580  * not so hard) make its val[0] unique mod 2^16, since some emulators only
581  * support 16-bit device numbers. We end up with unique val[0]'s for the
582  * first 2^16 calls and unique val[0]'s mod 2^16 for the first 2^8 calls.
583  *
584  * Keep in mind that several mounts may be running in parallel. Starting
585  * the search one past where the previous search terminated is both a
586  * micro-optimization and a defense against returning the same fsid to
587  * different mounts.
588  */
589 void
590 vfs_getnewfsid(struct mount *mp)
591 {
592  static uint16_t mntid_base;
593  struct mount *nmp;
594  fsid_t tfsid;
595  int mtype;
596 
597  CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
598  mtx_lock(&mntid_mtx);
599  mtype = mp->mnt_vfc->vfc_typenum;
600  tfsid.val[1] = mtype;
601  mtype = (mtype & 0xFF) << 24;
602  for (;;) {
603  tfsid.val[0] = makedev(255,
604  mtype | ((mntid_base & 0xFF00) << 8) | (mntid_base & 0xFF));
605  mntid_base++;
606  if ((nmp = vfs_getvfs(&tfsid)) == NULL)
607  break;
608  vfs_rel(nmp);
609  }
610  mp->mnt_stat.f_fsid.val[0] = tfsid.val[0];
611  mp->mnt_stat.f_fsid.val[1] = tfsid.val[1];
612  mtx_unlock(&mntid_mtx);
613 }
614 
615 /*
616  * Knob to control the precision of file timestamps:
617  *
618  * 0 = seconds only; nanoseconds zeroed.
619  * 1 = seconds and nanoseconds, accurate within 1/HZ.
620  * 2 = seconds and nanoseconds, truncated to microseconds.
621  * >=3 = seconds and nanoseconds, maximum precision.
622  */
623 enum { TSP_SEC, TSP_HZ, TSP_USEC, TSP_NSEC };
624 
625 static int timestamp_precision = TSP_USEC;
626 SYSCTL_INT(_vfs, OID_AUTO, timestamp_precision, CTLFLAG_RW,
627  &timestamp_precision, 0, "File timestamp precision (0: seconds, "
628  "1: sec + ns accurate to 1/HZ, 2: sec + ns truncated to ms, "
629  "3+: sec + ns (max. precision))");
630 
631 /*
632  * Get a current timestamp.
633  */
634 void
635 vfs_timestamp(struct timespec *tsp)
636 {
637  struct timeval tv;
638 
639  switch (timestamp_precision) {
640  case TSP_SEC:
641  tsp->tv_sec = time_second;
642  tsp->tv_nsec = 0;
643  break;
644  case TSP_HZ:
645  getnanotime(tsp);
646  break;
647  case TSP_USEC:
648  microtime(&tv);
649  TIMEVAL_TO_TIMESPEC(&tv, tsp);
650  break;
651  case TSP_NSEC:
652  default:
653  nanotime(tsp);
654  break;
655  }
656 }
657 
658 /*
659  * Set vnode attributes to VNOVAL
660  */
661 void
662 vattr_null(struct vattr *vap)
663 {
664 
665  vap->va_type = VNON;
666  vap->va_size = VNOVAL;
667  vap->va_bytes = VNOVAL;
668  vap->va_mode = VNOVAL;
669  vap->va_nlink = VNOVAL;
670  vap->va_uid = VNOVAL;
671  vap->va_gid = VNOVAL;
672  vap->va_fsid = VNOVAL;
673  vap->va_fileid = VNOVAL;
674  vap->va_blocksize = VNOVAL;
675  vap->va_rdev = VNOVAL;
676  vap->va_atime.tv_sec = VNOVAL;
677  vap->va_atime.tv_nsec = VNOVAL;
678  vap->va_mtime.tv_sec = VNOVAL;
679  vap->va_mtime.tv_nsec = VNOVAL;
680  vap->va_ctime.tv_sec = VNOVAL;
681  vap->va_ctime.tv_nsec = VNOVAL;
682  vap->va_birthtime.tv_sec = VNOVAL;
683  vap->va_birthtime.tv_nsec = VNOVAL;
684  vap->va_flags = VNOVAL;
685  vap->va_gen = VNOVAL;
686  vap->va_vaflags = 0;
687 }
688 
689 /*
690  * This routine is called when we have too many vnodes. It attempts
691  * to free <count> vnodes and will potentially free vnodes that still
692  * have VM backing store (VM backing store is typically the cause
693  * of a vnode blowout so we want to do this). Therefore, this operation
694  * is not considered cheap.
695  *
696  * A number of conditions may prevent a vnode from being reclaimed.
697  * the buffer cache may have references on the vnode, a directory
698  * vnode may still have references due to the namei cache representing
699  * underlying files, or the vnode may be in active use. It is not
700  * desireable to reuse such vnodes. These conditions may cause the
701  * number of vnodes to reach some minimum value regardless of what
702  * you set kern.maxvnodes to. Do not set kern.maxvnodes too low.
703  */
704 static int
705 vlrureclaim(struct mount *mp)
706 {
707  struct vnode *vp;
708  int done;
709  int trigger;
710  int usevnodes;
711  int count;
712 
713  /*
714  * Calculate the trigger point, don't allow user
715  * screwups to blow us up. This prevents us from
716  * recycling vnodes with lots of resident pages. We
717  * aren't trying to free memory, we are trying to
718  * free vnodes.
719  */
720  usevnodes = desiredvnodes;
721  if (usevnodes <= 0)
722  usevnodes = 1;
723  trigger = cnt.v_page_count * 2 / usevnodes;
724  done = 0;
725  vn_start_write(NULL, &mp, V_WAIT);
726  MNT_ILOCK(mp);
727  count = mp->mnt_nvnodelistsize / 10 + 1;
728  while (count != 0) {
729  vp = TAILQ_FIRST(&mp->mnt_nvnodelist);
730  while (vp != NULL && vp->v_type == VMARKER)
731  vp = TAILQ_NEXT(vp, v_nmntvnodes);
732  if (vp == NULL)
733  break;
734  TAILQ_REMOVE(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
735  TAILQ_INSERT_TAIL(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
736  --count;
737  if (!VI_TRYLOCK(vp))
738  goto next_iter;
739  /*
740  * If it's been deconstructed already, it's still
741  * referenced, or it exceeds the trigger, skip it.
742  */
743  if (vp->v_usecount ||
744  (!vlru_allow_cache_src &&
745  !LIST_EMPTY(&(vp)->v_cache_src)) ||
746  (vp->v_iflag & VI_DOOMED) != 0 || (vp->v_object != NULL &&
747  vp->v_object->resident_page_count > trigger)) {
748  VI_UNLOCK(vp);
749  goto next_iter;
750  }
751  MNT_IUNLOCK(mp);
752  vholdl(vp);
753  if (VOP_LOCK(vp, LK_INTERLOCK|LK_EXCLUSIVE|LK_NOWAIT)) {
754  vdrop(vp);
755  goto next_iter_mntunlocked;
756  }
757  VI_LOCK(vp);
758  /*
759  * v_usecount may have been bumped after VOP_LOCK() dropped
760  * the vnode interlock and before it was locked again.
761  *
762  * It is not necessary to recheck VI_DOOMED because it can
763  * only be set by another thread that holds both the vnode
764  * lock and vnode interlock. If another thread has the
765  * vnode lock before we get to VOP_LOCK() and obtains the
766  * vnode interlock after VOP_LOCK() drops the vnode
767  * interlock, the other thread will be unable to drop the
768  * vnode lock before our VOP_LOCK() call fails.
769  */
770  if (vp->v_usecount ||
771  (!vlru_allow_cache_src &&
772  !LIST_EMPTY(&(vp)->v_cache_src)) ||
773  (vp->v_object != NULL &&
774  vp->v_object->resident_page_count > trigger)) {
775  VOP_UNLOCK(vp, LK_INTERLOCK);
776  goto next_iter_mntunlocked;
777  }
778  KASSERT((vp->v_iflag & VI_DOOMED) == 0,
779  ("VI_DOOMED unexpectedly detected in vlrureclaim()"));
780  atomic_add_long(&recycles_count, 1);
781  vgonel(vp);
782  VOP_UNLOCK(vp, 0);
783  vdropl(vp);
784  done++;
785 next_iter_mntunlocked:
786  if (!should_yield())
787  goto relock_mnt;
788  goto yield;
789 next_iter:
790  if (!should_yield())
791  continue;
792  MNT_IUNLOCK(mp);
793 yield:
794  kern_yield(PRI_UNCHANGED);
795 relock_mnt:
796  MNT_ILOCK(mp);
797  }
798  MNT_IUNLOCK(mp);
799  vn_finished_write(mp);
800  return done;
801 }
802 
803 /*
804  * Attempt to keep the free list at wantfreevnodes length.
805  */
806 static void
807 vnlru_free(int count)
808 {
809  struct vnode *vp;
810  int vfslocked;
811 
812  mtx_assert(&vnode_free_list_mtx, MA_OWNED);
813  for (; count > 0; count--) {
814  vp = TAILQ_FIRST(&vnode_free_list);
815  /*
816  * The list can be modified while the free_list_mtx
817  * has been dropped and vp could be NULL here.
818  */
819  if (!vp)
820  break;
821  VNASSERT(vp->v_op != NULL, vp,
822  ("vnlru_free: vnode already reclaimed."));
823  KASSERT((vp->v_iflag & VI_FREE) != 0,
824  ("Removing vnode not on freelist"));
825  KASSERT((vp->v_iflag & VI_ACTIVE) == 0,
826  ("Mangling active vnode"));
827  TAILQ_REMOVE(&vnode_free_list, vp, v_actfreelist);
828  /*
829  * Don't recycle if we can't get the interlock.
830  */
831  if (!VI_TRYLOCK(vp)) {
832  TAILQ_INSERT_TAIL(&vnode_free_list, vp, v_actfreelist);
833  continue;
834  }
835  VNASSERT(VCANRECYCLE(vp), vp,
836  ("vp inconsistent on freelist"));
837  freevnodes--;
838  vp->v_iflag &= ~VI_FREE;
839  vholdl(vp);
840  mtx_unlock(&vnode_free_list_mtx);
841  VI_UNLOCK(vp);
842  vfslocked = VFS_LOCK_GIANT(vp->v_mount);
843  vtryrecycle(vp);
844  VFS_UNLOCK_GIANT(vfslocked);
845  /*
846  * If the recycled succeeded this vdrop will actually free
847  * the vnode. If not it will simply place it back on
848  * the free list.
849  */
850  vdrop(vp);
851  mtx_lock(&vnode_free_list_mtx);
852  }
853 }
854 /*
855  * Attempt to recycle vnodes in a context that is always safe to block.
856  * Calling vlrurecycle() from the bowels of filesystem code has some
857  * interesting deadlock problems.
858  */
859 static struct proc *vnlruproc;
860 static int vnlruproc_sig;
861 
862 static void
863 vnlru_proc(void)
864 {
865  struct mount *mp, *nmp;
866  int done, vfslocked;
867  struct proc *p = vnlruproc;
868 
869  EVENTHANDLER_REGISTER(shutdown_pre_sync, kproc_shutdown, p,
870  SHUTDOWN_PRI_FIRST);
871 
872  for (;;) {
873  kproc_suspend_check(p);
874  mtx_lock(&vnode_free_list_mtx);
875  if (freevnodes > wantfreevnodes)
876  vnlru_free(freevnodes - wantfreevnodes);
877  if (numvnodes <= desiredvnodes * 9 / 10) {
878  vnlruproc_sig = 0;
879  wakeup(&vnlruproc_sig);
880  msleep(vnlruproc, &vnode_free_list_mtx,
881  PVFS|PDROP, "vlruwt", hz);
882  continue;
883  }
884  mtx_unlock(&vnode_free_list_mtx);
885  done = 0;
886  mtx_lock(&mountlist_mtx);
887  for (mp = TAILQ_FIRST(&mountlist); mp != NULL; mp = nmp) {
888  if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK)) {
889  nmp = TAILQ_NEXT(mp, mnt_list);
890  continue;
891  }
892  vfslocked = VFS_LOCK_GIANT(mp);
893  done += vlrureclaim(mp);
894  VFS_UNLOCK_GIANT(vfslocked);
895  mtx_lock(&mountlist_mtx);
896  nmp = TAILQ_NEXT(mp, mnt_list);
897  vfs_unbusy(mp);
898  }
899  mtx_unlock(&mountlist_mtx);
900  if (done == 0) {
901 #if 0
902  /* These messages are temporary debugging aids */
903  if (vnlru_nowhere < 5)
904  printf("vnlru process getting nowhere..\n");
905  else if (vnlru_nowhere == 5)
906  printf("vnlru process messages stopped.\n");
907 #endif
908  vnlru_nowhere++;
909  tsleep(vnlruproc, PPAUSE, "vlrup", hz * 3);
910  } else
911  kern_yield(PRI_UNCHANGED);
912  }
913 }
914 
915 static struct kproc_desc vnlru_kp = {
916  "vnlru",
917  vnlru_proc,
918  &vnlruproc
919 };
920 SYSINIT(vnlru, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start,
921  &vnlru_kp);
922 
923 /*
924  * Routines having to do with the management of the vnode table.
925  */
926 
927 /*
928  * Try to recycle a freed vnode. We abort if anyone picks up a reference
929  * before we actually vgone(). This function must be called with the vnode
930  * held to prevent the vnode from being returned to the free list midway
931  * through vgone().
932  */
933 static int
934 vtryrecycle(struct vnode *vp)
935 {
936  struct mount *vnmp;
937 
938  CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
939  VNASSERT(vp->v_holdcnt, vp,
940  ("vtryrecycle: Recycling vp %p without a reference.", vp));
941  /*
942  * This vnode may found and locked via some other list, if so we
943  * can't recycle it yet.
944  */
945  if (VOP_LOCK(vp, LK_EXCLUSIVE | LK_NOWAIT) != 0) {
946  CTR2(KTR_VFS,
947  "%s: impossible to recycle, vp %p lock is already held",
948  __func__, vp);
949  return (EWOULDBLOCK);
950  }
951  /*
952  * Don't recycle if its filesystem is being suspended.
953  */
954  if (vn_start_write(vp, &vnmp, V_NOWAIT) != 0) {
955  VOP_UNLOCK(vp, 0);
956  CTR2(KTR_VFS,
957  "%s: impossible to recycle, cannot start the write for %p",
958  __func__, vp);
959  return (EBUSY);
960  }
961  /*
962  * If we got this far, we need to acquire the interlock and see if
963  * anyone picked up this vnode from another list. If not, we will
964  * mark it with DOOMED via vgonel() so that anyone who does find it
965  * will skip over it.
966  */
967  VI_LOCK(vp);
968  if (vp->v_usecount) {
969  VOP_UNLOCK(vp, LK_INTERLOCK);
970  vn_finished_write(vnmp);
971  CTR2(KTR_VFS,
972  "%s: impossible to recycle, %p is already referenced",
973  __func__, vp);
974  return (EBUSY);
975  }
976  if ((vp->v_iflag & VI_DOOMED) == 0) {
977  atomic_add_long(&recycles_count, 1);
978  vgonel(vp);
979  }
980  VOP_UNLOCK(vp, LK_INTERLOCK);
981  vn_finished_write(vnmp);
982  return (0);
983 }
984 
985 /*
986  * Wait for available vnodes.
987  */
988 static int
989 getnewvnode_wait(int suspended)
990 {
991 
992  mtx_assert(&vnode_free_list_mtx, MA_OWNED);
993  if (numvnodes > desiredvnodes) {
994  if (suspended) {
995  /*
996  * File system is beeing suspended, we cannot risk a
997  * deadlock here, so allocate new vnode anyway.
998  */
999  if (freevnodes > wantfreevnodes)
1000  vnlru_free(freevnodes - wantfreevnodes);
1001  return (0);
1002  }
1003  if (vnlruproc_sig == 0) {
1004  vnlruproc_sig = 1; /* avoid unnecessary wakeups */
1005  wakeup(vnlruproc);
1006  }
1007  msleep(&vnlruproc_sig, &vnode_free_list_mtx, PVFS,
1008  "vlruwk", hz);
1009  }
1010  return (numvnodes > desiredvnodes ? ENFILE : 0);
1011 }
1012 
1013 void
1014 getnewvnode_reserve(u_int count)
1015 {
1016  struct thread *td;
1017 
1018  td = curthread;
1019  /* First try to be quick and racy. */
1020  if (atomic_fetchadd_long(&numvnodes, count) + count <= desiredvnodes) {
1021  td->td_vp_reserv += count;
1022  return;
1023  } else
1024  atomic_subtract_long(&numvnodes, count);
1025 
1026  mtx_lock(&vnode_free_list_mtx);
1027  while (count > 0) {
1028  if (getnewvnode_wait(0) == 0) {
1029  count--;
1030  td->td_vp_reserv++;
1031  atomic_add_long(&numvnodes, 1);
1032  }
1033  }
1034  mtx_unlock(&vnode_free_list_mtx);
1035 }
1036 
1037 void
1038 getnewvnode_drop_reserve(void)
1039 {
1040  struct thread *td;
1041 
1042  td = curthread;
1043  atomic_subtract_long(&numvnodes, td->td_vp_reserv);
1044  td->td_vp_reserv = 0;
1045 }
1046 
1047 /*
1048  * Return the next vnode from the free list.
1049  */
1050 int
1051 getnewvnode(const char *tag, struct mount *mp, struct vop_vector *vops,
1052  struct vnode **vpp)
1053 {
1054  struct vnode *vp;
1055  struct bufobj *bo;
1056  struct thread *td;
1057  int error;
1058 
1059  CTR3(KTR_VFS, "%s: mp %p with tag %s", __func__, mp, tag);
1060  vp = NULL;
1061  td = curthread;
1062  if (td->td_vp_reserv > 0) {
1063  td->td_vp_reserv -= 1;
1064  goto alloc;
1065  }
1066  mtx_lock(&vnode_free_list_mtx);
1067  /*
1068  * Lend our context to reclaim vnodes if they've exceeded the max.
1069  */
1070  if (freevnodes > wantfreevnodes)
1071  vnlru_free(1);
1072  error = getnewvnode_wait(mp != NULL && (mp->mnt_kern_flag &
1073  MNTK_SUSPEND));
1074 #if 0 /* XXX Not all VFS_VGET/ffs_vget callers check returns. */
1075  if (error != 0) {
1076  mtx_unlock(&vnode_free_list_mtx);
1077  return (error);
1078  }
1079 #endif
1080  atomic_add_long(&numvnodes, 1);
1081  mtx_unlock(&vnode_free_list_mtx);
1082 alloc:
1083  atomic_add_long(&vnodes_created, 1);
1084  vp = (struct vnode *) uma_zalloc(vnode_zone, M_WAITOK|M_ZERO);
1085  /*
1086  * Setup locks.
1087  */
1088  vp->v_vnlock = &vp->v_lock;
1089  mtx_init(&vp->v_interlock, "vnode interlock", NULL, MTX_DEF);
1090  /*
1091  * By default, don't allow shared locks unless filesystems
1092  * opt-in.
1093  */
1094  lockinit(vp->v_vnlock, PVFS, tag, VLKTIMEOUT, LK_NOSHARE);
1095  /*
1096  * Initialize bufobj.
1097  */
1098  bo = &vp->v_bufobj;
1099  bo->__bo_vnode = vp;
1100  mtx_init(BO_MTX(bo), "bufobj interlock", NULL, MTX_DEF);
1101  bo->bo_ops = &buf_ops_bio;
1102  bo->bo_private = vp;
1103  TAILQ_INIT(&bo->bo_clean.bv_hd);
1104  TAILQ_INIT(&bo->bo_dirty.bv_hd);
1105  /*
1106  * Initialize namecache.
1107  */
1108  LIST_INIT(&vp->v_cache_src);
1109  TAILQ_INIT(&vp->v_cache_dst);
1110  /*
1111  * Finalize various vnode identity bits.
1112  */
1113  vp->v_type = VNON;
1114  vp->v_tag = tag;
1115  vp->v_op = vops;
1116  v_incr_usecount(vp);
1117  vp->v_data = 0;
1118 #ifdef MAC
1119  mac_vnode_init(vp);
1120  if (mp != NULL && (mp->mnt_flag & MNT_MULTILABEL) == 0)
1121  mac_vnode_associate_singlelabel(mp, vp);
1122  else if (mp == NULL && vops != &dead_vnodeops)
1123  printf("NULL mp in getnewvnode()\n");
1124 #endif
1125  if (mp != NULL) {
1126  bo->bo_bsize = mp->mnt_stat.f_iosize;
1127  if ((mp->mnt_kern_flag & MNTK_NOKNOTE) != 0)
1128  vp->v_vflag |= VV_NOKNOTE;
1129  }
1130  rangelock_init(&vp->v_rl);
1131 
1132  /*
1133  * For the filesystems which do not use vfs_hash_insert(),
1134  * still initialize v_hash to have vfs_hash_index() useful.
1135  * E.g., nullfs uses vfs_hash_index() on the lower vnode for
1136  * its own hashing.
1137  */
1138  vp->v_hash = (uintptr_t)vp >> vnsz2log;
1139 
1140  *vpp = vp;
1141  return (0);
1142 }
1143 
1144 /*
1145  * Delete from old mount point vnode list, if on one.
1146  */
1147 static void
1148 delmntque(struct vnode *vp)
1149 {
1150  struct mount *mp;
1151  int active;
1152 
1153  mp = vp->v_mount;
1154  if (mp == NULL)
1155  return;
1156  MNT_ILOCK(mp);
1157  VI_LOCK(vp);
1158  KASSERT(mp->mnt_activevnodelistsize <= mp->mnt_nvnodelistsize,
1159  ("Active vnode list size %d > Vnode list size %d",
1160  mp->mnt_activevnodelistsize, mp->mnt_nvnodelistsize));
1161  active = vp->v_iflag & VI_ACTIVE;
1162  vp->v_iflag &= ~VI_ACTIVE;
1163  if (active) {
1164  mtx_lock(&vnode_free_list_mtx);
1165  TAILQ_REMOVE(&mp->mnt_activevnodelist, vp, v_actfreelist);
1166  mp->mnt_activevnodelistsize--;
1167  mtx_unlock(&vnode_free_list_mtx);
1168  }
1169  vp->v_mount = NULL;
1170  VI_UNLOCK(vp);
1171  VNASSERT(mp->mnt_nvnodelistsize > 0, vp,
1172  ("bad mount point vnode list size"));
1173  TAILQ_REMOVE(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
1174  mp->mnt_nvnodelistsize--;
1175  MNT_REL(mp);
1176  MNT_IUNLOCK(mp);
1177 }
1178 
1179 static void
1180 insmntque_stddtr(struct vnode *vp, void *dtr_arg)
1181 {
1182 
1183  vp->v_data = NULL;
1184  vp->v_op = &dead_vnodeops;
1185  /* XXX non mp-safe fs may still call insmntque with vnode
1186  unlocked */
1187  if (!VOP_ISLOCKED(vp))
1188  vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1189  vgone(vp);
1190  vput(vp);
1191 }
1192 
1193 /*
1194  * Insert into list of vnodes for the new mount point, if available.
1195  */
1196 int
1197 insmntque1(struct vnode *vp, struct mount *mp,
1198  void (*dtr)(struct vnode *, void *), void *dtr_arg)
1199 {
1200  int locked;
1201 
1202  KASSERT(vp->v_mount == NULL,
1203  ("insmntque: vnode already on per mount vnode list"));
1204  VNASSERT(mp != NULL, vp, ("Don't call insmntque(foo, NULL)"));
1205 #ifdef DEBUG_VFS_LOCKS
1206  if (!VFS_NEEDSGIANT(mp))
1207  ASSERT_VOP_ELOCKED(vp,
1208  "insmntque: mp-safe fs and non-locked vp");
1209 #endif
1210  /*
1211  * We acquire the vnode interlock early to ensure that the
1212  * vnode cannot be recycled by another process releasing a
1213  * holdcnt on it before we get it on both the vnode list
1214  * and the active vnode list. The mount mutex protects only
1215  * manipulation of the vnode list and the vnode freelist
1216  * mutex protects only manipulation of the active vnode list.
1217  * Hence the need to hold the vnode interlock throughout.
1218  */
1219  MNT_ILOCK(mp);
1220  VI_LOCK(vp);
1221  if ((mp->mnt_kern_flag & MNTK_NOINSMNTQ) != 0 &&
1222  ((mp->mnt_kern_flag & MNTK_UNMOUNTF) != 0 ||
1223  mp->mnt_nvnodelistsize == 0)) {
1224  locked = VOP_ISLOCKED(vp);
1225  if (!locked || (locked == LK_EXCLUSIVE &&
1226  (vp->v_vflag & VV_FORCEINSMQ) == 0)) {
1227  VI_UNLOCK(vp);
1228  MNT_IUNLOCK(mp);
1229  if (dtr != NULL)
1230  dtr(vp, dtr_arg);
1231  return (EBUSY);
1232  }
1233  }
1234  vp->v_mount = mp;
1235  MNT_REF(mp);
1236  TAILQ_INSERT_TAIL(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
1237  VNASSERT(mp->mnt_nvnodelistsize >= 0, vp,
1238  ("neg mount point vnode list size"));
1239  mp->mnt_nvnodelistsize++;
1240  KASSERT((vp->v_iflag & VI_ACTIVE) == 0,
1241  ("Activating already active vnode"));
1242  vp->v_iflag |= VI_ACTIVE;
1243  mtx_lock(&vnode_free_list_mtx);
1244  TAILQ_INSERT_HEAD(&mp->mnt_activevnodelist, vp, v_actfreelist);
1245  mp->mnt_activevnodelistsize++;
1246  mtx_unlock(&vnode_free_list_mtx);
1247  VI_UNLOCK(vp);
1248  MNT_IUNLOCK(mp);
1249  return (0);
1250 }
1251 
1252 int
1253 insmntque(struct vnode *vp, struct mount *mp)
1254 {
1255 
1256  return (insmntque1(vp, mp, insmntque_stddtr, NULL));
1257 }
1258 
1259 /*
1260  * Flush out and invalidate all buffers associated with a bufobj
1261  * Called with the underlying object locked.
1262  */
1263 int
1264 bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo)
1265 {
1266  int error;
1267 
1268  BO_LOCK(bo);
1269  if (flags & V_SAVE) {
1270  error = bufobj_wwait(bo, slpflag, slptimeo);
1271  if (error) {
1272  BO_UNLOCK(bo);
1273  return (error);
1274  }
1275  if (bo->bo_dirty.bv_cnt > 0) {
1276  BO_UNLOCK(bo);
1277  if ((error = BO_SYNC(bo, MNT_WAIT)) != 0)
1278  return (error);
1279  /*
1280  * XXX We could save a lock/unlock if this was only
1281  * enabled under INVARIANTS
1282  */
1283  BO_LOCK(bo);
1284  if (bo->bo_numoutput > 0 || bo->bo_dirty.bv_cnt > 0)
1285  panic("vinvalbuf: dirty bufs");
1286  }
1287  }
1288  /*
1289  * If you alter this loop please notice that interlock is dropped and
1290  * reacquired in flushbuflist. Special care is needed to ensure that
1291  * no race conditions occur from this.
1292  */
1293  do {
1294  error = flushbuflist(&bo->bo_clean,
1295  flags, bo, slpflag, slptimeo);
1296  if (error == 0 && !(flags & V_CLEANONLY))
1297  error = flushbuflist(&bo->bo_dirty,
1298  flags, bo, slpflag, slptimeo);
1299  if (error != 0 && error != EAGAIN) {
1300  BO_UNLOCK(bo);
1301  return (error);
1302  }
1303  } while (error != 0);
1304 
1305  /*
1306  * Wait for I/O to complete. XXX needs cleaning up. The vnode can
1307  * have write I/O in-progress but if there is a VM object then the
1308  * VM object can also have read-I/O in-progress.
1309  */
1310  do {
1311  bufobj_wwait(bo, 0, 0);
1312  BO_UNLOCK(bo);
1313  if (bo->bo_object != NULL) {
1314  VM_OBJECT_LOCK(bo->bo_object);
1315  vm_object_pip_wait(bo->bo_object, "bovlbx");
1316  VM_OBJECT_UNLOCK(bo->bo_object);
1317  }
1318  BO_LOCK(bo);
1319  } while (bo->bo_numoutput > 0);
1320  BO_UNLOCK(bo);
1321 
1322  /*
1323  * Destroy the copy in the VM cache, too.
1324  */
1325  if (bo->bo_object != NULL &&
1326  (flags & (V_ALT | V_NORMAL | V_CLEANONLY)) == 0) {
1327  VM_OBJECT_LOCK(bo->bo_object);
1328  vm_object_page_remove(bo->bo_object, 0, 0, (flags & V_SAVE) ?
1329  OBJPR_CLEANONLY : 0);
1330  VM_OBJECT_UNLOCK(bo->bo_object);
1331  }
1332 
1333 #ifdef INVARIANTS
1334  BO_LOCK(bo);
1335  if ((flags & (V_ALT | V_NORMAL | V_CLEANONLY)) == 0 &&
1336  (bo->bo_dirty.bv_cnt > 0 || bo->bo_clean.bv_cnt > 0))
1337  panic("vinvalbuf: flush failed");
1338  BO_UNLOCK(bo);
1339 #endif
1340  return (0);
1341 }
1342 
1343 /*
1344  * Flush out and invalidate all buffers associated with a vnode.
1345  * Called with the underlying object locked.
1346  */
1347 int
1348 vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo)
1349 {
1350 
1351  CTR3(KTR_VFS, "%s: vp %p with flags %d", __func__, vp, flags);
1352  ASSERT_VOP_LOCKED(vp, "vinvalbuf");
1353  if (vp->v_object != NULL && vp->v_object->handle != vp)
1354  return (0);
1355  return (bufobj_invalbuf(&vp->v_bufobj, flags, slpflag, slptimeo));
1356 }
1357 
1358 /*
1359  * Flush out buffers on the specified list.
1360  *
1361  */
1362 static int
1363 flushbuflist( struct bufv *bufv, int flags, struct bufobj *bo, int slpflag,
1364  int slptimeo)
1365 {
1366  struct buf *bp, *nbp;
1367  int retval, error;
1368  daddr_t lblkno;
1369  b_xflags_t xflags;
1370 
1371  ASSERT_BO_LOCKED(bo);
1372 
1373  retval = 0;
1374  TAILQ_FOREACH_SAFE(bp, &bufv->bv_hd, b_bobufs, nbp) {
1375  if (((flags & V_NORMAL) && (bp->b_xflags & BX_ALTDATA)) ||
1376  ((flags & V_ALT) && (bp->b_xflags & BX_ALTDATA) == 0)) {
1377  continue;
1378  }
1379  lblkno = 0;
1380  xflags = 0;
1381  if (nbp != NULL) {
1382  lblkno = nbp->b_lblkno;
1383  xflags = nbp->b_xflags &
1384  (BX_BKGRDMARKER | BX_VNDIRTY | BX_VNCLEAN);
1385  }
1386  retval = EAGAIN;
1387  error = BUF_TIMELOCK(bp,
1388  LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, BO_MTX(bo),
1389  "flushbuf", slpflag, slptimeo);
1390  if (error) {
1391  BO_LOCK(bo);
1392  return (error != ENOLCK ? error : EAGAIN);
1393  }
1394  KASSERT(bp->b_bufobj == bo,
1395  ("bp %p wrong b_bufobj %p should be %p",
1396  bp, bp->b_bufobj, bo));
1397  if (bp->b_bufobj != bo) { /* XXX: necessary ? */
1398  BUF_UNLOCK(bp);
1399  BO_LOCK(bo);
1400  return (EAGAIN);
1401  }
1402  /*
1403  * XXX Since there are no node locks for NFS, I
1404  * believe there is a slight chance that a delayed
1405  * write will occur while sleeping just above, so
1406  * check for it.
1407  */
1408  if (((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI) &&
1409  (flags & V_SAVE)) {
1410  BO_LOCK(bo);
1411  bremfree(bp);
1412  BO_UNLOCK(bo);
1413  bp->b_flags |= B_ASYNC;
1414  bwrite(bp);
1415  BO_LOCK(bo);
1416  return (EAGAIN); /* XXX: why not loop ? */
1417  }
1418  BO_LOCK(bo);
1419  bremfree(bp);
1420  BO_UNLOCK(bo);
1421  bp->b_flags |= (B_INVAL | B_RELBUF);
1422  bp->b_flags &= ~B_ASYNC;
1423  brelse(bp);
1424  BO_LOCK(bo);
1425  if (nbp != NULL &&
1426  (nbp->b_bufobj != bo ||
1427  nbp->b_lblkno != lblkno ||
1428  (nbp->b_xflags &
1429  (BX_BKGRDMARKER | BX_VNDIRTY | BX_VNCLEAN)) != xflags))
1430  break; /* nbp invalid */
1431  }
1432  return (retval);
1433 }
1434 
1435 /*
1436  * Truncate a file's buffer and pages to a specified length. This
1437  * is in lieu of the old vinvalbuf mechanism, which performed unneeded
1438  * sync activity.
1439  */
1440 int
1441 vtruncbuf(struct vnode *vp, struct ucred *cred, struct thread *td,
1442  off_t length, int blksize)
1443 {
1444  struct buf *bp, *nbp;
1445  int anyfreed;
1446  int trunclbn;
1447  struct bufobj *bo;
1448 
1449  CTR5(KTR_VFS, "%s: vp %p with cred %p and block %d:%ju", __func__,
1450  vp, cred, blksize, (uintmax_t)length);
1451 
1452  /*
1453  * Round up to the *next* lbn.
1454  */
1455  trunclbn = (length + blksize - 1) / blksize;
1456 
1457  ASSERT_VOP_LOCKED(vp, "vtruncbuf");
1458 restart:
1459  bo = &vp->v_bufobj;
1460  BO_LOCK(bo);
1461  anyfreed = 1;
1462  for (;anyfreed;) {
1463  anyfreed = 0;
1464  TAILQ_FOREACH_SAFE(bp, &bo->bo_clean.bv_hd, b_bobufs, nbp) {
1465  if (bp->b_lblkno < trunclbn)
1466  continue;
1467  if (BUF_LOCK(bp,
1468  LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
1469  BO_MTX(bo)) == ENOLCK)
1470  goto restart;
1471 
1472  BO_LOCK(bo);
1473  bremfree(bp);
1474  BO_UNLOCK(bo);
1475  bp->b_flags |= (B_INVAL | B_RELBUF);
1476  bp->b_flags &= ~B_ASYNC;
1477  brelse(bp);
1478  anyfreed = 1;
1479 
1480  BO_LOCK(bo);
1481  if (nbp != NULL &&
1482  (((nbp->b_xflags & BX_VNCLEAN) == 0) ||
1483  (nbp->b_vp != vp) ||
1484  (nbp->b_flags & B_DELWRI))) {
1485  BO_UNLOCK(bo);
1486  goto restart;
1487  }
1488  }
1489 
1490  TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
1491  if (bp->b_lblkno < trunclbn)
1492  continue;
1493  if (BUF_LOCK(bp,
1494  LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
1495  BO_MTX(bo)) == ENOLCK)
1496  goto restart;
1497  BO_LOCK(bo);
1498  bremfree(bp);
1499  BO_UNLOCK(bo);
1500  bp->b_flags |= (B_INVAL | B_RELBUF);
1501  bp->b_flags &= ~B_ASYNC;
1502  brelse(bp);
1503  anyfreed = 1;
1504 
1505  BO_LOCK(bo);
1506  if (nbp != NULL &&
1507  (((nbp->b_xflags & BX_VNDIRTY) == 0) ||
1508  (nbp->b_vp != vp) ||
1509  (nbp->b_flags & B_DELWRI) == 0)) {
1510  BO_UNLOCK(bo);
1511  goto restart;
1512  }
1513  }
1514  }
1515 
1516  if (length > 0) {
1517 restartsync:
1518  TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
1519  if (bp->b_lblkno > 0)
1520  continue;
1521  /*
1522  * Since we hold the vnode lock this should only
1523  * fail if we're racing with the buf daemon.
1524  */
1525  if (BUF_LOCK(bp,
1526  LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
1527  BO_MTX(bo)) == ENOLCK) {
1528  goto restart;
1529  }
1530  VNASSERT((bp->b_flags & B_DELWRI), vp,
1531  ("buf(%p) on dirty queue without DELWRI", bp));
1532 
1533  BO_LOCK(bo);
1534  bremfree(bp);
1535  BO_UNLOCK(bo);
1536  bawrite(bp);
1537  BO_LOCK(bo);
1538  goto restartsync;
1539  }
1540  }
1541 
1542  bufobj_wwait(bo, 0, 0);
1543  BO_UNLOCK(bo);
1544  vnode_pager_setsize(vp, length);
1545 
1546  return (0);
1547 }
1548 
1549 /*
1550  * buf_splay() - splay tree core for the clean/dirty list of buffers in
1551  * a vnode.
1552  *
1553  * NOTE: We have to deal with the special case of a background bitmap
1554  * buffer, a situation where two buffers will have the same logical
1555  * block offset. We want (1) only the foreground buffer to be accessed
1556  * in a lookup and (2) must differentiate between the foreground and
1557  * background buffer in the splay tree algorithm because the splay
1558  * tree cannot normally handle multiple entities with the same 'index'.
1559  * We accomplish this by adding differentiating flags to the splay tree's
1560  * numerical domain.
1561  */
1562 static
1563 struct buf *
1564 buf_splay(daddr_t lblkno, b_xflags_t xflags, struct buf *root)
1565 {
1566  struct buf dummy;
1567  struct buf *lefttreemax, *righttreemin, *y;
1568 
1569  if (root == NULL)
1570  return (NULL);
1571  lefttreemax = righttreemin = &dummy;
1572  for (;;) {
1573  if (lblkno < root->b_lblkno ||
1574  (lblkno == root->b_lblkno &&
1575  (xflags & BX_BKGRDMARKER) < (root->b_xflags & BX_BKGRDMARKER))) {
1576  if ((y = root->b_left) == NULL)
1577  break;
1578  if (lblkno < y->b_lblkno) {
1579  /* Rotate right. */
1580  root->b_left = y->b_right;
1581  y->b_right = root;
1582  root = y;
1583  if ((y = root->b_left) == NULL)
1584  break;
1585  }
1586  /* Link into the new root's right tree. */
1587  righttreemin->b_left = root;
1588  righttreemin = root;
1589  } else if (lblkno > root->b_lblkno ||
1590  (lblkno == root->b_lblkno &&
1591  (xflags & BX_BKGRDMARKER) > (root->b_xflags & BX_BKGRDMARKER))) {
1592  if ((y = root->b_right) == NULL)
1593  break;
1594  if (lblkno > y->b_lblkno) {
1595  /* Rotate left. */
1596  root->b_right = y->b_left;
1597  y->b_left = root;
1598  root = y;
1599  if ((y = root->b_right) == NULL)
1600  break;
1601  }
1602  /* Link into the new root's left tree. */
1603  lefttreemax->b_right = root;
1604  lefttreemax = root;
1605  } else {
1606  break;
1607  }
1608  root = y;
1609  }
1610  /* Assemble the new root. */
1611  lefttreemax->b_right = root->b_left;
1612  righttreemin->b_left = root->b_right;
1613  root->b_left = dummy.b_right;
1614  root->b_right = dummy.b_left;
1615  return (root);
1616 }
1617 
1618 static void
1619 buf_vlist_remove(struct buf *bp)
1620 {
1621  struct buf *root;
1622  struct bufv *bv;
1623 
1624  KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
1625  ASSERT_BO_LOCKED(bp->b_bufobj);
1626  KASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) !=
1627  (BX_VNDIRTY|BX_VNCLEAN),
1628  ("buf_vlist_remove: Buf %p is on two lists", bp));
1629  if (bp->b_xflags & BX_VNDIRTY)
1630  bv = &bp->b_bufobj->bo_dirty;
1631  else
1632  bv = &bp->b_bufobj->bo_clean;
1633  if (bp != bv->bv_root) {
1634  root = buf_splay(bp->b_lblkno, bp->b_xflags, bv->bv_root);
1635  KASSERT(root == bp, ("splay lookup failed in remove"));
1636  }
1637  if (bp->b_left == NULL) {
1638  root = bp->b_right;
1639  } else {
1640  root = buf_splay(bp->b_lblkno, bp->b_xflags, bp->b_left);
1641  root->b_right = bp->b_right;
1642  }
1643  bv->bv_root = root;
1644  TAILQ_REMOVE(&bv->bv_hd, bp, b_bobufs);
1645  bv->bv_cnt--;
1646  bp->b_xflags &= ~(BX_VNDIRTY | BX_VNCLEAN);
1647 }
1648 
1649 /*
1650  * Add the buffer to the sorted clean or dirty block list using a
1651  * splay tree algorithm.
1652  *
1653  * NOTE: xflags is passed as a constant, optimizing this inline function!
1654  */
1655 static void
1656 buf_vlist_add(struct buf *bp, struct bufobj *bo, b_xflags_t xflags)
1657 {
1658  struct buf *root;
1659  struct bufv *bv;
1660 
1661  ASSERT_BO_LOCKED(bo);
1662  KASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0,
1663  ("buf_vlist_add: Buf %p has existing xflags %d", bp, bp->b_xflags));
1664  bp->b_xflags |= xflags;
1665  if (xflags & BX_VNDIRTY)
1666  bv = &bo->bo_dirty;
1667  else
1668  bv = &bo->bo_clean;
1669 
1670  root = buf_splay(bp->b_lblkno, bp->b_xflags, bv->bv_root);
1671  if (root == NULL) {
1672  bp->b_left = NULL;
1673  bp->b_right = NULL;
1674  TAILQ_INSERT_TAIL(&bv->bv_hd, bp, b_bobufs);
1675  } else if (bp->b_lblkno < root->b_lblkno ||
1676  (bp->b_lblkno == root->b_lblkno &&
1677  (bp->b_xflags & BX_BKGRDMARKER) < (root->b_xflags & BX_BKGRDMARKER))) {
1678  bp->b_left = root->b_left;
1679  bp->b_right = root;
1680  root->b_left = NULL;
1681  TAILQ_INSERT_BEFORE(root, bp, b_bobufs);
1682  } else {
1683  bp->b_right = root->b_right;
1684  bp->b_left = root;
1685  root->b_right = NULL;
1686  TAILQ_INSERT_AFTER(&bv->bv_hd, root, bp, b_bobufs);
1687  }
1688  bv->bv_cnt++;
1689  bv->bv_root = bp;
1690 }
1691 
1692 /*
1693  * Lookup a buffer using the splay tree. Note that we specifically avoid
1694  * shadow buffers used in background bitmap writes.
1695  *
1696  * This code isn't quite efficient as it could be because we are maintaining
1697  * two sorted lists and do not know which list the block resides in.
1698  *
1699  * During a "make buildworld" the desired buffer is found at one of
1700  * the roots more than 60% of the time. Thus, checking both roots
1701  * before performing either splay eliminates unnecessary splays on the
1702  * first tree splayed.
1703  */
1704 struct buf *
1705 gbincore(struct bufobj *bo, daddr_t lblkno)
1706 {
1707  struct buf *bp;
1708 
1709  ASSERT_BO_LOCKED(bo);
1710  if ((bp = bo->bo_clean.bv_root) != NULL &&
1711  bp->b_lblkno == lblkno && !(bp->b_xflags & BX_BKGRDMARKER))
1712  return (bp);
1713  if ((bp = bo->bo_dirty.bv_root) != NULL &&
1714  bp->b_lblkno == lblkno && !(bp->b_xflags & BX_BKGRDMARKER))
1715  return (bp);
1716  if ((bp = bo->bo_clean.bv_root) != NULL) {
1717  bo->bo_clean.bv_root = bp = buf_splay(lblkno, 0, bp);
1718  if (bp->b_lblkno == lblkno && !(bp->b_xflags & BX_BKGRDMARKER))
1719  return (bp);
1720  }
1721  if ((bp = bo->bo_dirty.bv_root) != NULL) {
1722  bo->bo_dirty.bv_root = bp = buf_splay(lblkno, 0, bp);
1723  if (bp->b_lblkno == lblkno && !(bp->b_xflags & BX_BKGRDMARKER))
1724  return (bp);
1725  }
1726  return (NULL);
1727 }
1728 
1729 /*
1730  * Associate a buffer with a vnode.
1731  */
1732 void
1733 bgetvp(struct vnode *vp, struct buf *bp)
1734 {
1735  struct bufobj *bo;
1736 
1737  bo = &vp->v_bufobj;
1738  ASSERT_BO_LOCKED(bo);
1739  VNASSERT(bp->b_vp == NULL, bp->b_vp, ("bgetvp: not free"));
1740 
1741  CTR3(KTR_BUF, "bgetvp(%p) vp %p flags %X", bp, vp, bp->b_flags);
1742  VNASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0, vp,
1743  ("bgetvp: bp already attached! %p", bp));
1744 
1745  vhold(vp);
1746  if (VFS_NEEDSGIANT(vp->v_mount) || bo->bo_flag & BO_NEEDSGIANT)
1747  bp->b_flags |= B_NEEDSGIANT;
1748  bp->b_vp = vp;
1749  bp->b_bufobj = bo;
1750  /*
1751  * Insert onto list for new vnode.
1752  */
1753  buf_vlist_add(bp, bo, BX_VNCLEAN);
1754 }
1755 
1756 /*
1757  * Disassociate a buffer from a vnode.
1758  */
1759 void
1760 brelvp(struct buf *bp)
1761 {
1762  struct bufobj *bo;
1763  struct vnode *vp;
1764 
1765  CTR3(KTR_BUF, "brelvp(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
1766  KASSERT(bp->b_vp != NULL, ("brelvp: NULL"));
1767 
1768  /*
1769  * Delete from old vnode list, if on one.
1770  */
1771  vp = bp->b_vp; /* XXX */
1772  bo = bp->b_bufobj;
1773  BO_LOCK(bo);
1774  if (bp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN))
1775  buf_vlist_remove(bp);
1776  else
1777  panic("brelvp: Buffer %p not on queue.", bp);
1778  if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) {
1779  bo->bo_flag &= ~BO_ONWORKLST;
1780  mtx_lock(&sync_mtx);
1781  LIST_REMOVE(bo, bo_synclist);
1782  syncer_worklist_len--;
1783  mtx_unlock(&sync_mtx);
1784  }
1785  bp->b_flags &= ~B_NEEDSGIANT;
1786  bp->b_vp = NULL;
1787  bp->b_bufobj = NULL;
1788  BO_UNLOCK(bo);
1789  vdrop(vp);
1790 }
1791 
1792 /*
1793  * Add an item to the syncer work queue.
1794  */
1795 static void
1796 vn_syncer_add_to_worklist(struct bufobj *bo, int delay)
1797 {
1798  int queue, slot;
1799 
1800  ASSERT_BO_LOCKED(bo);
1801 
1802  mtx_lock(&sync_mtx);
1803  if (bo->bo_flag & BO_ONWORKLST)
1804  LIST_REMOVE(bo, bo_synclist);
1805  else {
1806  bo->bo_flag |= BO_ONWORKLST;
1807  syncer_worklist_len++;
1808  }
1809 
1810  if (delay > syncer_maxdelay - 2)
1811  delay = syncer_maxdelay - 2;
1812  slot = (syncer_delayno + delay) & syncer_mask;
1813 
1814  queue = VFS_NEEDSGIANT(bo->__bo_vnode->v_mount) ? WI_GIANTQ :
1815  WI_MPSAFEQ;
1816  LIST_INSERT_HEAD(&syncer_workitem_pending[queue][slot], bo,
1817  bo_synclist);
1818  mtx_unlock(&sync_mtx);
1819 }
1820 
1821 static int
1822 sysctl_vfs_worklist_len(SYSCTL_HANDLER_ARGS)
1823 {
1824  int error, len;
1825 
1826  mtx_lock(&sync_mtx);
1827  len = syncer_worklist_len - sync_vnode_count;
1828  mtx_unlock(&sync_mtx);
1829  error = SYSCTL_OUT(req, &len, sizeof(len));
1830  return (error);
1831 }
1832 
1833 SYSCTL_PROC(_vfs, OID_AUTO, worklist_len, CTLTYPE_INT | CTLFLAG_RD, NULL, 0,
1834  sysctl_vfs_worklist_len, "I", "Syncer thread worklist length");
1835 
1836 static struct proc *updateproc;
1837 static void sched_sync(void);
1838 static struct kproc_desc up_kp = {
1839  "syncer",
1840  sched_sync,
1841  &updateproc
1842 };
1843 SYSINIT(syncer, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, &up_kp);
1844 
1845 static int
1846 sync_vnode(struct synclist *slp, struct bufobj **bo, struct thread *td)
1847 {
1848  struct vnode *vp;
1849  struct mount *mp;
1850 
1851  *bo = LIST_FIRST(slp);
1852  if (*bo == NULL)
1853  return (0);
1854  vp = (*bo)->__bo_vnode; /* XXX */
1855  if (VOP_ISLOCKED(vp) != 0 || VI_TRYLOCK(vp) == 0)
1856  return (1);
1857  /*
1858  * We use vhold in case the vnode does not
1859  * successfully sync. vhold prevents the vnode from
1860  * going away when we unlock the sync_mtx so that
1861  * we can acquire the vnode interlock.
1862  */
1863  vholdl(vp);
1864  mtx_unlock(&sync_mtx);
1865  VI_UNLOCK(vp);
1866  if (vn_start_write(vp, &mp, V_NOWAIT) != 0) {
1867  vdrop(vp);
1868  mtx_lock(&sync_mtx);
1869  return (*bo == LIST_FIRST(slp));
1870  }
1871  vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1872  (void) VOP_FSYNC(vp, MNT_LAZY, td);
1873  VOP_UNLOCK(vp, 0);
1874  vn_finished_write(mp);
1875  BO_LOCK(*bo);
1876  if (((*bo)->bo_flag & BO_ONWORKLST) != 0) {
1877  /*
1878  * Put us back on the worklist. The worklist
1879  * routine will remove us from our current
1880  * position and then add us back in at a later
1881  * position.
1882  */
1883  vn_syncer_add_to_worklist(*bo, syncdelay);
1884  }
1885  BO_UNLOCK(*bo);
1886  vdrop(vp);
1887  mtx_lock(&sync_mtx);
1888  return (0);
1889 }
1890 
1891 /*
1892  * System filesystem synchronizer daemon.
1893  */
1894 static void
1895 sched_sync(void)
1896 {
1897  struct synclist *gnext, *next;
1898  struct synclist *gslp, *slp;
1899  struct bufobj *bo;
1900  long starttime;
1901  struct thread *td = curthread;
1902  int last_work_seen;
1903  int net_worklist_len;
1904  int syncer_final_iter;
1905  int first_printf;
1906  int error;
1907 
1908  last_work_seen = 0;
1909  syncer_final_iter = 0;
1910  first_printf = 1;
1911  syncer_state = SYNCER_RUNNING;
1912  starttime = time_uptime;
1913  td->td_pflags |= TDP_NORUNNINGBUF;
1914 
1915  EVENTHANDLER_REGISTER(shutdown_pre_sync, syncer_shutdown, td->td_proc,
1916  SHUTDOWN_PRI_LAST);
1917 
1918  mtx_lock(&sync_mtx);
1919  for (;;) {
1920  if (syncer_state == SYNCER_FINAL_DELAY &&
1921  syncer_final_iter == 0) {
1922  mtx_unlock(&sync_mtx);
1923  kproc_suspend_check(td->td_proc);
1924  mtx_lock(&sync_mtx);
1925  }
1926  net_worklist_len = syncer_worklist_len - sync_vnode_count;
1927  if (syncer_state != SYNCER_RUNNING &&
1928  starttime != time_uptime) {
1929  if (first_printf) {
1930  printf("\nSyncing disks, vnodes remaining...");
1931  first_printf = 0;
1932  }
1933  printf("%d ", net_worklist_len);
1934  }
1935  starttime = time_uptime;
1936 
1937  /*
1938  * Push files whose dirty time has expired. Be careful
1939  * of interrupt race on slp queue.
1940  *
1941  * Skip over empty worklist slots when shutting down.
1942  */
1943  do {
1944  slp = &syncer_workitem_pending[WI_MPSAFEQ][syncer_delayno];
1945  gslp = &syncer_workitem_pending[WI_GIANTQ][syncer_delayno];
1946  syncer_delayno += 1;
1947  if (syncer_delayno == syncer_maxdelay)
1948  syncer_delayno = 0;
1949  next = &syncer_workitem_pending[WI_MPSAFEQ][syncer_delayno];
1950  gnext = &syncer_workitem_pending[WI_GIANTQ][syncer_delayno];
1951  /*
1952  * If the worklist has wrapped since the
1953  * it was emptied of all but syncer vnodes,
1954  * switch to the FINAL_DELAY state and run
1955  * for one more second.
1956  */
1957  if (syncer_state == SYNCER_SHUTTING_DOWN &&
1958  net_worklist_len == 0 &&
1959  last_work_seen == syncer_delayno) {
1960  syncer_state = SYNCER_FINAL_DELAY;
1961  syncer_final_iter = SYNCER_SHUTDOWN_SPEEDUP;
1962  }
1963  } while (syncer_state != SYNCER_RUNNING && LIST_EMPTY(slp) &&
1964  LIST_EMPTY(gslp) && syncer_worklist_len > 0);
1965 
1966  /*
1967  * Keep track of the last time there was anything
1968  * on the worklist other than syncer vnodes.
1969  * Return to the SHUTTING_DOWN state if any
1970  * new work appears.
1971  */
1972  if (net_worklist_len > 0 || syncer_state == SYNCER_RUNNING)
1973  last_work_seen = syncer_delayno;
1974  if (net_worklist_len > 0 && syncer_state == SYNCER_FINAL_DELAY)
1975  syncer_state = SYNCER_SHUTTING_DOWN;
1976  while (!LIST_EMPTY(slp)) {
1977  error = sync_vnode(slp, &bo, td);
1978  if (error == 1) {
1979  LIST_REMOVE(bo, bo_synclist);
1980  LIST_INSERT_HEAD(next, bo, bo_synclist);
1981  continue;
1982  }
1983 
1984  if (first_printf == 0)
1985  wdog_kern_pat(WD_LASTVAL);
1986 
1987  }
1988  if (!LIST_EMPTY(gslp)) {
1989  mtx_unlock(&sync_mtx);
1990  mtx_lock(&Giant);
1991  mtx_lock(&sync_mtx);
1992  while (!LIST_EMPTY(gslp)) {
1993  error = sync_vnode(gslp, &bo, td);
1994  if (error == 1) {
1995  LIST_REMOVE(bo, bo_synclist);
1996  LIST_INSERT_HEAD(gnext, bo,
1997  bo_synclist);
1998  continue;
1999  }
2000  }
2001  mtx_unlock(&Giant);
2002  }
2003  if (syncer_state == SYNCER_FINAL_DELAY && syncer_final_iter > 0)
2004  syncer_final_iter--;
2005  /*
2006  * The variable rushjob allows the kernel to speed up the
2007  * processing of the filesystem syncer process. A rushjob
2008  * value of N tells the filesystem syncer to process the next
2009  * N seconds worth of work on its queue ASAP. Currently rushjob
2010  * is used by the soft update code to speed up the filesystem
2011  * syncer process when the incore state is getting so far
2012  * ahead of the disk that the kernel memory pool is being
2013  * threatened with exhaustion.
2014  */
2015  if (rushjob > 0) {
2016  rushjob -= 1;
2017  continue;
2018  }
2019  /*
2020  * Just sleep for a short period of time between
2021  * iterations when shutting down to allow some I/O
2022  * to happen.
2023  *
2024  * If it has taken us less than a second to process the
2025  * current work, then wait. Otherwise start right over
2026  * again. We can still lose time if any single round
2027  * takes more than two seconds, but it does not really
2028  * matter as we are just trying to generally pace the
2029  * filesystem activity.
2030  */
2031  if (syncer_state != SYNCER_RUNNING ||
2032  time_uptime == starttime) {
2033  thread_lock(td);
2034  sched_prio(td, PPAUSE);
2035  thread_unlock(td);
2036  }
2037  if (syncer_state != SYNCER_RUNNING)
2038  cv_timedwait(&sync_wakeup, &sync_mtx,
2039  hz / SYNCER_SHUTDOWN_SPEEDUP);
2040  else if (time_uptime == starttime)
2041  cv_timedwait(&sync_wakeup, &sync_mtx, hz);
2042  }
2043 }
2044 
2045 /*
2046  * Request the syncer daemon to speed up its work.
2047  * We never push it to speed up more than half of its
2048  * normal turn time, otherwise it could take over the cpu.
2049  */
2050 int
2051 speedup_syncer(void)
2052 {
2053  int ret = 0;
2054 
2055  mtx_lock(&sync_mtx);
2056  if (rushjob < syncdelay / 2) {
2057  rushjob += 1;
2058  stat_rush_requests += 1;
2059  ret = 1;
2060  }
2061  mtx_unlock(&sync_mtx);
2062  cv_broadcast(&sync_wakeup);
2063  return (ret);
2064 }
2065 
2066 /*
2067  * Tell the syncer to speed up its work and run though its work
2068  * list several times, then tell it to shut down.
2069  */
2070 static void
2071 syncer_shutdown(void *arg, int howto)
2072 {
2073 
2074  if (howto & RB_NOSYNC)
2075  return;
2076  mtx_lock(&sync_mtx);
2077  syncer_state = SYNCER_SHUTTING_DOWN;
2078  rushjob = 0;
2079  mtx_unlock(&sync_mtx);
2080  cv_broadcast(&sync_wakeup);
2081  kproc_shutdown(arg, howto);
2082 }
2083 
2084 /*
2085  * Reassign a buffer from one vnode to another.
2086  * Used to assign file specific control information
2087  * (indirect blocks) to the vnode to which they belong.
2088  */
2089 void
2090 reassignbuf(struct buf *bp)
2091 {
2092  struct vnode *vp;
2093  struct bufobj *bo;
2094  int delay;
2095 #ifdef INVARIANTS
2096  struct bufv *bv;
2097 #endif
2098 
2099  vp = bp->b_vp;
2100  bo = bp->b_bufobj;
2101  ++reassignbufcalls;
2102 
2103  CTR3(KTR_BUF, "reassignbuf(%p) vp %p flags %X",
2104  bp, bp->b_vp, bp->b_flags);
2105  /*
2106  * B_PAGING flagged buffers cannot be reassigned because their vp
2107  * is not fully linked in.
2108  */
2109  if (bp->b_flags & B_PAGING)
2110  panic("cannot reassign paging buffer");
2111 
2112  /*
2113  * Delete from old vnode list, if on one.
2114  */
2115  BO_LOCK(bo);
2116  if (bp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN))
2117  buf_vlist_remove(bp);
2118  else
2119  panic("reassignbuf: Buffer %p not on queue.", bp);
2120  /*
2121  * If dirty, put on list of dirty buffers; otherwise insert onto list
2122  * of clean buffers.
2123  */
2124  if (bp->b_flags & B_DELWRI) {
2125  if ((bo->bo_flag & BO_ONWORKLST) == 0) {
2126  switch (vp->v_type) {
2127  case VDIR:
2128  delay = dirdelay;
2129  break;
2130  case VCHR:
2131  delay = metadelay;
2132  break;
2133  default:
2134  delay = filedelay;
2135  }
2136  vn_syncer_add_to_worklist(bo, delay);
2137  }
2138  buf_vlist_add(bp, bo, BX_VNDIRTY);
2139  } else {
2140  buf_vlist_add(bp, bo, BX_VNCLEAN);
2141 
2142  if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) {
2143  mtx_lock(&sync_mtx);
2144  LIST_REMOVE(bo, bo_synclist);
2145  syncer_worklist_len--;
2146  mtx_unlock(&sync_mtx);
2147  bo->bo_flag &= ~BO_ONWORKLST;
2148  }
2149  }
2150 #ifdef INVARIANTS
2151  bv = &bo->bo_clean;
2152  bp = TAILQ_FIRST(&bv->bv_hd);
2153  KASSERT(bp == NULL || bp->b_bufobj == bo,
2154  ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2155  bp = TAILQ_LAST(&bv->bv_hd, buflists);
2156  KASSERT(bp == NULL || bp->b_bufobj == bo,
2157  ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2158  bv = &bo->bo_dirty;
2159  bp = TAILQ_FIRST(&bv->bv_hd);
2160  KASSERT(bp == NULL || bp->b_bufobj == bo,
2161  ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2162  bp = TAILQ_LAST(&bv->bv_hd, buflists);
2163  KASSERT(bp == NULL || bp->b_bufobj == bo,
2164  ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2165 #endif
2166  BO_UNLOCK(bo);
2167 }
2168 
2169 /*
2170  * Increment the use and hold counts on the vnode, taking care to reference
2171  * the driver's usecount if this is a chardev. The vholdl() will remove
2172  * the vnode from the free list if it is presently free. Requires the
2173  * vnode interlock and returns with it held.
2174  */
2175 static void
2176 v_incr_usecount(struct vnode *vp)
2177 {
2178 
2179  CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2180  vp->v_usecount++;
2181  if (vp->v_type == VCHR && vp->v_rdev != NULL) {
2182  dev_lock();
2183  vp->v_rdev->si_usecount++;
2184  dev_unlock();
2185  }
2186  vholdl(vp);
2187 }
2188 
2189 /*
2190  * Turn a holdcnt into a use+holdcnt such that only one call to
2191  * v_decr_usecount is needed.
2192  */
2193 static void
2194 v_upgrade_usecount(struct vnode *vp)
2195 {
2196 
2197  CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2198  vp->v_usecount++;
2199  if (vp->v_type == VCHR && vp->v_rdev != NULL) {
2200  dev_lock();
2201  vp->v_rdev->si_usecount++;
2202  dev_unlock();
2203  }
2204 }
2205 
2206 /*
2207  * Decrement the vnode use and hold count along with the driver's usecount
2208  * if this is a chardev. The vdropl() below releases the vnode interlock
2209  * as it may free the vnode.
2210  */
2211 static void
2212 v_decr_usecount(struct vnode *vp)
2213 {
2214 
2215  ASSERT_VI_LOCKED(vp, __FUNCTION__);
2216  VNASSERT(vp->v_usecount > 0, vp,
2217  ("v_decr_usecount: negative usecount"));
2218  CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2219  vp->v_usecount--;
2220  if (vp->v_type == VCHR && vp->v_rdev != NULL) {
2221  dev_lock();
2222  vp->v_rdev->si_usecount--;
2223  dev_unlock();
2224  }
2225  vdropl(vp);
2226 }
2227 
2228 /*
2229  * Decrement only the use count and driver use count. This is intended to
2230  * be paired with a follow on vdropl() to release the remaining hold count.
2231  * In this way we may vgone() a vnode with a 0 usecount without risk of
2232  * having it end up on a free list because the hold count is kept above 0.
2233  */
2234 static void
2235 v_decr_useonly(struct vnode *vp)
2236 {
2237 
2238  ASSERT_VI_LOCKED(vp, __FUNCTION__);
2239  VNASSERT(vp->v_usecount > 0, vp,
2240  ("v_decr_useonly: negative usecount"));
2241  CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2242  vp->v_usecount--;
2243  if (vp->v_type == VCHR && vp->v_rdev != NULL) {
2244  dev_lock();
2245  vp->v_rdev->si_usecount--;
2246  dev_unlock();
2247  }
2248 }
2249 
2250 /*
2251  * Grab a particular vnode from the free list, increment its
2252  * reference count and lock it. VI_DOOMED is set if the vnode
2253  * is being destroyed. Only callers who specify LK_RETRY will
2254  * see doomed vnodes. If inactive processing was delayed in
2255  * vput try to do it here.
2256  */
2257 int
2258 vget(struct vnode *vp, int flags, struct thread *td)
2259 {
2260  int error;
2261 
2262  error = 0;
2263  VFS_ASSERT_GIANT(vp->v_mount);
2264  VNASSERT((flags & LK_TYPE_MASK) != 0, vp,
2265  ("vget: invalid lock operation"));
2266  CTR3(KTR_VFS, "%s: vp %p with flags %d", __func__, vp, flags);
2267 
2268  if ((flags & LK_INTERLOCK) == 0)
2269  VI_LOCK(vp);
2270  vholdl(vp);
2271  if ((error = vn_lock(vp, flags | LK_INTERLOCK)) != 0) {
2272  vdrop(vp);
2273  CTR2(KTR_VFS, "%s: impossible to lock vnode %p", __func__,
2274  vp);
2275  return (error);
2276  }
2277  if (vp->v_iflag & VI_DOOMED && (flags & LK_RETRY) == 0)
2278  panic("vget: vn_lock failed to return ENOENT\n");
2279  VI_LOCK(vp);
2280  /* Upgrade our holdcnt to a usecount. */
2281  v_upgrade_usecount(vp);
2282  /*
2283  * We don't guarantee that any particular close will
2284  * trigger inactive processing so just make a best effort
2285  * here at preventing a reference to a removed file. If
2286  * we don't succeed no harm is done.
2287  */
2288  if (vp->v_iflag & VI_OWEINACT) {
2289  if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE &&
2290  (flags & LK_NOWAIT) == 0)
2291  vinactive(vp, td);
2292  vp->v_iflag &= ~VI_OWEINACT;
2293  }
2294  VI_UNLOCK(vp);
2295  return (0);
2296 }
2297 
2298 /*
2299  * Increase the reference count of a vnode.
2300  */
2301 void
2302 vref(struct vnode *vp)
2303 {
2304 
2305  CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2306  VI_LOCK(vp);
2307  v_incr_usecount(vp);
2308  VI_UNLOCK(vp);
2309 }
2310 
2311 /*
2312  * Return reference count of a vnode.
2313  *
2314  * The results of this call are only guaranteed when some mechanism other
2315  * than the VI lock is used to stop other processes from gaining references
2316  * to the vnode. This may be the case if the caller holds the only reference.
2317  * This is also useful when stale data is acceptable as race conditions may
2318  * be accounted for by some other means.
2319  */
2320 int
2321 vrefcnt(struct vnode *vp)
2322 {
2323  int usecnt;
2324 
2325  VI_LOCK(vp);
2326  usecnt = vp->v_usecount;
2327  VI_UNLOCK(vp);
2328 
2329  return (usecnt);
2330 }
2331 
2332 #define VPUTX_VRELE 1
2333 #define VPUTX_VPUT 2
2334 #define VPUTX_VUNREF 3
2335 
2336 static void
2337 vputx(struct vnode *vp, int func)
2338 {
2339  int error;
2340 
2341  KASSERT(vp != NULL, ("vputx: null vp"));
2342  if (func == VPUTX_VUNREF)
2343  ASSERT_VOP_LOCKED(vp, "vunref");
2344  else if (func == VPUTX_VPUT)
2345  ASSERT_VOP_LOCKED(vp, "vput");
2346  else
2347  KASSERT(func == VPUTX_VRELE, ("vputx: wrong func"));
2348  VFS_ASSERT_GIANT(vp->v_mount);
2349  CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2350  VI_LOCK(vp);
2351 
2352  /* Skip this v_writecount check if we're going to panic below. */
2353  VNASSERT(vp->v_writecount < vp->v_usecount || vp->v_usecount < 1, vp,
2354  ("vputx: missed vn_close"));
2355  error = 0;
2356 
2357  if (vp->v_usecount > 1 || ((vp->v_iflag & VI_DOINGINACT) &&
2358  vp->v_usecount == 1)) {
2359  if (func == VPUTX_VPUT)
2360  VOP_UNLOCK(vp, 0);
2361  v_decr_usecount(vp);
2362  return;
2363  }
2364 
2365  if (vp->v_usecount != 1) {
2366  vprint("vputx: negative ref count", vp);
2367  panic("vputx: negative ref cnt");
2368  }
2369  CTR2(KTR_VFS, "%s: return vnode %p to the freelist", __func__, vp);
2370  /*
2371  * We want to hold the vnode until the inactive finishes to
2372  * prevent vgone() races. We drop the use count here and the
2373  * hold count below when we're done.
2374  */
2375  v_decr_useonly(vp);
2376  /*
2377  * We must call VOP_INACTIVE with the node locked. Mark
2378  * as VI_DOINGINACT to avoid recursion.
2379  */
2380  vp->v_iflag |= VI_OWEINACT;
2381  switch (func) {
2382  case VPUTX_VRELE:
2383  error = vn_lock(vp, LK_EXCLUSIVE | LK_INTERLOCK);
2384  VI_LOCK(vp);
2385  break;
2386  case VPUTX_VPUT:
2387  if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) {
2388  error = VOP_LOCK(vp, LK_UPGRADE | LK_INTERLOCK |
2389  LK_NOWAIT);
2390  VI_LOCK(vp);
2391  }
2392  break;
2393  case VPUTX_VUNREF:
2394  if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) {
2395  error = VOP_LOCK(vp, LK_TRYUPGRADE | LK_INTERLOCK);
2396  VI_LOCK(vp);
2397  }
2398  break;
2399  }
2400  if (vp->v_usecount > 0)
2401  vp->v_iflag &= ~VI_OWEINACT;
2402  if (error == 0) {
2403  if (vp->v_iflag & VI_OWEINACT)
2404  vinactive(vp, curthread);
2405  if (func != VPUTX_VUNREF)
2406  VOP_UNLOCK(vp, 0);
2407  }
2408  vdropl(vp);
2409 }
2410 
2411 /*
2412  * Vnode put/release.
2413  * If count drops to zero, call inactive routine and return to freelist.
2414  */
2415 void
2416 vrele(struct vnode *vp)
2417 {
2418 
2419  vputx(vp, VPUTX_VRELE);
2420 }
2421 
2422 /*
2423  * Release an already locked vnode. This give the same effects as
2424  * unlock+vrele(), but takes less time and avoids releasing and
2425  * re-aquiring the lock (as vrele() acquires the lock internally.)
2426  */
2427 void
2428 vput(struct vnode *vp)
2429 {
2430 
2431  vputx(vp, VPUTX_VPUT);
2432 }
2433 
2434 /*
2435  * Release an exclusively locked vnode. Do not unlock the vnode lock.
2436  */
2437 void
2438 vunref(struct vnode *vp)
2439 {
2440 
2441  vputx(vp, VPUTX_VUNREF);
2442 }
2443 
2444 /*
2445  * Somebody doesn't want the vnode recycled.
2446  */
2447 void
2448 vhold(struct vnode *vp)
2449 {
2450 
2451  VI_LOCK(vp);
2452  vholdl(vp);
2453  VI_UNLOCK(vp);
2454 }
2455 
2456 /*
2457  * Increase the hold count and activate if this is the first reference.
2458  */
2459 void
2460 vholdl(struct vnode *vp)
2461 {
2462  struct mount *mp;
2463 
2464  CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2465  vp->v_holdcnt++;
2466  if (!VSHOULDBUSY(vp))
2467  return;
2468  ASSERT_VI_LOCKED(vp, "vholdl");
2469  VNASSERT((vp->v_iflag & VI_FREE) != 0, vp, ("vnode not free"));
2470  VNASSERT(vp->v_op != NULL, vp, ("vholdl: vnode already reclaimed."));
2471  /*
2472  * Remove a vnode from the free list, mark it as in use,
2473  * and put it on the active list.
2474  */
2475  mtx_lock(&vnode_free_list_mtx);
2476  TAILQ_REMOVE(&vnode_free_list, vp, v_actfreelist);
2477  freevnodes--;
2478  vp->v_iflag &= ~(VI_FREE|VI_AGE);
2479  KASSERT((vp->v_iflag & VI_ACTIVE) == 0,
2480  ("Activating already active vnode"));
2481  vp->v_iflag |= VI_ACTIVE;
2482  mp = vp->v_mount;
2483  TAILQ_INSERT_HEAD(&mp->mnt_activevnodelist, vp, v_actfreelist);
2484  mp->mnt_activevnodelistsize++;
2485  mtx_unlock(&vnode_free_list_mtx);
2486 }
2487 
2488 /*
2489  * Note that there is one less who cares about this vnode.
2490  * vdrop() is the opposite of vhold().
2491  */
2492 void
2493 vdrop(struct vnode *vp)
2494 {
2495 
2496  VI_LOCK(vp);
2497  vdropl(vp);
2498 }
2499 
2500 /*
2501  * Drop the hold count of the vnode. If this is the last reference to
2502  * the vnode we place it on the free list unless it has been vgone'd
2503  * (marked VI_DOOMED) in which case we will free it.
2504  */
2505 void
2506 vdropl(struct vnode *vp)
2507 {
2508  struct bufobj *bo;
2509  struct mount *mp;
2510  int active;
2511 
2512  ASSERT_VI_LOCKED(vp, "vdropl");
2513  CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2514  if (vp->v_holdcnt <= 0)
2515  panic("vdrop: holdcnt %d", vp->v_holdcnt);
2516  vp->v_holdcnt--;
2517  if (vp->v_holdcnt > 0) {
2518  VI_UNLOCK(vp);
2519  return;
2520  }
2521  if ((vp->v_iflag & VI_DOOMED) == 0) {
2522  /*
2523  * Mark a vnode as free: remove it from its active list
2524  * and put it up for recycling on the freelist.
2525  */
2526  VNASSERT(vp->v_op != NULL, vp,
2527  ("vdropl: vnode already reclaimed."));
2528  VNASSERT((vp->v_iflag & VI_FREE) == 0, vp,
2529  ("vnode already free"));
2530  VNASSERT(VSHOULDFREE(vp), vp,
2531  ("vdropl: freeing when we shouldn't"));
2532  active = vp->v_iflag & VI_ACTIVE;
2533  vp->v_iflag &= ~VI_ACTIVE;
2534  mp = vp->v_mount;
2535  mtx_lock(&vnode_free_list_mtx);
2536  if (active) {
2537  TAILQ_REMOVE(&mp->mnt_activevnodelist, vp,
2538  v_actfreelist);
2539  mp->mnt_activevnodelistsize--;
2540  }
2541  if (vp->v_iflag & VI_AGE) {
2542  TAILQ_INSERT_HEAD(&vnode_free_list, vp, v_actfreelist);
2543  } else {
2544  TAILQ_INSERT_TAIL(&vnode_free_list, vp, v_actfreelist);
2545  }
2546  freevnodes++;
2547  vp->v_iflag &= ~VI_AGE;
2548  vp->v_iflag |= VI_FREE;
2549  mtx_unlock(&vnode_free_list_mtx);
2550  VI_UNLOCK(vp);
2551  return;
2552  }
2553  /*
2554  * The vnode has been marked for destruction, so free it.
2555  */
2556  CTR2(KTR_VFS, "%s: destroying the vnode %p", __func__, vp);
2557  atomic_subtract_long(&numvnodes, 1);
2558  bo = &vp->v_bufobj;
2559  VNASSERT((vp->v_iflag & VI_FREE) == 0, vp,
2560  ("cleaned vnode still on the free list."));
2561  VNASSERT(vp->v_data == NULL, vp, ("cleaned vnode isn't"));
2562  VNASSERT(vp->v_holdcnt == 0, vp, ("Non-zero hold count"));
2563  VNASSERT(vp->v_usecount == 0, vp, ("Non-zero use count"));
2564  VNASSERT(vp->v_writecount == 0, vp, ("Non-zero write count"));
2565  VNASSERT(bo->bo_numoutput == 0, vp, ("Clean vnode has pending I/O's"));
2566  VNASSERT(bo->bo_clean.bv_cnt == 0, vp, ("cleanbufcnt not 0"));
2567  VNASSERT(bo->bo_clean.bv_root == NULL, vp, ("cleanblkroot not NULL"));
2568  VNASSERT(bo->bo_dirty.bv_cnt == 0, vp, ("dirtybufcnt not 0"));
2569  VNASSERT(bo->bo_dirty.bv_root == NULL, vp, ("dirtyblkroot not NULL"));
2570  VNASSERT(TAILQ_EMPTY(&vp->v_cache_dst), vp, ("vp has namecache dst"));
2571  VNASSERT(LIST_EMPTY(&vp->v_cache_src), vp, ("vp has namecache src"));
2572  VNASSERT(vp->v_cache_dd == NULL, vp, ("vp has namecache for .."));
2573  VI_UNLOCK(vp);
2574 #ifdef MAC
2575  mac_vnode_destroy(vp);
2576 #endif
2577  if (vp->v_pollinfo != NULL)
2578  destroy_vpollinfo(vp->v_pollinfo);
2579 #ifdef INVARIANTS
2580  /* XXX Elsewhere we detect an already freed vnode via NULL v_op. */
2581  vp->v_op = NULL;
2582 #endif
2583  rangelock_destroy(&vp->v_rl);
2584  lockdestroy(vp->v_vnlock);
2585  mtx_destroy(&vp->v_interlock);
2586  mtx_destroy(BO_MTX(bo));
2587  uma_zfree(vnode_zone, vp);
2588 }
2589 
2590 /*
2591  * Call VOP_INACTIVE on the vnode and manage the DOINGINACT and OWEINACT
2592  * flags. DOINGINACT prevents us from recursing in calls to vinactive.
2593  * OWEINACT tracks whether a vnode missed a call to inactive due to a
2594  * failed lock upgrade.
2595  */
2596 void
2597 vinactive(struct vnode *vp, struct thread *td)
2598 {
2599  struct vm_object *obj;
2600 
2601  ASSERT_VOP_ELOCKED(vp, "vinactive");
2602  ASSERT_VI_LOCKED(vp, "vinactive");
2603  VNASSERT((vp->v_iflag & VI_DOINGINACT) == 0, vp,
2604  ("vinactive: recursed on VI_DOINGINACT"));
2605  CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2606  vp->v_iflag |= VI_DOINGINACT;
2607  vp->v_iflag &= ~VI_OWEINACT;
2608  VI_UNLOCK(vp);
2609  /*
2610  * Before moving off the active list, we must be sure that any
2611  * modified pages are on the vnode's dirty list since these will
2612  * no longer be checked once the vnode is on the inactive list.
2613  * Because the vnode vm object keeps a hold reference on the vnode
2614  * if there is at least one resident non-cached page, the vnode
2615  * cannot leave the active list without the page cleanup done.
2616  */
2617  obj = vp->v_object;
2618  if (obj != NULL && (obj->flags & OBJ_MIGHTBEDIRTY) != 0) {
2619  VM_OBJECT_LOCK(obj);
2620  vm_object_page_clean(obj, 0, 0, OBJPC_NOSYNC);
2621  VM_OBJECT_UNLOCK(obj);
2622  }
2623  VOP_INACTIVE(vp, td);
2624  VI_LOCK(vp);
2625  VNASSERT(vp->v_iflag & VI_DOINGINACT, vp,
2626  ("vinactive: lost VI_DOINGINACT"));
2627  vp->v_iflag &= ~VI_DOINGINACT;
2628 }
2629 
2630 /*
2631  * Remove any vnodes in the vnode table belonging to mount point mp.
2632  *
2633  * If FORCECLOSE is not specified, there should not be any active ones,
2634  * return error if any are found (nb: this is a user error, not a
2635  * system error). If FORCECLOSE is specified, detach any active vnodes
2636  * that are found.
2637  *
2638  * If WRITECLOSE is set, only flush out regular file vnodes open for
2639  * writing.
2640  *
2641  * SKIPSYSTEM causes any vnodes marked VV_SYSTEM to be skipped.
2642  *
2643  * `rootrefs' specifies the base reference count for the root vnode
2644  * of this filesystem. The root vnode is considered busy if its
2645  * v_usecount exceeds this value. On a successful return, vflush(, td)
2646  * will call vrele() on the root vnode exactly rootrefs times.
2647  * If the SKIPSYSTEM or WRITECLOSE flags are specified, rootrefs must
2648  * be zero.
2649  */
2650 #ifdef DIAGNOSTIC
2651 static int busyprt = 0; /* print out busy vnodes */
2652 SYSCTL_INT(_debug, OID_AUTO, busyprt, CTLFLAG_RW, &busyprt, 0, "Print out busy vnodes");
2653 #endif
2654 
2655 int
2656 vflush(struct mount *mp, int rootrefs, int flags, struct thread *td)
2657 {
2658  struct vnode *vp, *mvp, *rootvp = NULL;
2659  struct vattr vattr;
2660  int busy = 0, error;
2661 
2662  CTR4(KTR_VFS, "%s: mp %p with rootrefs %d and flags %d", __func__, mp,
2663  rootrefs, flags);
2664  if (rootrefs > 0) {
2665  KASSERT((flags & (SKIPSYSTEM | WRITECLOSE)) == 0,
2666  ("vflush: bad args"));
2667  /*
2668  * Get the filesystem root vnode. We can vput() it
2669  * immediately, since with rootrefs > 0, it won't go away.
2670  */
2671  if ((error = VFS_ROOT(mp, LK_EXCLUSIVE, &rootvp)) != 0) {
2672  CTR2(KTR_VFS, "%s: vfs_root lookup failed with %d",
2673  __func__, error);
2674  return (error);
2675  }
2676  vput(rootvp);
2677  }
2678 loop:
2679  MNT_VNODE_FOREACH_ALL(vp, mp, mvp) {
2680  vholdl(vp);
2681  error = vn_lock(vp, LK_INTERLOCK | LK_EXCLUSIVE);
2682  if (error) {
2683  vdrop(vp);
2684  MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
2685  goto loop;
2686  }
2687  /*
2688  * Skip over a vnodes marked VV_SYSTEM.
2689  */
2690  if ((flags & SKIPSYSTEM) && (vp->v_vflag & VV_SYSTEM)) {
2691  VOP_UNLOCK(vp, 0);
2692  vdrop(vp);
2693  continue;
2694  }
2695  /*
2696  * If WRITECLOSE is set, flush out unlinked but still open
2697  * files (even if open only for reading) and regular file
2698  * vnodes open for writing.
2699  */
2700  if (flags & WRITECLOSE) {
2701  if (vp->v_object != NULL) {
2702  VM_OBJECT_LOCK(vp->v_object);
2703  vm_object_page_clean(vp->v_object, 0, 0, 0);
2704  VM_OBJECT_UNLOCK(vp->v_object);
2705  }
2706  error = VOP_FSYNC(vp, MNT_WAIT, td);
2707  if (error != 0) {
2708  VOP_UNLOCK(vp, 0);
2709  vdrop(vp);
2710  MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
2711  return (error);
2712  }
2713  error = VOP_GETATTR(vp, &vattr, td->td_ucred);
2714  VI_LOCK(vp);
2715 
2716  if ((vp->v_type == VNON ||
2717  (error == 0 && vattr.va_nlink > 0)) &&
2718  (vp->v_writecount == 0 || vp->v_type != VREG)) {
2719  VOP_UNLOCK(vp, 0);
2720  vdropl(vp);
2721  continue;
2722  }
2723  } else
2724  VI_LOCK(vp);
2725  /*
2726  * With v_usecount == 0, all we need to do is clear out the
2727  * vnode data structures and we are done.
2728  *
2729  * If FORCECLOSE is set, forcibly close the vnode.
2730  */
2731  if (vp->v_usecount == 0 || (flags & FORCECLOSE)) {
2732  VNASSERT(vp->v_usecount == 0 ||
2733  (vp->v_type != VCHR && vp->v_type != VBLK), vp,
2734  ("device VNODE %p is FORCECLOSED", vp));
2735  vgonel(vp);
2736  } else {
2737  busy++;
2738 #ifdef DIAGNOSTIC
2739  if (busyprt)
2740  vprint("vflush: busy vnode", vp);
2741 #endif
2742  }
2743  VOP_UNLOCK(vp, 0);
2744  vdropl(vp);
2745  }
2746  if (rootrefs > 0 && (flags & FORCECLOSE) == 0) {
2747  /*
2748  * If just the root vnode is busy, and if its refcount
2749  * is equal to `rootrefs', then go ahead and kill it.
2750  */
2751  VI_LOCK(rootvp);
2752  KASSERT(busy > 0, ("vflush: not busy"));
2753  VNASSERT(rootvp->v_usecount >= rootrefs, rootvp,
2754  ("vflush: usecount %d < rootrefs %d",
2755  rootvp->v_usecount, rootrefs));
2756  if (busy == 1 && rootvp->v_usecount == rootrefs) {
2757  VOP_LOCK(rootvp, LK_EXCLUSIVE|LK_INTERLOCK);
2758  vgone(rootvp);
2759  VOP_UNLOCK(rootvp, 0);
2760  busy = 0;
2761  } else
2762  VI_UNLOCK(rootvp);
2763  }
2764  if (busy) {
2765  CTR2(KTR_VFS, "%s: failing as %d vnodes are busy", __func__,
2766  busy);
2767  return (EBUSY);
2768  }
2769  for (; rootrefs > 0; rootrefs--)
2770  vrele(rootvp);
2771  return (0);
2772 }
2773 
2774 /*
2775  * Recycle an unused vnode to the front of the free list.
2776  */
2777 int
2778 vrecycle(struct vnode *vp, struct thread *td)
2779 {
2780  int recycled;
2781 
2782  ASSERT_VOP_ELOCKED(vp, "vrecycle");
2783  CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2784  recycled = 0;
2785  VI_LOCK(vp);
2786  if (vp->v_usecount == 0) {
2787  recycled = 1;
2788  vgonel(vp);
2789  }
2790  VI_UNLOCK(vp);
2791  return (recycled);
2792 }
2793 
2794 /*
2795  * Eliminate all activity associated with a vnode
2796  * in preparation for reuse.
2797  */
2798 void
2799 vgone(struct vnode *vp)
2800 {
2801  VI_LOCK(vp);
2802  vgonel(vp);
2803  VI_UNLOCK(vp);
2804 }
2805 
2806 static void
2807 notify_lowervp_vfs_dummy(struct mount *mp __unused,
2808  struct vnode *lowervp __unused)
2809 {
2810 }
2811 
2812 /*
2813  * Notify upper mounts about reclaimed or unlinked vnode.
2814  */
2815 void
2816 vfs_notify_upper(struct vnode *vp, int event)
2817 {
2818  static struct vfsops vgonel_vfsops = {
2819  .vfs_reclaim_lowervp = notify_lowervp_vfs_dummy,
2820  .vfs_unlink_lowervp = notify_lowervp_vfs_dummy,
2821  };
2822  struct mount *mp, *ump, *mmp;
2823 
2824  mp = vp->v_mount;
2825  if (mp == NULL)
2826  return;
2827 
2828  MNT_ILOCK(mp);
2829  if (TAILQ_EMPTY(&mp->mnt_uppers))
2830  goto unlock;
2831  MNT_IUNLOCK(mp);
2832  mmp = malloc(sizeof(struct mount), M_TEMP, M_WAITOK | M_ZERO);
2833  mmp->mnt_op = &vgonel_vfsops;
2834  mmp->mnt_kern_flag |= MNTK_MARKER;
2835  MNT_ILOCK(mp);
2836  mp->mnt_kern_flag |= MNTK_VGONE_UPPER;
2837  for (ump = TAILQ_FIRST(&mp->mnt_uppers); ump != NULL;) {
2838  if ((ump->mnt_kern_flag & MNTK_MARKER) != 0) {
2839  ump = TAILQ_NEXT(ump, mnt_upper_link);
2840  continue;
2841  }
2842  TAILQ_INSERT_AFTER(&mp->mnt_uppers, ump, mmp, mnt_upper_link);
2843  MNT_IUNLOCK(mp);
2844  switch (event) {
2845  case VFS_NOTIFY_UPPER_RECLAIM:
2846  VFS_RECLAIM_LOWERVP(ump, vp);
2847  break;
2848  case VFS_NOTIFY_UPPER_UNLINK:
2849  VFS_UNLINK_LOWERVP(ump, vp);
2850  break;
2851  default:
2852  KASSERT(0, ("invalid event %d", event));
2853  break;
2854  }
2855  MNT_ILOCK(mp);
2856  ump = TAILQ_NEXT(mmp, mnt_upper_link);
2857  TAILQ_REMOVE(&mp->mnt_uppers, mmp, mnt_upper_link);
2858  }
2859  free(mmp, M_TEMP);
2860  mp->mnt_kern_flag &= ~MNTK_VGONE_UPPER;
2861  if ((mp->mnt_kern_flag & MNTK_VGONE_WAITER) != 0) {
2862  mp->mnt_kern_flag &= ~MNTK_VGONE_WAITER;
2863  wakeup(&mp->mnt_uppers);
2864  }
2865 unlock:
2866  MNT_IUNLOCK(mp);
2867 }
2868 
2869 /*
2870  * vgone, with the vp interlock held.
2871  */
2872 void
2873 vgonel(struct vnode *vp)
2874 {
2875  struct thread *td;
2876  int oweinact;
2877  int active;
2878  struct mount *mp;
2879 
2880  ASSERT_VOP_ELOCKED(vp, "vgonel");
2881  ASSERT_VI_LOCKED(vp, "vgonel");
2882  VNASSERT(vp->v_holdcnt, vp,
2883  ("vgonel: vp %p has no reference.", vp));
2884  CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
2885  td = curthread;
2886 
2887  /*
2888  * Don't vgonel if we're already doomed.
2889  */
2890  if (vp->v_iflag & VI_DOOMED)
2891  return;
2892  vp->v_iflag |= VI_DOOMED;
2893 
2894  /*
2895  * Check to see if the vnode is in use. If so, we have to call
2896  * VOP_CLOSE() and VOP_INACTIVE().
2897  */
2898  active = vp->v_usecount;
2899  oweinact = (vp->v_iflag & VI_OWEINACT);
2900  VI_UNLOCK(vp);
2901  vfs_notify_upper(vp, VFS_NOTIFY_UPPER_RECLAIM);
2902 
2903  /*
2904  * Clean out any buffers associated with the vnode.
2905  * If the flush fails, just toss the buffers.
2906  */
2907  mp = NULL;
2908  if (!TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd))
2909  (void) vn_start_secondary_write(vp, &mp, V_WAIT);
2910  if (vinvalbuf(vp, V_SAVE, 0, 0) != 0)
2911  vinvalbuf(vp, 0, 0, 0);
2912 
2913  /*
2914  * If purging an active vnode, it must be closed and
2915  * deactivated before being reclaimed.
2916  */
2917  if (active)
2918  VOP_CLOSE(vp, FNONBLOCK, NOCRED, td);
2919  if (oweinact || active) {
2920  VI_LOCK(vp);
2921  if ((vp->v_iflag & VI_DOINGINACT) == 0)
2922  vinactive(vp, td);
2923  VI_UNLOCK(vp);
2924  }
2925  if (vp->v_type == VSOCK)
2926  vfs_unp_reclaim(vp);
2927  /*
2928  * Reclaim the vnode.
2929  */
2930  if (VOP_RECLAIM(vp, td))
2931  panic("vgone: cannot reclaim");
2932  if (mp != NULL)
2933  vn_finished_secondary_write(mp);
2934  VNASSERT(vp->v_object == NULL, vp,
2935  ("vop_reclaim left v_object vp=%p, tag=%s", vp, vp->v_tag));
2936  /*
2937  * Clear the advisory locks and wake up waiting threads.
2938  */
2939  (void)VOP_ADVLOCKPURGE(vp);
2940  /*
2941  * Delete from old mount point vnode list.
2942  */
2943  delmntque(vp);
2944  cache_purge(vp);
2945  /*
2946  * Done with purge, reset to the standard lock and invalidate
2947  * the vnode.
2948  */
2949  VI_LOCK(vp);
2950  vp->v_vnlock = &vp->v_lock;
2951  vp->v_op = &dead_vnodeops;
2952  vp->v_tag = "none";
2953  vp->v_type = VBAD;
2954 }
2955 
2956 /*
2957  * Calculate the total number of references to a special device.
2958  */
2959 int
2960 vcount(struct vnode *vp)
2961 {
2962  int count;
2963 
2964  dev_lock();
2965  count = vp->v_rdev->si_usecount;
2966  dev_unlock();
2967  return (count);
2968 }
2969 
2970 /*
2971  * Same as above, but using the struct cdev *as argument
2972  */
2973 int
2974 count_dev(struct cdev *dev)
2975 {
2976  int count;
2977 
2978  dev_lock();
2979  count = dev->si_usecount;
2980  dev_unlock();
2981  return(count);
2982 }
2983 
2984 /*
2985  * Print out a description of a vnode.
2986  */
2987 static char *typename[] =
2988 {"VNON", "VREG", "VDIR", "VBLK", "VCHR", "VLNK", "VSOCK", "VFIFO", "VBAD",
2989  "VMARKER"};
2990 
2991 void
2992 vn_printf(struct vnode *vp, const char *fmt, ...)
2993 {
2994  va_list ap;
2995  char buf[256], buf2[16];
2996  u_long flags;
2997 
2998  va_start(ap, fmt);
2999  vprintf(fmt, ap);
3000  va_end(ap);
3001  printf("%p: ", (void *)vp);
3002  printf("tag %s, type %s\n", vp->v_tag, typename[vp->v_type]);
3003  printf(" usecount %d, writecount %d, refcount %d mountedhere %p\n",
3004  vp->v_usecount, vp->v_writecount, vp->v_holdcnt, vp->v_mountedhere);
3005  buf[0] = '\0';
3006  buf[1] = '\0';
3007  if (vp->v_vflag & VV_ROOT)
3008  strlcat(buf, "|VV_ROOT", sizeof(buf));
3009  if (vp->v_vflag & VV_ISTTY)
3010  strlcat(buf, "|VV_ISTTY", sizeof(buf));
3011  if (vp->v_vflag & VV_NOSYNC)
3012  strlcat(buf, "|VV_NOSYNC", sizeof(buf));
3013  if (vp->v_vflag & VV_ETERNALDEV)
3014  strlcat(buf, "|VV_ETERNALDEV", sizeof(buf));
3015  if (vp->v_vflag & VV_CACHEDLABEL)
3016  strlcat(buf, "|VV_CACHEDLABEL", sizeof(buf));
3017  if (vp->v_vflag & VV_TEXT)
3018  strlcat(buf, "|VV_TEXT", sizeof(buf));
3019  if (vp->v_vflag & VV_COPYONWRITE)
3020  strlcat(buf, "|VV_COPYONWRITE", sizeof(buf));
3021  if (vp->v_vflag & VV_SYSTEM)
3022  strlcat(buf, "|VV_SYSTEM", sizeof(buf));
3023  if (vp->v_vflag & VV_PROCDEP)
3024  strlcat(buf, "|VV_PROCDEP", sizeof(buf));
3025  if (vp->v_vflag & VV_NOKNOTE)
3026  strlcat(buf, "|VV_NOKNOTE", sizeof(buf));
3027  if (vp->v_vflag & VV_DELETED)
3028  strlcat(buf, "|VV_DELETED", sizeof(buf));
3029  if (vp->v_vflag & VV_MD)
3030  strlcat(buf, "|VV_MD", sizeof(buf));
3031  if (vp->v_vflag & VV_FORCEINSMQ)
3032  strlcat(buf, "|VV_FORCEINSMQ", sizeof(buf));
3033  flags = vp->v_vflag & ~(VV_ROOT | VV_ISTTY | VV_NOSYNC | VV_ETERNALDEV |
3034  VV_CACHEDLABEL | VV_TEXT | VV_COPYONWRITE | VV_SYSTEM | VV_PROCDEP |
3035  VV_NOKNOTE | VV_DELETED | VV_MD | VV_FORCEINSMQ);
3036  if (flags != 0) {
3037  snprintf(buf2, sizeof(buf2), "|VV(0x%lx)", flags);
3038  strlcat(buf, buf2, sizeof(buf));
3039  }
3040  if (vp->v_iflag & VI_MOUNT)
3041  strlcat(buf, "|VI_MOUNT", sizeof(buf));
3042  if (vp->v_iflag & VI_AGE)
3043  strlcat(buf, "|VI_AGE", sizeof(buf));
3044  if (vp->v_iflag & VI_DOOMED)
3045  strlcat(buf, "|VI_DOOMED", sizeof(buf));
3046  if (vp->v_iflag & VI_FREE)
3047  strlcat(buf, "|VI_FREE", sizeof(buf));
3048  if (vp->v_iflag & VI_ACTIVE)
3049  strlcat(buf, "|VI_ACTIVE", sizeof(buf));
3050  if (vp->v_iflag & VI_DOINGINACT)
3051  strlcat(buf, "|VI_DOINGINACT", sizeof(buf));
3052  if (vp->v_iflag & VI_OWEINACT)
3053  strlcat(buf, "|VI_OWEINACT", sizeof(buf));
3054  flags = vp->v_iflag & ~(VI_MOUNT | VI_AGE | VI_DOOMED | VI_FREE |
3055  VI_ACTIVE | VI_DOINGINACT | VI_OWEINACT);
3056  if (flags != 0) {
3057  snprintf(buf2, sizeof(buf2), "|VI(0x%lx)", flags);
3058  strlcat(buf, buf2, sizeof(buf));
3059  }
3060  printf(" flags (%s)\n", buf + 1);
3061  if (mtx_owned(VI_MTX(vp)))
3062  printf(" VI_LOCKed");
3063  if (vp->v_object != NULL)
3064  printf(" v_object %p ref %d pages %d "
3065  "cleanbuf %d dirtybuf %d\n",
3066  vp->v_object, vp->v_object->ref_count,
3067  vp->v_object->resident_page_count,
3068  vp->v_bufobj.bo_dirty.bv_cnt,
3069  vp->v_bufobj.bo_clean.bv_cnt);
3070  printf(" ");
3071  lockmgr_printinfo(vp->v_vnlock);
3072  if (vp->v_data != NULL)
3073  VOP_PRINT(vp);
3074 }
3075 
3076 #ifdef DDB
3077 /*
3078  * List all of the locked vnodes in the system.
3079  * Called when debugging the kernel.
3080  */
3081 DB_SHOW_COMMAND(lockedvnods, lockedvnodes)
3082 {
3083  struct mount *mp, *nmp;
3084  struct vnode *vp;
3085 
3086  /*
3087  * Note: because this is DDB, we can't obey the locking semantics
3088  * for these structures, which means we could catch an inconsistent
3089  * state and dereference a nasty pointer. Not much to be done
3090  * about that.
3091  */
3092  db_printf("Locked vnodes\n");
3093  for (mp = TAILQ_FIRST(&mountlist); mp != NULL; mp = nmp) {
3094  nmp = TAILQ_NEXT(mp, mnt_list);
3095  TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
3096  if (vp->v_type != VMARKER &&
3097  VOP_ISLOCKED(vp))
3098  vprint("", vp);
3099  }
3100  nmp = TAILQ_NEXT(mp, mnt_list);
3101  }
3102 }
3103 
3104 /*
3105  * Show details about the given vnode.
3106  */
3107 DB_SHOW_COMMAND(vnode, db_show_vnode)
3108 {
3109  struct vnode *vp;
3110 
3111  if (!have_addr)
3112  return;
3113  vp = (struct vnode *)addr;
3114  vn_printf(vp, "vnode ");
3115 }
3116 
3117 /*
3118  * Show details about the given mount point.
3119  */
3120 DB_SHOW_COMMAND(mount, db_show_mount)
3121 {
3122  struct mount *mp;
3123  struct vfsopt *opt;
3124  struct statfs *sp;
3125  struct vnode *vp;
3126  char buf[512];
3127  uint64_t mflags;
3128  u_int flags;
3129 
3130  if (!have_addr) {
3131  /* No address given, print short info about all mount points. */
3132  TAILQ_FOREACH(mp, &mountlist, mnt_list) {
3133  db_printf("%p %s on %s (%s)\n", mp,
3134  mp->mnt_stat.f_mntfromname,
3135  mp->mnt_stat.f_mntonname,
3136  mp->mnt_stat.f_fstypename);
3137  if (db_pager_quit)
3138  break;
3139  }
3140  db_printf("\nMore info: show mount <addr>\n");
3141  return;
3142  }
3143 
3144  mp = (struct mount *)addr;
3145  db_printf("%p %s on %s (%s)\n", mp, mp->mnt_stat.f_mntfromname,
3146  mp->mnt_stat.f_mntonname, mp->mnt_stat.f_fstypename);
3147 
3148  buf[0] = '\0';
3149  mflags = mp->mnt_flag;
3150 #define MNT_FLAG(flag) do { \
3151  if (mflags & (flag)) { \
3152  if (buf[0] != '\0') \
3153  strlcat(buf, ", ", sizeof(buf)); \
3154  strlcat(buf, (#flag) + 4, sizeof(buf)); \
3155  mflags &= ~(flag); \
3156  } \
3157 } while (0)
3158  MNT_FLAG(MNT_RDONLY);
3159  MNT_FLAG(MNT_SYNCHRONOUS);
3160  MNT_FLAG(MNT_NOEXEC);
3161  MNT_FLAG(MNT_NOSUID);
3162  MNT_FLAG(MNT_NFS4ACLS);
3163  MNT_FLAG(MNT_UNION);
3164  MNT_FLAG(MNT_ASYNC);
3165  MNT_FLAG(MNT_SUIDDIR);
3166  MNT_FLAG(MNT_SOFTDEP);
3167  MNT_FLAG(MNT_NOSYMFOLLOW);
3168  MNT_FLAG(MNT_GJOURNAL);
3169  MNT_FLAG(MNT_MULTILABEL);
3170  MNT_FLAG(MNT_ACLS);
3171  MNT_FLAG(MNT_NOATIME);
3172  MNT_FLAG(MNT_NOCLUSTERR);
3173  MNT_FLAG(MNT_NOCLUSTERW);
3174  MNT_FLAG(MNT_SUJ);
3175  MNT_FLAG(MNT_EXRDONLY);
3176  MNT_FLAG(MNT_EXPORTED);
3177  MNT_FLAG(MNT_DEFEXPORTED);
3178  MNT_FLAG(MNT_EXPORTANON);
3179  MNT_FLAG(MNT_EXKERB);
3180  MNT_FLAG(MNT_EXPUBLIC);
3181  MNT_FLAG(MNT_LOCAL);
3182  MNT_FLAG(MNT_QUOTA);
3183  MNT_FLAG(MNT_ROOTFS);
3184  MNT_FLAG(MNT_USER);
3185  MNT_FLAG(MNT_IGNORE);
3186  MNT_FLAG(MNT_UPDATE);
3187  MNT_FLAG(MNT_DELEXPORT);
3188  MNT_FLAG(MNT_RELOAD);
3189  MNT_FLAG(MNT_FORCE);
3190  MNT_FLAG(MNT_SNAPSHOT);
3191  MNT_FLAG(MNT_BYFSID);
3192 #undef MNT_FLAG
3193  if (mflags != 0) {
3194  if (buf[0] != '\0')
3195  strlcat(buf, ", ", sizeof(buf));
3196  snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf),
3197  "0x%016jx", mflags);
3198  }
3199  db_printf(" mnt_flag = %s\n", buf);
3200 
3201  buf[0] = '\0';
3202  flags = mp->mnt_kern_flag;
3203 #define MNT_KERN_FLAG(flag) do { \
3204  if (flags & (flag)) { \
3205  if (buf[0] != '\0') \
3206  strlcat(buf, ", ", sizeof(buf)); \
3207  strlcat(buf, (#flag) + 5, sizeof(buf)); \
3208  flags &= ~(flag); \
3209  } \
3210 } while (0)
3211  MNT_KERN_FLAG(MNTK_UNMOUNTF);
3212  MNT_KERN_FLAG(MNTK_ASYNC);
3213  MNT_KERN_FLAG(MNTK_SOFTDEP);
3214  MNT_KERN_FLAG(MNTK_NOINSMNTQ);
3215  MNT_KERN_FLAG(MNTK_DRAINING);
3216  MNT_KERN_FLAG(MNTK_REFEXPIRE);
3217  MNT_KERN_FLAG(MNTK_EXTENDED_SHARED);
3218  MNT_KERN_FLAG(MNTK_SHARED_WRITES);
3219  MNT_KERN_FLAG(MNTK_NO_IOPF);
3220  MNT_KERN_FLAG(MNTK_VGONE_UPPER);
3221  MNT_KERN_FLAG(MNTK_VGONE_WAITER);
3222  MNT_KERN_FLAG(MNTK_LOOKUP_EXCL_DOTDOT);
3223  MNT_KERN_FLAG(MNTK_MARKER);
3224  MNT_KERN_FLAG(MNTK_NOASYNC);
3225  MNT_KERN_FLAG(MNTK_UNMOUNT);
3226  MNT_KERN_FLAG(MNTK_MWAIT);
3227  MNT_KERN_FLAG(MNTK_SUSPEND);
3228  MNT_KERN_FLAG(MNTK_SUSPEND2);
3229  MNT_KERN_FLAG(MNTK_SUSPENDED);
3230  MNT_KERN_FLAG(MNTK_MPSAFE);
3231  MNT_KERN_FLAG(MNTK_LOOKUP_SHARED);
3232  MNT_KERN_FLAG(MNTK_NOKNOTE);
3233 #undef MNT_KERN_FLAG
3234  if (flags != 0) {
3235  if (buf[0] != '\0')
3236  strlcat(buf, ", ", sizeof(buf));
3237  snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf),
3238  "0x%08x", flags);
3239  }
3240  db_printf(" mnt_kern_flag = %s\n", buf);
3241 
3242  db_printf(" mnt_opt = ");
3243  opt = TAILQ_FIRST(mp->mnt_opt);
3244  if (opt != NULL) {
3245  db_printf("%s", opt->name);
3246  opt = TAILQ_NEXT(opt, link);
3247  while (opt != NULL) {
3248  db_printf(", %s", opt->name);
3249  opt = TAILQ_NEXT(opt, link);
3250  }
3251  }
3252  db_printf("\n");
3253 
3254  sp = &mp->mnt_stat;
3255  db_printf(" mnt_stat = { version=%u type=%u flags=0x%016jx "
3256  "bsize=%ju iosize=%ju blocks=%ju bfree=%ju bavail=%jd files=%ju "
3257  "ffree=%jd syncwrites=%ju asyncwrites=%ju syncreads=%ju "
3258  "asyncreads=%ju namemax=%u owner=%u fsid=[%d, %d] }\n",
3259  (u_int)sp->f_version, (u_int)sp->f_type, (uintmax_t)sp->f_flags,
3260  (uintmax_t)sp->f_bsize, (uintmax_t)sp->f_iosize,
3261  (uintmax_t)sp->f_blocks, (uintmax_t)sp->f_bfree,
3262  (intmax_t)sp->f_bavail, (uintmax_t)sp->f_files,
3263  (intmax_t)sp->f_ffree, (uintmax_t)sp->f_syncwrites,
3264  (uintmax_t)sp->f_asyncwrites, (uintmax_t)sp->f_syncreads,
3265  (uintmax_t)sp->f_asyncreads, (u_int)sp->f_namemax,
3266  (u_int)sp->f_owner, (int)sp->f_fsid.val[0], (int)sp->f_fsid.val[1]);
3267 
3268  db_printf(" mnt_cred = { uid=%u ruid=%u",
3269  (u_int)mp->mnt_cred->cr_uid, (u_int)mp->mnt_cred->cr_ruid);
3270  if (jailed(mp->mnt_cred))
3271  db_printf(", jail=%d", mp->mnt_cred->cr_prison->pr_id);
3272  db_printf(" }\n");
3273  db_printf(" mnt_ref = %d\n", mp->mnt_ref);
3274  db_printf(" mnt_gen = %d\n", mp->mnt_gen);
3275  db_printf(" mnt_nvnodelistsize = %d\n", mp->mnt_nvnodelistsize);
3276  db_printf(" mnt_activevnodelistsize = %d\n",
3277  mp->mnt_activevnodelistsize);
3278  db_printf(" mnt_writeopcount = %d\n", mp->mnt_writeopcount);
3279  db_printf(" mnt_maxsymlinklen = %d\n", mp->mnt_maxsymlinklen);
3280  db_printf(" mnt_iosize_max = %d\n", mp->mnt_iosize_max);
3281  db_printf(" mnt_hashseed = %u\n", mp->mnt_hashseed);
3282  db_printf(" mnt_lockref = %d\n", mp->mnt_lockref);
3283  db_printf(" mnt_secondary_writes = %d\n", mp->mnt_secondary_writes);
3284  db_printf(" mnt_secondary_accwrites = %d\n",
3285  mp->mnt_secondary_accwrites);
3286  db_printf(" mnt_gjprovider = %s\n",
3287  mp->mnt_gjprovider != NULL ? mp->mnt_gjprovider : "NULL");
3288 
3289  db_printf("\n\nList of active vnodes\n");
3290  TAILQ_FOREACH(vp, &mp->mnt_activevnodelist, v_actfreelist) {
3291  if (vp->v_type != VMARKER) {
3292  vn_printf(vp, "vnode ");
3293  if (db_pager_quit)
3294  break;
3295  }
3296  }
3297  db_printf("\n\nList of inactive vnodes\n");
3298  TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
3299  if (vp->v_type != VMARKER && (vp->v_iflag & VI_ACTIVE) == 0) {
3300  vn_printf(vp, "vnode ");
3301  if (db_pager_quit)
3302  break;
3303  }
3304  }
3305 }
3306 #endif /* DDB */
3307 
3308 /*
3309  * Fill in a struct xvfsconf based on a struct vfsconf.
3310  */
3311 static int
3312 vfsconf2x(struct sysctl_req *req, struct vfsconf *vfsp)
3313 {
3314  struct xvfsconf xvfsp;
3315 
3316  bzero(&xvfsp, sizeof(xvfsp));
3317  strcpy(xvfsp.vfc_name, vfsp->vfc_name);
3318  xvfsp.vfc_typenum = vfsp->vfc_typenum;
3319  xvfsp.vfc_refcount = vfsp->vfc_refcount;
3320  xvfsp.vfc_flags = vfsp->vfc_flags;
3321  /*
3322  * These are unused in userland, we keep them
3323  * to not break binary compatibility.
3324  */
3325  xvfsp.vfc_vfsops = NULL;
3326  xvfsp.vfc_next = NULL;
3327  return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp)));
3328 }
3329 
3330 #ifdef COMPAT_32BIT
3331 struct xvfsconf32 {
3332  uint32_t vfc_vfsops;
3333  char vfc_name[MFSNAMELEN];
3334  int32_t vfc_typenum;
3335  int32_t vfc_refcount;
3336  int32_t vfc_flags;
3337  uint32_t vfc_next;
3338 };
3339 
3340 static int
3341 vfsconf2x32(struct sysctl_req *req, struct vfsconf *vfsp)
3342 {
3343  struct xvfsconf32 xvfsp;
3344 
3345  strcpy(xvfsp.vfc_name, vfsp->vfc_name);
3346  xvfsp.vfc_typenum = vfsp->vfc_typenum;
3347  xvfsp.vfc_refcount = vfsp->vfc_refcount;
3348  xvfsp.vfc_flags = vfsp->vfc_flags;
3349  xvfsp.vfc_vfsops = 0;
3350  xvfsp.vfc_next = 0;
3351  return (SYSCTL_OUT(req, &xvfsp, sizeof(xvfsp)));
3352 }
3353 #endif
3354 
3355 /*
3356  * Top level filesystem related information gathering.
3357  */
3358 static int
3359 sysctl_vfs_conflist(SYSCTL_HANDLER_ARGS)
3360 {
3361  struct vfsconf *vfsp;
3362  int error;
3363 
3364  error = 0;
3365  TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) {
3366 #ifdef COMPAT_32BIT
3367  if (req->flags & SCTL_MASK32)
3368  error = vfsconf2x32(req, vfsp);
3369  else
3370 #endif
3371  error = vfsconf2x(req, vfsp);
3372  if (error)
3373  break;
3374  }
3375  return (error);
3376 }
3377 
3378 SYSCTL_PROC(_vfs, OID_AUTO, conflist, CTLTYPE_OPAQUE | CTLFLAG_RD,
3379  NULL, 0, sysctl_vfs_conflist,
3380  "S,xvfsconf", "List of all configured filesystems");
3381 
3382 #ifndef BURN_BRIDGES
3383 static int sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS);
3384 
3385 static int
3386 vfs_sysctl(SYSCTL_HANDLER_ARGS)
3387 {
3388  int *name = (int *)arg1 - 1; /* XXX */
3389  u_int namelen = arg2 + 1; /* XXX */
3390  struct vfsconf *vfsp;
3391 
3392  printf("WARNING: userland calling deprecated sysctl, "
3393  "please rebuild world\n");
3394 
3395 #if 1 || defined(COMPAT_PRELITE2)
3396  /* Resolve ambiguity between VFS_VFSCONF and VFS_GENERIC. */
3397  if (namelen == 1)
3398  return (sysctl_ovfs_conf(oidp, arg1, arg2, req));
3399 #endif
3400 
3401  switch (name[1]) {
3402  case VFS_MAXTYPENUM:
3403  if (namelen != 2)
3404  return (ENOTDIR);
3405  return (SYSCTL_OUT(req, &maxvfsconf, sizeof(int)));
3406  case VFS_CONF:
3407  if (namelen != 3)
3408  return (ENOTDIR); /* overloaded */
3409  TAILQ_FOREACH(vfsp, &vfsconf, vfc_list)
3410  if (vfsp->vfc_typenum == name[2])
3411  break;
3412  if (vfsp == NULL)
3413  return (EOPNOTSUPP);
3414 #ifdef COMPAT_32BIT
3415  if (req->flags & SCTL_MASK32)
3416  return (vfsconf2x32(req, vfsp));
3417  else
3418 #endif
3419  return (vfsconf2x(req, vfsp));
3420  }
3421  return (EOPNOTSUPP);
3422 }
3423 
3424 static SYSCTL_NODE(_vfs, VFS_GENERIC, generic, CTLFLAG_RD | CTLFLAG_SKIP,
3425  vfs_sysctl, "Generic filesystem");
3426 
3427 #if 1 || defined(COMPAT_PRELITE2)
3428 
3429 static int
3430 sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS)
3431 {
3432  int error;
3433  struct vfsconf *vfsp;
3434  struct ovfsconf ovfs;
3435 
3436  TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) {
3437  bzero(&ovfs, sizeof(ovfs));
3438  ovfs.vfc_vfsops = vfsp->vfc_vfsops; /* XXX used as flag */
3439  strcpy(ovfs.vfc_name, vfsp->vfc_name);
3440  ovfs.vfc_index = vfsp->vfc_typenum;
3441  ovfs.vfc_refcount = vfsp->vfc_refcount;
3442  ovfs.vfc_flags = vfsp->vfc_flags;
3443  error = SYSCTL_OUT(req, &ovfs, sizeof ovfs);
3444  if (error)
3445  return error;
3446  }
3447  return 0;
3448 }
3449 
3450 #endif /* 1 || COMPAT_PRELITE2 */
3451 #endif /* !BURN_BRIDGES */
3452 
3453 #define KINFO_VNODESLOP 10
3454 #ifdef notyet
3455 /*
3456  * Dump vnode list (via sysctl).
3457  */
3458 /* ARGSUSED */
3459 static int
3460 sysctl_vnode(SYSCTL_HANDLER_ARGS)
3461 {
3462  struct xvnode *xvn;
3463  struct mount *mp;
3464  struct vnode *vp;
3465  int error, len, n;
3466 
3467  /*
3468  * Stale numvnodes access is not fatal here.
3469  */
3470  req->lock = 0;
3471  len = (numvnodes + KINFO_VNODESLOP) * sizeof *xvn;
3472  if (!req->oldptr)
3473  /* Make an estimate */
3474  return (SYSCTL_OUT(req, 0, len));
3475 
3476  error = sysctl_wire_old_buffer(req, 0);
3477  if (error != 0)
3478  return (error);
3479  xvn = malloc(len, M_TEMP, M_ZERO | M_WAITOK);
3480  n = 0;
3481  mtx_lock(&mountlist_mtx);
3482  TAILQ_FOREACH(mp, &mountlist, mnt_list) {
3483  if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK))
3484  continue;
3485  MNT_ILOCK(mp);
3486  TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
3487  if (n == len)
3488  break;
3489  vref(vp);
3490  xvn[n].xv_size = sizeof *xvn;
3491  xvn[n].xv_vnode = vp;
3492  xvn[n].xv_id = 0; /* XXX compat */
3493 #define XV_COPY(field) xvn[n].xv_##field = vp->v_##field
3494  XV_COPY(usecount);
3495  XV_COPY(writecount);
3496  XV_COPY(holdcnt);
3497  XV_COPY(mount);
3498  XV_COPY(numoutput);
3499  XV_COPY(type);
3500 #undef XV_COPY
3501  xvn[n].xv_flag = vp->v_vflag;
3502 
3503  switch (vp->v_type) {
3504  case VREG:
3505  case VDIR:
3506  case VLNK:
3507  break;
3508  case VBLK:
3509  case VCHR:
3510  if (vp->v_rdev == NULL) {
3511  vrele(vp);
3512  continue;
3513  }
3514  xvn[n].xv_dev = dev2udev(vp->v_rdev);
3515  break;
3516  case VSOCK:
3517  xvn[n].xv_socket = vp->v_socket;
3518  break;
3519  case VFIFO:
3520  xvn[n].xv_fifo = vp->v_fifoinfo;
3521  break;
3522  case VNON:
3523  case VBAD:
3524  default:
3525  /* shouldn't happen? */
3526  vrele(vp);
3527  continue;
3528  }
3529  vrele(vp);
3530  ++n;
3531  }
3532  MNT_IUNLOCK(mp);
3533  mtx_lock(&mountlist_mtx);
3534  vfs_unbusy(mp);
3535  if (n == len)
3536  break;
3537  }
3538  mtx_unlock(&mountlist_mtx);
3539 
3540  error = SYSCTL_OUT(req, xvn, n * sizeof *xvn);
3541  free(xvn, M_TEMP);
3542  return (error);
3543 }
3544 
3545 SYSCTL_PROC(_kern, KERN_VNODE, vnode, CTLTYPE_OPAQUE|CTLFLAG_RD,
3546  0, 0, sysctl_vnode, "S,xvnode", "");
3547 #endif
3548 
3549 /*
3550  * Unmount all filesystems. The list is traversed in reverse order
3551  * of mounting to avoid dependencies.
3552  */
3553 void
3554 vfs_unmountall(void)
3555 {
3556  struct mount *mp;
3557  struct thread *td;
3558  int error;
3559 
3560  CTR1(KTR_VFS, "%s: unmounting all filesystems", __func__);
3561  td = curthread;
3562 
3563  /*
3564  * Since this only runs when rebooting, it is not interlocked.
3565  */
3566  while(!TAILQ_EMPTY(&mountlist)) {
3567  mp = TAILQ_LAST(&mountlist, mntlist);
3568  error = dounmount(mp, MNT_FORCE, td);
3569  if (error) {
3570  TAILQ_REMOVE(&mountlist, mp, mnt_list);
3571  /*
3572  * XXX: Due to the way in which we mount the root
3573  * file system off of devfs, devfs will generate a
3574  * "busy" warning when we try to unmount it before
3575  * the root. Don't print a warning as a result in
3576  * order to avoid false positive errors that may
3577  * cause needless upset.
3578  */
3579  if (strcmp(mp->mnt_vfc->vfc_name, "devfs") != 0) {
3580  printf("unmount of %s failed (",
3581  mp->mnt_stat.f_mntonname);
3582  if (error == EBUSY)
3583  printf("BUSY)\n");
3584  else
3585  printf("%d)\n", error);
3586  }
3587  } else {
3588  /* The unmount has removed mp from the mountlist */
3589  }
3590  }
3591 }
3592 
3593 /*
3594  * perform msync on all vnodes under a mount point
3595  * the mount point must be locked.
3596  */
3597 void
3598 vfs_msync(struct mount *mp, int flags)
3599 {
3600  struct vnode *vp, *mvp;
3601  struct vm_object *obj;
3602 
3603  CTR2(KTR_VFS, "%s: mp %p", __func__, mp);
3604  MNT_VNODE_FOREACH_ACTIVE(vp, mp, mvp) {
3605  obj = vp->v_object;
3606  if (obj != NULL && (obj->flags & OBJ_MIGHTBEDIRTY) != 0 &&
3607  (flags == MNT_WAIT || VOP_ISLOCKED(vp) == 0)) {
3608  if (!vget(vp,
3609  LK_EXCLUSIVE | LK_RETRY | LK_INTERLOCK,
3610  curthread)) {
3611  if (vp->v_vflag & VV_NOSYNC) { /* unlinked */
3612  vput(vp);
3613  continue;
3614  }
3615 
3616  obj = vp->v_object;
3617  if (obj != NULL) {
3618  VM_OBJECT_LOCK(obj);
3619  vm_object_page_clean(obj, 0, 0,
3620  flags == MNT_WAIT ?
3621  OBJPC_SYNC : OBJPC_NOSYNC);
3622  VM_OBJECT_UNLOCK(obj);
3623  }
3624  vput(vp);
3625  }
3626  } else
3627  VI_UNLOCK(vp);
3628  }
3629 }
3630 
3631 static void
3632 destroy_vpollinfo_free(struct vpollinfo *vi)
3633 {
3634 
3635  knlist_destroy(&vi->vpi_selinfo.si_note);
3636  mtx_destroy(&vi->vpi_lock);
3637  uma_zfree(vnodepoll_zone, vi);
3638 }
3639 
3640 static void
3641 destroy_vpollinfo(struct vpollinfo *vi)
3642 {
3643 
3644  knlist_clear(&vi->vpi_selinfo.si_note, 1);
3645  seldrain(&vi->vpi_selinfo);
3646  destroy_vpollinfo_free(vi);
3647 }
3648 
3649 /*
3650  * Initalize per-vnode helper structure to hold poll-related state.
3651  */
3652 void
3653 v_addpollinfo(struct vnode *vp)
3654 {
3655  struct vpollinfo *vi;
3656 
3657  if (vp->v_pollinfo != NULL)
3658  return;
3659  vi = uma_zalloc(vnodepoll_zone, M_WAITOK);
3660  mtx_init(&vi->vpi_lock, "vnode pollinfo", NULL, MTX_DEF);
3661  knlist_init(&vi->vpi_selinfo.si_note, vp, vfs_knllock,
3662  vfs_knlunlock, vfs_knl_assert_locked, vfs_knl_assert_unlocked);
3663  VI_LOCK(vp);
3664  if (vp->v_pollinfo != NULL) {
3665  VI_UNLOCK(vp);
3666  destroy_vpollinfo_free(vi);
3667  return;
3668  }
3669  vp->v_pollinfo = vi;
3670  VI_UNLOCK(vp);
3671 }
3672 
3673 /*
3674  * Record a process's interest in events which might happen to
3675  * a vnode. Because poll uses the historic select-style interface
3676  * internally, this routine serves as both the ``check for any
3677  * pending events'' and the ``record my interest in future events''
3678  * functions. (These are done together, while the lock is held,
3679  * to avoid race conditions.)
3680  */
3681 int
3682 vn_pollrecord(struct vnode *vp, struct thread *td, int events)
3683 {
3684 
3685  v_addpollinfo(vp);
3686  mtx_lock(&vp->v_pollinfo->vpi_lock);
3687  if (vp->v_pollinfo->vpi_revents & events) {
3688  /*
3689  * This leaves events we are not interested
3690  * in available for the other process which
3691  * which presumably had requested them
3692  * (otherwise they would never have been
3693  * recorded).
3694  */
3695  events &= vp->v_pollinfo->vpi_revents;
3696  vp->v_pollinfo->vpi_revents &= ~events;
3697 
3698  mtx_unlock(&vp->v_pollinfo->vpi_lock);
3699  return (events);
3700  }
3701  vp->v_pollinfo->vpi_events |= events;
3702  selrecord(td, &vp->v_pollinfo->vpi_selinfo);
3703  mtx_unlock(&vp->v_pollinfo->vpi_lock);
3704  return (0);
3705 }
3706 
3707 /*
3708  * Routine to create and manage a filesystem syncer vnode.
3709  */
3710 #define sync_close ((int (*)(struct vop_close_args *))nullop)
3711 static int sync_fsync(struct vop_fsync_args *);
3712 static int sync_inactive(struct vop_inactive_args *);
3713 static int sync_reclaim(struct vop_reclaim_args *);
3714 
3715 static struct vop_vector sync_vnodeops = {
3716  .vop_bypass = VOP_EOPNOTSUPP,
3717  .vop_close = sync_close, /* close */
3718  .vop_fsync = sync_fsync, /* fsync */
3719  .vop_inactive = sync_inactive, /* inactive */
3720  .vop_reclaim = sync_reclaim, /* reclaim */
3721  .vop_lock1 = vop_stdlock, /* lock */
3722  .vop_unlock = vop_stdunlock, /* unlock */
3723  .vop_islocked = vop_stdislocked, /* islocked */
3724 };
3725 
3726 /*
3727  * Create a new filesystem syncer vnode for the specified mount point.
3728  */
3729 void
3730 vfs_allocate_syncvnode(struct mount *mp)
3731 {
3732  struct vnode *vp;
3733  struct bufobj *bo;
3734  static long start, incr, next;
3735  int error;
3736 
3737  /* Allocate a new vnode */
3738  error = getnewvnode("syncer", mp, &sync_vnodeops, &vp);
3739  if (error != 0)
3740  panic("vfs_allocate_syncvnode: getnewvnode() failed");
3741  vp->v_type = VNON;
3742  vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
3743  vp->v_vflag |= VV_FORCEINSMQ;
3744  error = insmntque(vp, mp);
3745  if (error != 0)
3746  panic("vfs_allocate_syncvnode: insmntque() failed");
3747  vp->v_vflag &= ~VV_FORCEINSMQ;
3748  VOP_UNLOCK(vp, 0);
3749  /*
3750  * Place the vnode onto the syncer worklist. We attempt to
3751  * scatter them about on the list so that they will go off
3752  * at evenly distributed times even if all the filesystems
3753  * are mounted at once.
3754  */
3755  next += incr;
3756  if (next == 0 || next > syncer_maxdelay) {
3757  start /= 2;
3758  incr /= 2;
3759  if (start == 0) {
3760  start = syncer_maxdelay / 2;
3761  incr = syncer_maxdelay;
3762  }
3763  next = start;
3764  }
3765  bo = &vp->v_bufobj;
3766  BO_LOCK(bo);
3767  vn_syncer_add_to_worklist(bo, syncdelay > 0 ? next % syncdelay : 0);
3768  /* XXX - vn_syncer_add_to_worklist() also grabs and drops sync_mtx. */
3769  mtx_lock(&sync_mtx);
3770  sync_vnode_count++;
3771  if (mp->mnt_syncer == NULL) {
3772  mp->mnt_syncer = vp;
3773  vp = NULL;
3774  }
3775  mtx_unlock(&sync_mtx);
3776  BO_UNLOCK(bo);
3777  if (vp != NULL) {
3778  vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
3779  vgone(vp);
3780  vput(vp);
3781  }
3782 }
3783 
3784 void
3785 vfs_deallocate_syncvnode(struct mount *mp)
3786 {
3787  struct vnode *vp;
3788 
3789  mtx_lock(&sync_mtx);
3790  vp = mp->mnt_syncer;
3791  if (vp != NULL)
3792  mp->mnt_syncer = NULL;
3793  mtx_unlock(&sync_mtx);
3794  if (vp != NULL)
3795  vrele(vp);
3796 }
3797 
3798 /*
3799  * Do a lazy sync of the filesystem.
3800  */
3801 static int
3802 sync_fsync(struct vop_fsync_args *ap)
3803 {
3804  struct vnode *syncvp = ap->a_vp;
3805  struct mount *mp = syncvp->v_mount;
3806  int error, save;
3807  struct bufobj *bo;
3808 
3809  /*
3810  * We only need to do something if this is a lazy evaluation.
3811  */
3812  if (ap->a_waitfor != MNT_LAZY)
3813  return (0);
3814 
3815  /*
3816  * Move ourselves to the back of the sync list.
3817  */
3818  bo = &syncvp->v_bufobj;
3819  BO_LOCK(bo);
3820  vn_syncer_add_to_worklist(bo, syncdelay);
3821  BO_UNLOCK(bo);
3822 
3823  /*
3824  * Walk the list of vnodes pushing all that are dirty and
3825  * not already on the sync list.
3826  */
3827  if (vfs_busy(mp, MBF_NOWAIT) != 0)
3828  return (0);
3829  if (vn_start_write(NULL, &mp, V_NOWAIT) != 0) {
3830  vfs_unbusy(mp);
3831  return (0);
3832  }
3833  save = curthread_pflags_set(TDP_SYNCIO);
3834  vfs_msync(mp, MNT_NOWAIT);
3835  error = VFS_SYNC(mp, MNT_LAZY);
3836  curthread_pflags_restore(save);
3837  vn_finished_write(mp);
3838  vfs_unbusy(mp);
3839  return (error);
3840 }
3841 
3842 /*
3843  * The syncer vnode is no referenced.
3844  */
3845 static int
3846 sync_inactive(struct vop_inactive_args *ap)
3847 {
3848 
3849  vgone(ap->a_vp);
3850  return (0);
3851 }
3852 
3853 /*
3854  * The syncer vnode is no longer needed and is being decommissioned.
3855  *
3856  * Modifications to the worklist must be protected by sync_mtx.
3857  */
3858 static int
3859 sync_reclaim(struct vop_reclaim_args *ap)
3860 {
3861  struct vnode *vp = ap->a_vp;
3862  struct bufobj *bo;
3863 
3864  bo = &vp->v_bufobj;
3865  BO_LOCK(bo);
3866  mtx_lock(&sync_mtx);
3867  if (vp->v_mount->mnt_syncer == vp)
3868  vp->v_mount->mnt_syncer = NULL;
3869  if (bo->bo_flag & BO_ONWORKLST) {
3870  LIST_REMOVE(bo, bo_synclist);
3871  syncer_worklist_len--;
3872  sync_vnode_count--;
3873  bo->bo_flag &= ~BO_ONWORKLST;
3874  }
3875  mtx_unlock(&sync_mtx);
3876  BO_UNLOCK(bo);
3877 
3878  return (0);
3879 }
3880 
3881 /*
3882  * Check if vnode represents a disk device
3883  */
3884 int
3885 vn_isdisk(struct vnode *vp, int *errp)
3886 {
3887  int error;
3888 
3889  error = 0;
3890  dev_lock();
3891  if (vp->v_type != VCHR)
3892  error = ENOTBLK;
3893  else if (vp->v_rdev == NULL)
3894  error = ENXIO;
3895  else if (vp->v_rdev->si_devsw == NULL)
3896  error = ENXIO;
3897  else if (!(vp->v_rdev->si_devsw->d_flags & D_DISK))
3898  error = ENOTBLK;
3899  dev_unlock();
3900  if (errp != NULL)
3901  *errp = error;
3902  return (error == 0);
3903 }
3904 
3905 /*
3906  * Common filesystem object access control check routine. Accepts a
3907  * vnode's type, "mode", uid and gid, requested access mode, credentials,
3908  * and optional call-by-reference privused argument allowing vaccess()
3909  * to indicate to the caller whether privilege was used to satisfy the
3910  * request (obsoleted). Returns 0 on success, or an errno on failure.
3911  */
3912 int
3913 vaccess(enum vtype type, mode_t file_mode, uid_t file_uid, gid_t file_gid,
3914  accmode_t accmode, struct ucred *cred, int *privused)
3915 {
3916  accmode_t dac_granted;
3917  accmode_t priv_granted;
3918 
3919  KASSERT((accmode & ~(VEXEC | VWRITE | VREAD | VADMIN | VAPPEND)) == 0,
3920  ("invalid bit in accmode"));
3921  KASSERT((accmode & VAPPEND) == 0 || (accmode & VWRITE),
3922  ("VAPPEND without VWRITE"));
3923 
3924  /*
3925  * Look for a normal, non-privileged way to access the file/directory
3926  * as requested. If it exists, go with that.
3927  */
3928 
3929  if (privused != NULL)
3930  *privused = 0;
3931 
3932  dac_granted = 0;
3933 
3934  /* Check the owner. */
3935  if (cred->cr_uid == file_uid) {
3936  dac_granted |= VADMIN;
3937  if (file_mode & S_IXUSR)
3938  dac_granted |= VEXEC;
3939  if (file_mode & S_IRUSR)
3940  dac_granted |= VREAD;
3941  if (file_mode & S_IWUSR)
3942  dac_granted |= (VWRITE | VAPPEND);
3943 
3944  if ((accmode & dac_granted) == accmode)
3945  return (0);
3946 
3947  goto privcheck;
3948  }
3949 
3950  /* Otherwise, check the groups (first match) */
3951  if (groupmember(file_gid, cred)) {
3952  if (file_mode & S_IXGRP)
3953  dac_granted |= VEXEC;
3954  if (file_mode & S_IRGRP)
3955  dac_granted |= VREAD;
3956  if (file_mode & S_IWGRP)
3957  dac_granted |= (VWRITE | VAPPEND);
3958 
3959  if ((accmode & dac_granted) == accmode)
3960  return (0);
3961 
3962  goto privcheck;
3963  }
3964 
3965  /* Otherwise, check everyone else. */
3966  if (file_mode & S_IXOTH)
3967  dac_granted |= VEXEC;
3968  if (file_mode & S_IROTH)
3969  dac_granted |= VREAD;
3970  if (file_mode & S_IWOTH)
3971  dac_granted |= (VWRITE | VAPPEND);
3972  if ((accmode & dac_granted) == accmode)
3973  return (0);
3974 
3975 privcheck:
3976  /*
3977  * Build a privilege mask to determine if the set of privileges
3978  * satisfies the requirements when combined with the granted mask
3979  * from above. For each privilege, if the privilege is required,
3980  * bitwise or the request type onto the priv_granted mask.
3981  */
3982  priv_granted = 0;
3983 
3984  if (type == VDIR) {
3985  /*
3986  * For directories, use PRIV_VFS_LOOKUP to satisfy VEXEC
3987  * requests, instead of PRIV_VFS_EXEC.
3988  */
3989  if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) &&
3990  !priv_check_cred(cred, PRIV_VFS_LOOKUP, 0))
3991  priv_granted |= VEXEC;
3992  } else {
3993  /*
3994  * Ensure that at least one execute bit is on. Otherwise,
3995  * a privileged user will always succeed, and we don't want
3996  * this to happen unless the file really is executable.
3997  */
3998  if ((accmode & VEXEC) && ((dac_granted & VEXEC) == 0) &&
3999  (file_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) != 0 &&
4000  !priv_check_cred(cred, PRIV_VFS_EXEC, 0))
4001  priv_granted |= VEXEC;
4002  }
4003 
4004  if ((accmode & VREAD) && ((dac_granted & VREAD) == 0) &&
4005  !priv_check_cred(cred, PRIV_VFS_READ, 0))
4006  priv_granted |= VREAD;
4007 
4008  if ((accmode & VWRITE) && ((dac_granted & VWRITE) == 0) &&
4009  !priv_check_cred(cred, PRIV_VFS_WRITE, 0))
4010  priv_granted |= (VWRITE | VAPPEND);
4011 
4012  if ((accmode & VADMIN) && ((dac_granted & VADMIN) == 0) &&
4013  !priv_check_cred(cred, PRIV_VFS_ADMIN, 0))
4014  priv_granted |= VADMIN;
4015 
4016  if ((accmode & (priv_granted | dac_granted)) == accmode) {
4017  /* XXX audit: privilege used */
4018  if (privused != NULL)
4019  *privused = 1;
4020  return (0);
4021  }
4022 
4023  return ((accmode & VADMIN) ? EPERM : EACCES);
4024 }
4025 
4026 /*
4027  * Credential check based on process requesting service, and per-attribute
4028  * permissions.
4029  */
4030 int
4031 extattr_check_cred(struct vnode *vp, int attrnamespace, struct ucred *cred,
4032  struct thread *td, accmode_t accmode)
4033 {
4034 
4035  /*
4036  * Kernel-invoked always succeeds.
4037  */
4038  if (cred == NOCRED)
4039  return (0);
4040 
4041  /*
4042  * Do not allow privileged processes in jail to directly manipulate
4043  * system attributes.
4044  */
4045  switch (attrnamespace) {
4046  case EXTATTR_NAMESPACE_SYSTEM:
4047  /* Potentially should be: return (EPERM); */
4048  return (priv_check_cred(cred, PRIV_VFS_EXTATTR_SYSTEM, 0));
4049  case EXTATTR_NAMESPACE_USER:
4050  return (VOP_ACCESS(vp, accmode, cred, td));
4051  default:
4052  return (EPERM);
4053  }
4054 }
4055 
4056 #ifdef DEBUG_VFS_LOCKS
4057 /*
4058  * This only exists to supress warnings from unlocked specfs accesses. It is
4059  * no longer ok to have an unlocked VFS.
4060  */
4061 #define IGNORE_LOCK(vp) (panicstr != NULL || (vp) == NULL || \
4062  (vp)->v_type == VCHR || (vp)->v_type == VBAD)
4063 
4064 int vfs_badlock_ddb = 1; /* Drop into debugger on violation. */
4065 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_ddb, CTLFLAG_RW, &vfs_badlock_ddb, 0,
4066  "Drop into debugger on lock violation");
4067 
4068 int vfs_badlock_mutex = 1; /* Check for interlock across VOPs. */
4069 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_mutex, CTLFLAG_RW, &vfs_badlock_mutex,
4070  0, "Check for interlock across VOPs");
4071 
4072 int vfs_badlock_print = 1; /* Print lock violations. */
4073 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_print, CTLFLAG_RW, &vfs_badlock_print,
4074  0, "Print lock violations");
4075 
4076 #ifdef KDB
4077 int vfs_badlock_backtrace = 1; /* Print backtrace at lock violations. */
4078 SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_backtrace, CTLFLAG_RW,
4079  &vfs_badlock_backtrace, 0, "Print backtrace at lock violations");
4080 #endif
4081 
4082 static void
4083 vfs_badlock(const char *msg, const char *str, struct vnode *vp)
4084 {
4085 
4086 #ifdef KDB
4087  if (vfs_badlock_backtrace)
4088  kdb_backtrace();
4089 #endif
4090  if (vfs_badlock_print)
4091  printf("%s: %p %s\n", str, (void *)vp, msg);
4092  if (vfs_badlock_ddb)
4093  kdb_enter(KDB_WHY_VFSLOCK, "lock violation");
4094 }
4095 
4096 void
4097 assert_vi_locked(struct vnode *vp, const char *str)
4098 {
4099 
4100  if (vfs_badlock_mutex && !mtx_owned(VI_MTX(vp)))
4101  vfs_badlock("interlock is not locked but should be", str, vp);
4102 }
4103 
4104 void
4105 assert_vi_unlocked(struct vnode *vp, const char *str)
4106 {
4107 
4108  if (vfs_badlock_mutex && mtx_owned(VI_MTX(vp)))
4109  vfs_badlock("interlock is locked but should not be", str, vp);
4110 }
4111 
4112 void
4113 assert_vop_locked(struct vnode *vp, const char *str)
4114 {
4115  int locked;
4116 
4117  if (!IGNORE_LOCK(vp)) {
4118  locked = VOP_ISLOCKED(vp);
4119  if (locked == 0 || locked == LK_EXCLOTHER)
4120  vfs_badlock("is not locked but should be", str, vp);
4121  }
4122 }
4123 
4124 void
4125 assert_vop_unlocked(struct vnode *vp, const char *str)
4126 {
4127 
4128  if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) == LK_EXCLUSIVE)
4129  vfs_badlock("is locked but should not be", str, vp);
4130 }
4131 
4132 void
4133 assert_vop_elocked(struct vnode *vp, const char *str)
4134 {
4135 
4136  if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) != LK_EXCLUSIVE)
4137  vfs_badlock("is not exclusive locked but should be", str, vp);
4138 }
4139 
4140 #if 0
4141 void
4142 assert_vop_elocked_other(struct vnode *vp, const char *str)
4143 {
4144 
4145  if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) != LK_EXCLOTHER)
4146  vfs_badlock("is not exclusive locked by another thread",
4147  str, vp);
4148 }
4149 
4150 void
4151 assert_vop_slocked(struct vnode *vp, const char *str)
4152 {
4153 
4154  if (!IGNORE_LOCK(vp) && VOP_ISLOCKED(vp) != LK_SHARED)
4155  vfs_badlock("is not locked shared but should be", str, vp);
4156 }
4157 #endif /* 0 */
4158 #endif /* DEBUG_VFS_LOCKS */
4159 
4160 void
4161 vop_rename_fail(struct vop_rename_args *ap)
4162 {
4163 
4164  if (ap->a_tvp != NULL)
4165  vput(ap->a_tvp);
4166  if (ap->a_tdvp == ap->a_tvp)
4167  vrele(ap->a_tdvp);
4168  else
4169  vput(ap->a_tdvp);
4170  vrele(ap->a_fdvp);
4171  vrele(ap->a_fvp);
4172 }
4173 
4174 void
4175 vop_rename_pre(void *ap)
4176 {
4177  struct vop_rename_args *a = ap;
4178 
4179 #ifdef DEBUG_VFS_LOCKS
4180  if (a->a_tvp)
4181  ASSERT_VI_UNLOCKED(a->a_tvp, "VOP_RENAME");
4182  ASSERT_VI_UNLOCKED(a->a_tdvp, "VOP_RENAME");
4183  ASSERT_VI_UNLOCKED(a->a_fvp, "VOP_RENAME");
4184  ASSERT_VI_UNLOCKED(a->a_fdvp, "VOP_RENAME");
4185 
4186  /* Check the source (from). */
4187  if (a->a_tdvp->v_vnlock != a->a_fdvp->v_vnlock &&
4188  (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fdvp->v_vnlock))
4189  ASSERT_VOP_UNLOCKED(a->a_fdvp, "vop_rename: fdvp locked");
4190  if (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fvp->v_vnlock)
4191  ASSERT_VOP_UNLOCKED(a->a_fvp, "vop_rename: fvp locked");
4192 
4193  /* Check the target. */
4194  if (a->a_tvp)
4195  ASSERT_VOP_LOCKED(a->a_tvp, "vop_rename: tvp not locked");
4196  ASSERT_VOP_LOCKED(a->a_tdvp, "vop_rename: tdvp not locked");
4197 #endif
4198  if (a->a_tdvp != a->a_fdvp)
4199  vhold(a->a_fdvp);
4200  if (a->a_tvp != a->a_fvp)
4201  vhold(a->a_fvp);
4202  vhold(a->a_tdvp);
4203  if (a->a_tvp)
4204  vhold(a->a_tvp);
4205 }
4206 
4207 void
4208 vop_strategy_pre(void *ap)
4209 {
4210 #ifdef DEBUG_VFS_LOCKS
4211  struct vop_strategy_args *a;
4212  struct buf *bp;
4213 
4214  a = ap;
4215  bp = a->a_bp;
4216 
4217  /*
4218  * Cluster ops lock their component buffers but not the IO container.
4219  */
4220  if ((bp->b_flags & B_CLUSTER) != 0)
4221  return;
4222 
4223  if (panicstr == NULL && !BUF_ISLOCKED(bp)) {
4224  if (vfs_badlock_print)
4225  printf(
4226  "VOP_STRATEGY: bp is not locked but should be\n");
4227  if (vfs_badlock_ddb)
4228  kdb_enter(KDB_WHY_VFSLOCK, "lock violation");
4229  }
4230 #endif
4231 }
4232 
4233 void
4234 vop_lock_pre(void *ap)
4235 {
4236 #ifdef DEBUG_VFS_LOCKS
4237  struct vop_lock1_args *a = ap;
4238 
4239  if ((a->a_flags & LK_INTERLOCK) == 0)
4240  ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK");
4241  else
4242  ASSERT_VI_LOCKED(a->a_vp, "VOP_LOCK");
4243 #endif
4244 }
4245 
4246 void
4247 vop_lock_post(void *ap, int rc)
4248 {
4249 #ifdef DEBUG_VFS_LOCKS
4250  struct vop_lock1_args *a = ap;
4251 
4252  ASSERT_VI_UNLOCKED(a->a_vp, "VOP_LOCK");
4253  if (rc == 0 && (a->a_flags & LK_EXCLOTHER) == 0)
4254  ASSERT_VOP_LOCKED(a->a_vp, "VOP_LOCK");
4255 #endif
4256 }
4257 
4258 void
4259 vop_unlock_pre(void *ap)
4260 {
4261 #ifdef DEBUG_VFS_LOCKS
4262  struct vop_unlock_args *a = ap;
4263 
4264  if (a->a_flags & LK_INTERLOCK)
4265  ASSERT_VI_LOCKED(a->a_vp, "VOP_UNLOCK");
4266  ASSERT_VOP_LOCKED(a->a_vp, "VOP_UNLOCK");
4267 #endif
4268 }
4269 
4270 void
4271 vop_unlock_post(void *ap, int rc)
4272 {
4273 #ifdef DEBUG_VFS_LOCKS
4274  struct vop_unlock_args *a = ap;
4275 
4276  if (a->a_flags & LK_INTERLOCK)
4277  ASSERT_VI_UNLOCKED(a->a_vp, "VOP_UNLOCK");
4278 #endif
4279 }
4280 
4281 void
4282 vop_create_post(void *ap, int rc)
4283 {
4284  struct vop_create_args *a = ap;
4285 
4286  if (!rc)
4287  VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE);
4288 }
4289 
4290 void
4291 vop_deleteextattr_post(void *ap, int rc)
4292 {
4293  struct vop_deleteextattr_args *a = ap;
4294 
4295  if (!rc)
4296  VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB);
4297 }
4298 
4299 void
4300 vop_link_post(void *ap, int rc)
4301 {
4302  struct vop_link_args *a = ap;
4303 
4304  if (!rc) {
4305  VFS_KNOTE_LOCKED(a->a_vp, NOTE_LINK);
4306  VFS_KNOTE_LOCKED(a->a_tdvp, NOTE_WRITE);
4307  }
4308 }
4309 
4310 void
4311 vop_mkdir_post(void *ap, int rc)
4312 {
4313  struct vop_mkdir_args *a = ap;
4314 
4315  if (!rc)
4316  VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE | NOTE_LINK);
4317 }
4318 
4319 void
4320 vop_mknod_post(void *ap, int rc)
4321 {
4322  struct vop_mknod_args *a = ap;
4323 
4324  if (!rc)
4325  VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE);
4326 }
4327 
4328 void
4329 vop_remove_post(void *ap, int rc)
4330 {
4331  struct vop_remove_args *a = ap;
4332 
4333  if (!rc) {
4334  VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE);
4335  VFS_KNOTE_LOCKED(a->a_vp, NOTE_DELETE);
4336  }
4337 }
4338 
4339 void
4340 vop_rename_post(void *ap, int rc)
4341 {
4342  struct vop_rename_args *a = ap;
4343 
4344  if (!rc) {
4345  VFS_KNOTE_UNLOCKED(a->a_fdvp, NOTE_WRITE);
4346  VFS_KNOTE_UNLOCKED(a->a_tdvp, NOTE_WRITE);
4347  VFS_KNOTE_UNLOCKED(a->a_fvp, NOTE_RENAME);
4348  if (a->a_tvp)
4349  VFS_KNOTE_UNLOCKED(a->a_tvp, NOTE_DELETE);
4350  }
4351  if (a->a_tdvp != a->a_fdvp)
4352  vdrop(a->a_fdvp);
4353  if (a->a_tvp != a->a_fvp)
4354  vdrop(a->a_fvp);
4355  vdrop(a->a_tdvp);
4356  if (a->a_tvp)
4357  vdrop(a->a_tvp);
4358 }
4359 
4360 void
4361 vop_rmdir_post(void *ap, int rc)
4362 {
4363  struct vop_rmdir_args *a = ap;
4364 
4365  if (!rc) {
4366  VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE | NOTE_LINK);
4367  VFS_KNOTE_LOCKED(a->a_vp, NOTE_DELETE);
4368  }
4369 }
4370 
4371 void
4372 vop_setattr_post(void *ap, int rc)
4373 {
4374  struct vop_setattr_args *a = ap;
4375 
4376  if (!rc)
4377  VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB);
4378 }
4379 
4380 void
4381 vop_setextattr_post(void *ap, int rc)
4382 {
4383  struct vop_setextattr_args *a = ap;
4384 
4385  if (!rc)
4386  VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB);
4387 }
4388 
4389 void
4390 vop_symlink_post(void *ap, int rc)
4391 {
4392  struct vop_symlink_args *a = ap;
4393 
4394  if (!rc)
4395  VFS_KNOTE_LOCKED(a->a_dvp, NOTE_WRITE);
4396 }
4397 
4398 static struct knlist fs_knlist;
4399 
4400 static void
4401 vfs_event_init(void *arg)
4402 {
4403  knlist_init_mtx(&fs_knlist, NULL);
4404 }
4405 /* XXX - correct order? */
4406 SYSINIT(vfs_knlist, SI_SUB_VFS, SI_ORDER_ANY, vfs_event_init, NULL);
4407 
4408 void
4409 vfs_event_signal(fsid_t *fsid, uint32_t event, intptr_t data __unused)
4410 {
4411 
4412  KNOTE_UNLOCKED(&fs_knlist, event);
4413 }
4414 
4415 static int filt_fsattach(struct knote *kn);
4416 static void filt_fsdetach(struct knote *kn);
4417 static int filt_fsevent(struct knote *kn, long hint);
4418 
4419 struct filterops fs_filtops = {
4420  .f_isfd = 0,
4421  .f_attach = filt_fsattach,
4422  .f_detach = filt_fsdetach,
4423  .f_event = filt_fsevent
4424 };
4425 
4426 static int
4427 filt_fsattach(struct knote *kn)
4428 {
4429 
4430  kn->kn_flags |= EV_CLEAR;
4431  knlist_add(&fs_knlist, kn, 0);
4432  return (0);
4433 }
4434 
4435 static void
4436 filt_fsdetach(struct knote *kn)
4437 {
4438 
4439  knlist_remove(&fs_knlist, kn, 0);
4440 }
4441 
4442 static int
4443 filt_fsevent(struct knote *kn, long hint)
4444 {
4445 
4446  kn->kn_fflags |= hint;
4447  return (kn->kn_fflags != 0);
4448 }
4449 
4450 static int
4451 sysctl_vfs_ctl(SYSCTL_HANDLER_ARGS)
4452 {
4453  struct vfsidctl vc;
4454  int error;
4455  struct mount *mp;
4456 
4457  error = SYSCTL_IN(req, &vc, sizeof(vc));
4458  if (error)
4459  return (error);
4460  if (vc.vc_vers != VFS_CTL_VERS1)
4461  return (EINVAL);
4462  mp = vfs_getvfs(&vc.vc_fsid);
4463  if (mp == NULL)
4464  return (ENOENT);
4465  /* ensure that a specific sysctl goes to the right filesystem. */
4466  if (strcmp(vc.vc_fstypename, "*") != 0 &&
4467  strcmp(vc.vc_fstypename, mp->mnt_vfc->vfc_name) != 0) {
4468  vfs_rel(mp);
4469  return (EINVAL);
4470  }
4471  VCTLTOREQ(&vc, req);
4472  error = VFS_SYSCTL(mp, vc.vc_op, req);
4473  vfs_rel(mp);
4474  return (error);
4475 }
4476 
4477 SYSCTL_PROC(_vfs, OID_AUTO, ctl, CTLTYPE_OPAQUE | CTLFLAG_WR,
4478  NULL, 0, sysctl_vfs_ctl, "",
4479  "Sysctl by fsid");
4480 
4481 /*
4482  * Function to initialize a va_filerev field sensibly.
4483  * XXX: Wouldn't a random number make a lot more sense ??
4484  */
4485 u_quad_t
4486 init_va_filerev(void)
4487 {
4488  struct bintime bt;
4489 
4490  getbinuptime(&bt);
4491  return (((u_quad_t)bt.sec << 32LL) | (bt.frac >> 32LL));
4492 }
4493 
4494 static int filt_vfsread(struct knote *kn, long hint);
4495 static int filt_vfswrite(struct knote *kn, long hint);
4496 static int filt_vfsvnode(struct knote *kn, long hint);
4497 static void filt_vfsdetach(struct knote *kn);
4498 static struct filterops vfsread_filtops = {
4499  .f_isfd = 1,
4500  .f_detach = filt_vfsdetach,
4501  .f_event = filt_vfsread
4502 };
4503 static struct filterops vfswrite_filtops = {
4504  .f_isfd = 1,
4505  .f_detach = filt_vfsdetach,
4506  .f_event = filt_vfswrite
4507 };
4508 static struct filterops vfsvnode_filtops = {
4509  .f_isfd = 1,
4510  .f_detach = filt_vfsdetach,
4511  .f_event = filt_vfsvnode
4512 };
4513 
4514 static void
4515 vfs_knllock(void *arg)
4516 {
4517  struct vnode *vp = arg;
4518 
4519  vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
4520 }
4521 
4522 static void
4523 vfs_knlunlock(void *arg)
4524 {
4525  struct vnode *vp = arg;
4526 
4527  VOP_UNLOCK(vp, 0);
4528 }
4529 
4530 static void
4531 vfs_knl_assert_locked(void *arg)
4532 {
4533 #ifdef DEBUG_VFS_LOCKS
4534  struct vnode *vp = arg;
4535 
4536  ASSERT_VOP_LOCKED(vp, "vfs_knl_assert_locked");
4537 #endif
4538 }
4539 
4540 static void
4541 vfs_knl_assert_unlocked(void *arg)
4542 {
4543 #ifdef DEBUG_VFS_LOCKS
4544  struct vnode *vp = arg;
4545 
4546  ASSERT_VOP_UNLOCKED(vp, "vfs_knl_assert_unlocked");
4547 #endif
4548 }
4549 
4550 int
4551 vfs_kqfilter(struct vop_kqfilter_args *ap)
4552 {
4553  struct vnode *vp = ap->a_vp;
4554  struct knote *kn = ap->a_kn;
4555  struct knlist *knl;
4556 
4557  switch (kn->kn_filter) {
4558  case EVFILT_READ:
4559  kn->kn_fop = &vfsread_filtops;
4560  break;
4561  case EVFILT_WRITE:
4562  kn->kn_fop = &vfswrite_filtops;
4563  break;
4564  case EVFILT_VNODE:
4565  kn->kn_fop = &vfsvnode_filtops;
4566  break;
4567  default:
4568  return (EINVAL);
4569  }
4570 
4571  kn->kn_hook = (caddr_t)vp;
4572 
4573  v_addpollinfo(vp);
4574  if (vp->v_pollinfo == NULL)
4575  return (ENOMEM);
4576  knl = &vp->v_pollinfo->vpi_selinfo.si_note;
4577  vhold(vp);
4578  knlist_add(knl, kn, 0);
4579 
4580  return (0);
4581 }
4582 
4583 /*
4584  * Detach knote from vnode
4585  */
4586 static void
4587 filt_vfsdetach(struct knote *kn)
4588 {
4589  struct vnode *vp = (struct vnode *)kn->kn_hook;
4590 
4591  KASSERT(vp->v_pollinfo != NULL, ("Missing v_pollinfo"));
4592  knlist_remove(&vp->v_pollinfo->vpi_selinfo.si_note, kn, 0);
4593  vdrop(vp);
4594 }
4595 
4596 /*ARGSUSED*/
4597 static int
4598 filt_vfsread(struct knote *kn, long hint)
4599 {
4600  struct vnode *vp = (struct vnode *)kn->kn_hook;
4601  struct vattr va;
4602  int res;
4603 
4604  /*
4605  * filesystem is gone, so set the EOF flag and schedule
4606  * the knote for deletion.
4607  */
4608  if (hint == NOTE_REVOKE) {
4609  VI_LOCK(vp);
4610  kn->kn_flags |= (EV_EOF | EV_ONESHOT);
4611  VI_UNLOCK(vp);
4612  return (1);
4613  }
4614 
4615  if (VOP_GETATTR(vp, &va, curthread->td_ucred))
4616  return (0);
4617 
4618  VI_LOCK(vp);
4619  kn->kn_data = va.va_size - kn->kn_fp->f_offset;
4620  res = (kn->kn_data != 0);
4621  VI_UNLOCK(vp);
4622  return (res);
4623 }
4624 
4625 /*ARGSUSED*/
4626 static int
4627 filt_vfswrite(struct knote *kn, long hint)
4628 {
4629  struct vnode *vp = (struct vnode *)kn->kn_hook;
4630 
4631  VI_LOCK(vp);
4632 
4633  /*
4634  * filesystem is gone, so set the EOF flag and schedule
4635  * the knote for deletion.
4636  */
4637  if (hint == NOTE_REVOKE)
4638  kn->kn_flags |= (EV_EOF | EV_ONESHOT);
4639 
4640  kn->kn_data = 0;
4641  VI_UNLOCK(vp);
4642  return (1);
4643 }
4644 
4645 static int
4646 filt_vfsvnode(struct knote *kn, long hint)
4647 {
4648  struct vnode *vp = (struct vnode *)kn->kn_hook;
4649  int res;
4650 
4651  VI_LOCK(vp);
4652  if (kn->kn_sfflags & hint)
4653  kn->kn_fflags |= hint;
4654  if (hint == NOTE_REVOKE) {
4655  kn->kn_flags |= EV_EOF;
4656  VI_UNLOCK(vp);
4657  return (1);
4658  }
4659  res = (kn->kn_fflags != 0);
4660  VI_UNLOCK(vp);
4661  return (res);
4662 }
4663 
4664 int
4665 vfs_read_dirent(struct vop_readdir_args *ap, struct dirent *dp, off_t off)
4666 {
4667  int error;
4668 
4669  if (dp->d_reclen > ap->a_uio->uio_resid)
4670  return (ENAMETOOLONG);
4671  error = uiomove(dp, dp->d_reclen, ap->a_uio);
4672  if (error) {
4673  if (ap->a_ncookies != NULL) {
4674  if (ap->a_cookies != NULL)
4675  free(ap->a_cookies, M_TEMP);
4676  ap->a_cookies = NULL;
4677  *ap->a_ncookies = 0;
4678  }
4679  return (error);
4680  }
4681  if (ap->a_ncookies == NULL)
4682  return (0);
4683 
4684  KASSERT(ap->a_cookies,
4685  ("NULL ap->a_cookies value with non-NULL ap->a_ncookies!"));
4686 
4687  *ap->a_cookies = realloc(*ap->a_cookies,
4688  (*ap->a_ncookies + 1) * sizeof(u_long), M_TEMP, M_WAITOK | M_ZERO);
4689  (*ap->a_cookies)[*ap->a_ncookies] = off;
4690  *ap->a_ncookies += 1;
4691  return (0);
4692 }
4693 
4694 /*
4695  * Mark for update the access time of the file if the filesystem
4696  * supports VOP_MARKATIME. This functionality is used by execve and
4697  * mmap, so we want to avoid the I/O implied by directly setting
4698  * va_atime for the sake of efficiency.
4699  */
4700 void
4701 vfs_mark_atime(struct vnode *vp, struct ucred *cred)
4702 {
4703  struct mount *mp;
4704 
4705  mp = vp->v_mount;
4706  VFS_ASSERT_GIANT(mp);
4707  ASSERT_VOP_LOCKED(vp, "vfs_mark_atime");
4708  if (mp != NULL && (mp->mnt_flag & (MNT_NOATIME | MNT_RDONLY)) == 0)
4709  (void)VOP_MARKATIME(vp);
4710 }
4711 
4712 /*
4713  * The purpose of this routine is to remove granularity from accmode_t,
4714  * reducing it into standard unix access bits - VEXEC, VREAD, VWRITE,
4715  * VADMIN and VAPPEND.
4716  *
4717  * If it returns 0, the caller is supposed to continue with the usual
4718  * access checks using 'accmode' as modified by this routine. If it
4719  * returns nonzero value, the caller is supposed to return that value
4720  * as errno.
4721  *
4722  * Note that after this routine runs, accmode may be zero.
4723  */
4724 int
4725 vfs_unixify_accmode(accmode_t *accmode)
4726 {
4727  /*
4728  * There is no way to specify explicit "deny" rule using
4729  * file mode or POSIX.1e ACLs.
4730  */
4731  if (*accmode & VEXPLICIT_DENY) {
4732  *accmode = 0;
4733  return (0);
4734  }
4735 
4736  /*
4737  * None of these can be translated into usual access bits.
4738  * Also, the common case for NFSv4 ACLs is to not contain
4739  * either of these bits. Caller should check for VWRITE
4740  * on the containing directory instead.
4741  */
4742  if (*accmode & (VDELETE_CHILD | VDELETE))
4743  return (EPERM);
4744 
4745  if (*accmode & VADMIN_PERMS) {
4746  *accmode &= ~VADMIN_PERMS;
4747  *accmode |= VADMIN;
4748  }
4749 
4750  /*
4751  * There is no way to deny VREAD_ATTRIBUTES, VREAD_ACL
4752  * or VSYNCHRONIZE using file mode or POSIX.1e ACL.
4753  */
4754  *accmode &= ~(VSTAT_PERMS | VSYNCHRONIZE);
4755 
4756  return (0);
4757 }
4758 
4759 /*
4760  * These are helper functions for filesystems to traverse all
4761  * their vnodes. See MNT_VNODE_FOREACH_ALL() in sys/mount.h.
4762  *
4763  * This interface replaces MNT_VNODE_FOREACH.
4764  */
4765 
4766 MALLOC_DEFINE(M_VNODE_MARKER, "vnodemarker", "vnode marker");
4767 
4768 struct vnode *
4769 __mnt_vnode_next_all(struct vnode **mvp, struct mount *mp)
4770 {
4771  struct vnode *vp;
4772 
4773  if (should_yield())
4774  kern_yield(PRI_UNCHANGED);
4775  MNT_ILOCK(mp);
4776  KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch"));
4777  vp = TAILQ_NEXT(*mvp, v_nmntvnodes);
4778  while (vp != NULL && (vp->v_type == VMARKER ||
4779  (vp->v_iflag & VI_DOOMED) != 0))
4780  vp = TAILQ_NEXT(vp, v_nmntvnodes);
4781 
4782  /* Check if we are done */
4783  if (vp == NULL) {
4784  __mnt_vnode_markerfree_all(mvp, mp);
4785  /* MNT_IUNLOCK(mp); -- done in above function */
4786  mtx_assert(MNT_MTX(mp), MA_NOTOWNED);
4787  return (NULL);
4788  }
4789  TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes);
4790  TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes);
4791  VI_LOCK(vp);
4792  MNT_IUNLOCK(mp);
4793  return (vp);
4794 }
4795 
4796 struct vnode *
4797 __mnt_vnode_first_all(struct vnode **mvp, struct mount *mp)
4798 {
4799  struct vnode *vp;
4800 
4801  *mvp = malloc(sizeof(struct vnode), M_VNODE_MARKER, M_WAITOK | M_ZERO);
4802  MNT_ILOCK(mp);
4803  MNT_REF(mp);
4804  (*mvp)->v_type = VMARKER;
4805 
4806  vp = TAILQ_FIRST(&mp->mnt_nvnodelist);
4807  while (vp != NULL && (vp->v_type == VMARKER ||
4808  (vp->v_iflag & VI_DOOMED) != 0))
4809  vp = TAILQ_NEXT(vp, v_nmntvnodes);
4810 
4811  /* Check if we are done */
4812  if (vp == NULL) {
4813  MNT_REL(mp);
4814  MNT_IUNLOCK(mp);
4815  free(*mvp, M_VNODE_MARKER);
4816  *mvp = NULL;
4817  return (NULL);
4818  }
4819  (*mvp)->v_mount = mp;
4820  TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes);
4821  VI_LOCK(vp);
4822  MNT_IUNLOCK(mp);
4823  return (vp);
4824 }
4825 
4826 
4827 void
4828 __mnt_vnode_markerfree_all(struct vnode **mvp, struct mount *mp)
4829 {
4830 
4831  if (*mvp == NULL) {
4832  MNT_IUNLOCK(mp);
4833  return;
4834  }
4835 
4836  mtx_assert(MNT_MTX(mp), MA_OWNED);
4837 
4838  KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch"));
4839  TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes);
4840  MNT_REL(mp);
4841  MNT_IUNLOCK(mp);
4842  free(*mvp, M_VNODE_MARKER);
4843  *mvp = NULL;
4844 }
4845 
4846 /*
4847  * These are helper functions for filesystems to traverse their
4848  * active vnodes. See MNT_VNODE_FOREACH_ACTIVE() in sys/mount.h
4849  */
4850 static void
4851 mnt_vnode_markerfree_active(struct vnode **mvp, struct mount *mp)
4852 {
4853 
4854  KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch"));
4855 
4856  MNT_ILOCK(mp);
4857  MNT_REL(mp);
4858  MNT_IUNLOCK(mp);
4859  free(*mvp, M_VNODE_MARKER);
4860  *mvp = NULL;
4861 }
4862 
4863 #ifdef SMP
4864 #define ALWAYS_YIELD (mp_ncpus == 1)
4865 #else
4866 #define ALWAYS_YIELD 1
4867 #endif
4868 
4869 static struct vnode *
4870 mnt_vnode_next_active(struct vnode **mvp, struct mount *mp)
4871 {
4872  struct vnode *vp, *nvp;
4873 
4874  mtx_assert(&vnode_free_list_mtx, MA_OWNED);
4875  KASSERT((*mvp)->v_mount == mp, ("marker vnode mount list mismatch"));
4876 restart:
4877  vp = TAILQ_NEXT(*mvp, v_actfreelist);
4878  TAILQ_REMOVE(&mp->mnt_activevnodelist, *mvp, v_actfreelist);
4879  while (vp != NULL) {
4880  if (vp->v_type == VMARKER) {
4881  vp = TAILQ_NEXT(vp, v_actfreelist);
4882  continue;
4883  }
4884  if (!VI_TRYLOCK(vp)) {
4885  if (ALWAYS_YIELD || should_yield()) {
4886  TAILQ_INSERT_BEFORE(vp, *mvp, v_actfreelist);
4887  mtx_unlock(&vnode_free_list_mtx);
4888  pause("vnacti", 1);
4889  mtx_lock(&vnode_free_list_mtx);
4890  goto restart;
4891  }
4892  continue;
4893  }
4894  KASSERT(vp->v_type != VMARKER, ("locked marker %p", vp));
4895  KASSERT(vp->v_mount == mp || vp->v_mount == NULL,
4896  ("alien vnode on the active list %p %p", vp, mp));
4897  if (vp->v_mount == mp && (vp->v_iflag & VI_DOOMED) == 0)
4898  break;
4899  nvp = TAILQ_NEXT(vp, v_actfreelist);
4900  VI_UNLOCK(vp);
4901  vp = nvp;
4902  }
4903 
4904  /* Check if we are done */
4905  if (vp == NULL) {
4906  mtx_unlock(&vnode_free_list_mtx);
4907  mnt_vnode_markerfree_active(mvp, mp);
4908  return (NULL);
4909  }
4910  TAILQ_INSERT_AFTER(&mp->mnt_activevnodelist, vp, *mvp, v_actfreelist);
4911  mtx_unlock(&vnode_free_list_mtx);
4912  ASSERT_VI_LOCKED(vp, "active iter");
4913  KASSERT((vp->v_iflag & VI_ACTIVE) != 0, ("Non-active vp %p", vp));
4914  return (vp);
4915 }
4916 #undef ALWAYS_YIELD
4917 
4918 struct vnode *
4919 __mnt_vnode_next_active(struct vnode **mvp, struct mount *mp)
4920 {
4921 
4922  if (should_yield())
4923  kern_yield(PRI_UNCHANGED);
4924  mtx_lock(&vnode_free_list_mtx);
4925  return (mnt_vnode_next_active(mvp, mp));
4926 }
4927 
4928 struct vnode *
4929 __mnt_vnode_first_active(struct vnode **mvp, struct mount *mp)
4930 {
4931  struct vnode *vp;
4932 
4933  *mvp = malloc(sizeof(struct vnode), M_VNODE_MARKER, M_WAITOK | M_ZERO);
4934  MNT_ILOCK(mp);
4935  MNT_REF(mp);
4936  MNT_IUNLOCK(mp);
4937  (*mvp)->v_type = VMARKER;
4938  (*mvp)->v_mount = mp;
4939 
4940  mtx_lock(&vnode_free_list_mtx);
4941  vp = TAILQ_FIRST(&mp->mnt_activevnodelist);
4942  if (vp == NULL) {
4943  mtx_unlock(&vnode_free_list_mtx);
4944  mnt_vnode_markerfree_active(mvp, mp);
4945  return (NULL);
4946  }
4947  TAILQ_INSERT_BEFORE(vp, *mvp, v_actfreelist);
4948  return (mnt_vnode_next_active(mvp, mp));
4949 }
4950 
4951 void
4952 __mnt_vnode_markerfree_active(struct vnode **mvp, struct mount *mp)
4953 {
4954 
4955  if (*mvp == NULL)
4956  return;
4957 
4958  mtx_lock(&vnode_free_list_mtx);
4959  TAILQ_REMOVE(&mp->mnt_activevnodelist, *mvp, v_actfreelist);
4960  mtx_unlock(&vnode_free_list_mtx);
4961  mnt_vnode_markerfree_active(mvp, mp);
4962 }
sync_close
#define sync_close
Definition: vfs_subr.c:3710
vnlru_free
static void vnlru_free(int)
Definition: vfs_subr.c:807
lockdestroy
void lockdestroy(struct lock *lk)
Definition: kern_lock.c:431
vlrureclaim
static int vlrureclaim(struct mount *mp)
Definition: vfs_subr.c:705
vfs_event_signal
void vfs_event_signal(fsid_t *fsid, uint32_t event, intptr_t data __unused)
Definition: vfs_subr.c:4409
vfs_suser
int vfs_suser(struct mount *mp, struct thread *td)
Definition: vfs_subr.c:544
count_dev
int count_dev(struct cdev *dev)
Definition: vfs_subr.c:2974
should_yield
int should_yield(void)
Definition: kern_synch.c:577
vfs_allocate_syncvnode
void vfs_allocate_syncvnode(struct mount *mp)
Definition: vfs_subr.c:3730
VCANRECYCLE
#define VCANRECYCLE(vp)
Definition: vfs_subr.c:289
__FBSDID
__FBSDID("$BSDSUniX$")
vinvalbuf
int vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo)
Definition: vfs_subr.c:1348
mnt_vnode_markerfree_active
static void mnt_vnode_markerfree_active(struct vnode **mvp, struct mount *mp)
Definition: vfs_subr.c:4851
time_second
volatile time_t time_second
Definition: kern_tc.c:94
sysctl_vfs_conflist
static int sysctl_vfs_conflist(SYSCTL_HANDLER_ARGS)
Definition: vfs_subr.c:3359
vfs_sysctl
static int vfs_sysctl(SYSCTL_HANDLER_ARGS)
Definition: vfs_subr.c:3386
realloc
void * realloc(void *addr, unsigned long size, struct malloc_type *mtp, int flags)
Definition: kern_malloc.c:616
sched_prio
void sched_prio(struct thread *td, u_char prio)
Definition: sched_4bsd.c:897
TSP_HZ
Definition: vfs_subr.c:623
buf
struct buf * buf
Definition: vfs_bio.c:97
hashinit
void * hashinit(int elements, struct malloc_type *type, u_long *hashmask)
Definition: subr_hash.c:83
insmntque_stddtr
static void insmntque_stddtr(struct vnode *vp, void *dtr_arg)
Definition: vfs_subr.c:1180
sysctl_vfs_ctl
static int sysctl_vfs_ctl(SYSCTL_HANDLER_ARGS)
Definition: vfs_subr.c:4451
bufobj_wwait
int bufobj_wwait(struct bufobj *bo, int slpflag, int timeo)
Definition: vfs_bio.c:4503
filt_fsdetach
static void filt_fsdetach(struct knote *kn)
Definition: vfs_subr.c:4436
vttoif_tab
int vttoif_tab[10]
Definition: vfs_subr.c:138
v_upgrade_usecount
static void v_upgrade_usecount(struct vnode *)
Definition: vfs_subr.c:2194
snprintf
int snprintf(char *str, size_t size, const char *format,...)
Definition: subr_prf.c:509
vfs_deallocate_syncvnode
void vfs_deallocate_syncvnode(struct mount *mp)
Definition: vfs_subr.c:3785
priv_check_cred
int priv_check_cred(struct ucred *cred, int priv, int flags)
Definition: kern_priv.c:76
TSP_SEC
Definition: vfs_subr.c:623
bootverbose
int bootverbose
Definition: init_main.c:107
start
void *** start
Definition: linker_if.m:86
selrecord
void selrecord(struct thread *selector, struct selinfo *sip)
Definition: sys_generic.c:1606
vfs_rel
void vfs_rel(struct mount *mp)
Definition: vfs_mount.c:439
vnlru_nowhere
static int vnlru_nowhere
Definition: vfs_subr.c:281
ALWAYS_YIELD
#define ALWAYS_YIELD
Definition: vfs_subr.c:4866
filt_fsattach
static int filt_fsattach(struct knote *kn)
Definition: vfs_subr.c:4427
flushbuflist
static int flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, int slpflag, int slptimeo)
Definition: vfs_subr.c:1363
vputx
static void vputx(struct vnode *vp, int func)
Definition: vfs_subr.c:2337
LIST_HEAD
static LIST_HEAD(alq)
Definition: kern_alq.c:97
malloc
void * malloc(unsigned long size, struct malloc_type *mtp, int flags)
Definition: kern_malloc.c:454
iftovt_tab
enum vtype iftovt_tab[16]
Definition: vfs_subr.c:134
vop_symlink_post
void vop_symlink_post(void *ap, int rc)
Definition: vfs_subr.c:4390
cache_purge
void cache_purge(struct vnode *vp)
Definition: vfs_cache.c:941
rangelock_destroy
void rangelock_destroy(struct rangelock *lock)
Definition: kern_rangelock.c:80
v_decr_useonly
static void v_decr_useonly(struct vnode *)
Definition: vfs_subr.c:2235
panicstr
const char * panicstr
Definition: kern_shutdown.c:149
destroy_vpollinfo
static void destroy_vpollinfo(struct vpollinfo *vi)
Definition: vfs_subr.c:3641
__mnt_vnode_markerfree_active
void __mnt_vnode_markerfree_active(struct vnode **mvp, struct mount *mp)
Definition: vfs_subr.c:4952
panic
void panic(const char *fmt,...)
Definition: kern_shutdown.c:557
vget
int vget(struct vnode *vp, int flags, struct thread *td)
Definition: vfs_subr.c:2258
vop_unlock_pre
void vop_unlock_pre(void *ap)
Definition: vfs_subr.c:4259
vn_finished_write
void vn_finished_write(struct mount *mp)
Definition: vfs_vnops.c:1599
sync_vnodeops
static struct vop_vector sync_vnodeops
Definition: vfs_subr.c:3715
FSID_CACHE_SIZE
#define FSID_CACHE_SIZE
knote
void knote(struct knlist *list, long hint, int lockflags)
Definition: kern_event.c:1806
kproc_shutdown
void kproc_shutdown(void *arg, int howto)
Definition: kern_shutdown.c:681
prison_check
int prison_check(struct ucred *cred1, struct ucred *cred2)
Definition: kern_jail.c:3450
vop_create_post
void vop_create_post(void *ap, int rc)
Definition: vfs_subr.c:4282
sync_reclaim
static int sync_reclaim(struct vop_reclaim_args *)
Definition: vfs_subr.c:3859
vattr_null
void vattr_null(struct vattr *vap)
Definition: vfs_subr.c:662
getnewvnode_wait
static int getnewvnode_wait(int suspended)
Definition: vfs_subr.c:989
desiredvnodes
int desiredvnodes
Definition: vfs_subr.c:276
vnlru_kp
static struct kproc_desc vnlru_kp
Definition: vfs_subr.c:915
vfs_kqfilter
int vfs_kqfilter(struct vop_kqfilter_args *ap)
Definition: vfs_subr.c:4551
VPUTX_VUNREF
#define VPUTX_VUNREF
Definition: vfs_subr.c:2334
filt_vfsdetach
static void filt_vfsdetach(struct knote *kn)
Definition: vfs_subr.c:4587
vnsz2log
static int vnsz2log
Definition: vfs_subr.c:294
name
const char * name
Definition: kern_fail.c:97
KINFO_VNODESLOP
#define KINFO_VNODESLOP
Definition: vfs_subr.c:3453
brelse
void brelse(struct buf *bp)
Definition: vfs_bio.c:1451
maxproc
int maxproc
Definition: subr_param.c:87
dounmount
int dounmount(struct mount *mp, int flags, struct thread *td)
Definition: vfs_mount.c:1224
vfs_notify_upper
void vfs_notify_upper(struct vnode *vp, int event)
Definition: vfs_subr.c:2816
SYSCTL_PROC
SYSCTL_PROC(_vfs, OID_AUTO, worklist_len, CTLTYPE_INT|CTLFLAG_RD, NULL, 0, sysctl_vfs_worklist_len,"I","Syncer thread worklist length")
TSP_USEC
Definition: vfs_subr.c:623
vfsvnode_filtops
static struct filterops vfsvnode_filtops
Definition: vfs_subr.c:4508
vrefcnt
int vrefcnt(struct vnode *vp)
Definition: vfs_subr.c:2321
vop_stdislocked
int vop_stdislocked(struct vop_islocked_args *ap)
Definition: vfs_default.c:537
knlist_destroy
void knlist_destroy(struct knlist *knl)
Definition: kern_event.c:2002
vop_rename_fail
void vop_rename_fail(struct vop_rename_args *ap)
Definition: vfs_subr.c:4161
vop_setattr_post
void vop_setattr_post(void *ap, int rc)
Definition: vfs_subr.c:4372
vaccess
int vaccess(enum vtype type, mode_t file_mode, uid_t file_uid, gid_t file_gid, accmode_t accmode, struct ucred *cred, int *privused)
Definition: vfs_subr.c:3913
v_incr_usecount
static void v_incr_usecount(struct vnode *)
Definition: vfs_subr.c:2176
buf_vlist_add
static void buf_vlist_add(struct buf *bp, struct bufobj *bo, b_xflags_t xflags)
Definition: vfs_subr.c:1656
up_kp
static struct kproc_desc up_kp
Definition: vfs_subr.c:1838
notify_lowervp_vfs_dummy
static void notify_lowervp_vfs_dummy(struct mount *mp __unused, struct vnode *lowervp __unused)
Definition: vfs_subr.c:2807
vfsconf
struct vfsconfhead vfsconf
Definition: vfs_init.c:66
__mnt_vnode_first_active
struct vnode * __mnt_vnode_first_active(struct vnode **mvp, struct mount *mp)
Definition: vfs_subr.c:4929
type
int * type
Definition: cpufreq_if.m:98
accmode
accmode_t accmode
Definition: subr_acl_nfs4.c:66
vdrop
void vdrop(struct vnode *vp)
Definition: vfs_subr.c:2493
insmntque1
int insmntque1(struct vnode *vp, struct mount *mp, void(*dtr)(struct vnode *, void *), void *dtr_arg)
Definition: vfs_subr.c:1197
updateproc
static struct proc * updateproc
Definition: vfs_subr.c:1836
VPUTX_VRELE
#define VPUTX_VRELE
Definition: vfs_subr.c:2332
busy
static u_int busy
Definition: kern_clocksource.c:104
vfs_timestamp
void vfs_timestamp(struct timespec *tsp)
Definition: vfs_subr.c:635
vnlru_proc
static void vnlru_proc(void)
Definition: vfs_subr.c:863
__mnt_vnode_first_all
struct vnode * __mnt_vnode_first_all(struct vnode **mvp, struct mount *mp)
Definition: vfs_subr.c:4797
priv_check
int priv_check(struct thread *td, int priv)
Definition: kern_priv.c:170
SYSCTL_NODE
static SYSCTL_NODE(_vfs, VFS_GENERIC, generic, CTLFLAG_RD|CTLFLAG_SKIP, vfs_sysctl,"Generic filesystem")
vref
void vref(struct vnode *vp)
Definition: vfs_subr.c:2302
SYSCTL_ULONG
SYSCTL_ULONG(_vfs, OID_AUTO, numvnodes, CTLFLAG_RD,&numvnodes, 0,"Number of vnodes in existence")
vput
void vput(struct vnode *vp)
Definition: vfs_subr.c:2428
destroy_vpollinfo_free
static void destroy_vpollinfo_free(struct vpollinfo *vi)
Definition: vfs_subr.c:3632
extattr_check_cred
int extattr_check_cred(struct vnode *vp, int attrnamespace, struct ucred *cred, struct thread *td, accmode_t accmode)
Definition: vfs_subr.c:4031
vinactive
void vinactive(struct vnode *vp, struct thread *td)
Definition: vfs_subr.c:2597
reassignbuf
void reassignbuf(struct buf *bp)
Definition: vfs_subr.c:2090
vfswrite_filtops
static struct filterops vfswrite_filtops
Definition: vfs_subr.c:4503
vrecycle
int vrecycle(struct vnode *vp, struct thread *td)
Definition: vfs_subr.c:2778
vfs_getnewfsid
void vfs_getnewfsid(struct mount *mp)
Definition: vfs_subr.c:590
dummy
static int dummy
Definition: uipc_sockbuf.c:1116
buf_splay
static struct buf * buf_splay(daddr_t lblkno, b_xflags_t xflags, struct buf *root)
Definition: vfs_subr.c:1564
vfs_ref
void vfs_ref(struct mount *mp)
Definition: vfs_mount.c:429
knlist_remove
void knlist_remove(struct knlist *knl, struct knote *kn, int islocked)
Definition: kern_event.c:1909
Giant
struct mtx Giant
Definition: kern_mutex.c:140
knlist_init
void knlist_init(struct knlist *knl, void *lock, void(*kl_lock)(void *), void(*kl_unlock)(void *), void(*kl_assert_locked)(void *), void(*kl_assert_unlocked)(void *))
Definition: kern_event.c:1964
vfs_unixify_accmode
int vfs_unixify_accmode(accmode_t *accmode)
Definition: vfs_subr.c:4725
vdropl
void vdropl(struct vnode *vp)
Definition: vfs_subr.c:2506
vn_syncer_add_to_worklist
static void vn_syncer_add_to_worklist(struct bufobj *bo, int delay)
Definition: vfs_subr.c:1796
vn_printf
void vn_printf(struct vnode *vp, const char *fmt,...)
Definition: vfs_subr.c:2992
vfs_event_init
static void vfs_event_init(void *arg)
Definition: vfs_subr.c:4401
vop_rename_post
void vop_rename_post(void *ap, int rc)
Definition: vfs_subr.c:4340
jailed
int jailed(struct ucred *cred)
Definition: kern_jail.c:3474
vfs_unbusy
void vfs_unbusy(struct mount *mp)
Definition: vfs_subr.c:442
vop_strategy_pre
void vop_strategy_pre(void *ap)
Definition: vfs_subr.c:4208
vunref
void vunref(struct vnode *vp)
Definition: vfs_subr.c:2438
seldrain
void seldrain(struct selinfo *sip)
Definition: sys_generic.c:1587
TAILQ_HEAD
static TAILQ_HEAD(freelst, vnode)
Definition: vfs_subr.c:146
MALLOC_DEFINE
MALLOC_DEFINE(M_VNODE_MARKER,"vnodemarker","vnode marker")
kdb_backtrace
void kdb_backtrace(void)
Definition: subr_kdb.c:362
vop_mkdir_post
void vop_mkdir_post(void *ap, int rc)
Definition: vfs_subr.c:4311
sync_fsync
static int sync_fsync(struct vop_fsync_args *)
Definition: vfs_subr.c:3802
buf_ops_bio
struct buf_ops buf_ops_bio
Definition: vfs_bio.c:85
VSHOULDBUSY
#define VSHOULDBUSY(vp)
Definition: vfs_subr.c:291
vholdl
void vholdl(struct vnode *vp)
Definition: vfs_subr.c:2460
vntblinit
static void vntblinit(void *dummy __unused)
Definition: vfs_subr.c:307
getnewvnode_reserve
void getnewvnode_reserve(u_int count)
Definition: vfs_subr.c:1014
rangelock_init
void rangelock_init(struct rangelock *lock)
Definition: kern_rangelock.c:72
vcount
int vcount(struct vnode *vp)
Definition: vfs_subr.c:2960
vop_remove_post
void vop_remove_post(void *ap, int rc)
Definition: vfs_subr.c:4329
nanotime
void nanotime(struct timespec *tsp)
Definition: kern_tc.c:211
vfs_busyfs
struct mount * vfs_busyfs(fsid_t *fsid)
Definition: vfs_subr.c:493
TSP_NSEC
Definition: vfs_subr.c:623
vn_pollrecord
int vn_pollrecord(struct vnode *vp, struct thread *td, int events)
Definition: vfs_subr.c:3682
knlist_add
void knlist_add(struct knlist *knl, struct knote *kn, int islocked)
Definition: kern_event.c:1866
vtruncbuf
int vtruncbuf(struct vnode *vp, struct ucred *cred, struct thread *td, off_t length, int blksize)
Definition: vfs_subr.c:1441
SYSCTL_INT
SYSCTL_INT(_kern, KERN_MAXVNODES, maxvnodes, CTLFLAG_RW,&desiredvnodes, 0,"Maximum number of vnodes")
vtryrecycle
static int vtryrecycle(struct vnode *vp)
Definition: vfs_subr.c:934
vop_unlock_post
void vop_unlock_post(void *ap, int rc)
Definition: vfs_subr.c:4271
vfsread_filtops
static struct filterops vfsread_filtops
Definition: vfs_subr.c:4498
vop_lock_post
void vop_lock_post(void *ap, int rc)
Definition: vfs_subr.c:4247
getnewvnode
int getnewvnode(const char *tag, struct mount *mp, struct vop_vector *vops, struct vnode **vpp)
Definition: vfs_subr.c:1051
sched_sync
static void sched_sync(void)
Definition: vfs_subr.c:1895
vop_stdlock
int vop_stdlock(struct vop_lock1_args *ap)
Definition: vfs_default.c:507
v_addpollinfo
void v_addpollinfo(struct vnode *vp)
Definition: vfs_subr.c:3653
getnanotime
void getnanotime(struct timespec *tsp)
Definition: kern_tc.c:282
cv_init
void cv_init(struct cv *cvp, const char *desc)
Definition: kern_condvar.c:63
vfsconf2x
static int vfsconf2x(struct sysctl_req *req, struct vfsconf *vfsp)
Definition: vfs_subr.c:3312
bawrite
void bawrite(struct buf *bp)
Definition: vfs_bio.c:1357
vfs_unp_reclaim
void vfs_unp_reclaim(struct vnode *vp)
Definition: uipc_usrreq.c:2328
groupmember
int groupmember(gid_t gid, struct ucred *cred)
Definition: kern_prot.c:1267
uiomove
int uiomove(void *cp, int n, struct uio *uio)
Definition: subr_uio.c:202
fs_filtops
struct filterops fs_filtops
Definition: vfs_subr.c:4419
vgonel
static void vgonel(struct vnode *)
Definition: vfs_subr.c:2873
sysctl_vfs_worklist_len
static int sysctl_vfs_worklist_len(SYSCTL_HANDLER_ARGS)
Definition: vfs_subr.c:1822
vfs_knl_assert_unlocked
static void vfs_knl_assert_unlocked(void *arg)
Definition: vfs_subr.c:4541
mountlist_mtx
struct mtx mountlist_mtx
Definition: vfs_mount.c:88
init_va_filerev
u_quad_t init_va_filerev(void)
Definition: vfs_subr.c:4486
vflush
int vflush(struct mount *mp, int rootrefs, int flags, struct thread *td)
Definition: vfs_subr.c:2656
pause
int pause(const char *wmesg, int timo)
Definition: kern_synch.c:350
free
void free(void *addr, struct malloc_type *mtp)
Definition: kern_malloc.c:554
kdb_enter
void kdb_enter(const char *why, const char *msg)
Definition: subr_kdb.c:433
gbincore
struct buf * gbincore(struct bufobj *bo, daddr_t lblkno)
Definition: vfs_subr.c:1705
SYNCER_MAXDELAY
#define SYNCER_MAXDELAY
printf
int printf(const char *fmt,...)
Definition: subr_prf.c:367
__mnt_vnode_next_all
struct vnode * __mnt_vnode_next_all(struct vnode **mvp, struct mount *mp)
Definition: vfs_subr.c:4769
delmntque
static void delmntque(struct vnode *vp)
Definition: vfs_subr.c:1148
vfs_busy
int vfs_busy(struct mount *mp, int flags)
Definition: vfs_subr.c:395
sync_vnode
static int sync_vnode(struct synclist *slp, struct bufobj **bo, struct thread *td)
Definition: vfs_subr.c:1846
kproc_suspend_check
void kproc_suspend_check(struct proc *p)
Definition: kern_kthread.c:208
filt_fsevent
static int filt_fsevent(struct knote *kn, long hint)
Definition: vfs_subr.c:4443
vop_stdunlock
int vop_stdunlock(struct vop_unlock_args *ap)
Definition: vfs_default.c:524
dev_unlock
void dev_unlock(void)
Definition: kern_conf.c:132
VSHOULDFREE
#define VSHOULDFREE(vp)
Definition: vfs_subr.c:290
kern_yield
void kern_yield(int prio)
Definition: kern_synch.c:592
__mnt_vnode_markerfree_all
void __mnt_vnode_markerfree_all(struct vnode **mvp, struct mount *mp)
Definition: vfs_subr.c:4828
sysctl_wire_old_buffer
int sysctl_wire_old_buffer(struct sysctl_req *req, size_t len)
Definition: kern_sysctl.c:1364
sysctl_ovfs_conf
static int sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS)
Definition: vfs_subr.c:3430
vn_finished_secondary_write
void vn_finished_secondary_write(struct mount *mp)
Definition: vfs_vnops.c:1622
vfs_knlunlock
static void vfs_knlunlock(void *arg)
Definition: vfs_subr.c:4523
bintime
void bintime(struct bintime *bt)
Definition: kern_tc.c:203
vhold
void vhold(struct vnode *vp)
Definition: vfs_subr.c:2448
mtx_init
void mtx_init(struct mtx *m, const char *name, const char *type, int opts)
Definition: kern_mutex.c:837
wakeup
void wakeup(void *ident)
Definition: kern_synch.c:378
lockmgr_printinfo
void lockmgr_printinfo(struct lock *lk)
Definition: kern_lock.c:1296
mountlist
struct mntlist mountlist
Definition: vfs_mount.c:85
insmntque
int insmntque(struct vnode *vp, struct mount *mp)
Definition: vfs_subr.c:1253
MAXVNODES_MAX
#define MAXVNODES_MAX
Definition: vfs_subr.c:304
VPUTX_VPUT
#define VPUTX_VPUT
Definition: vfs_subr.c:2333
vrele
void vrele(struct vnode *vp)
Definition: vfs_subr.c:2416
vn_start_write
int vn_start_write(struct vnode *vp, struct mount **mpp, int flags)
Definition: vfs_vnops.c:1491
timestamp_precision
static int timestamp_precision
Definition: vfs_subr.c:625
getbinuptime
void getbinuptime(struct bintime *bt)
Definition: kern_tc.c:229
kproc_start
void kproc_start(void *udata) const
Definition: kern_kthread.c:57
syncer_state
syncer_state
Definition: vfs_subr.c:267
vop_setextattr_post
void vop_setextattr_post(void *ap, int rc)
Definition: vfs_subr.c:4381
maxvfsconf
int maxvfsconf
Definition: vfs_init.c:60
fs_knlist
static struct knlist fs_knlist
Definition: vfs_subr.c:4398
vfs_getvfs
struct mount * vfs_getvfs(fsid_t *fsid)
Definition: vfs_subr.c:463
buf_vlist_remove
static void buf_vlist_remove(struct buf *bp)
Definition: vfs_subr.c:1619
vgone
void vgone(struct vnode *vp)
Definition: vfs_subr.c:2799
filt_vfswrite
static int filt_vfswrite(struct knote *kn, long hint)
Definition: vfs_subr.c:4627
bgetvp
void bgetvp(struct vnode *vp, struct buf *bp)
Definition: vfs_subr.c:1733
vnlruproc_sig
static int vnlruproc_sig
Definition: vfs_subr.c:860
vprintf
int vprintf(const char *fmt, va_list ap)
Definition: subr_prf.c:380
vnlruproc
static struct proc * vnlruproc
Definition: vfs_subr.c:859
vn_isdisk
int vn_isdisk(struct vnode *vp, int *errp)
Definition: vfs_subr.c:3885
SYSINIT
SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_FIRST, vntblinit, NULL)
filt_vfsvnode
static int filt_vfsvnode(struct knote *kn, long hint)
Definition: vfs_subr.c:4646
vm_kmem_size
u_long vm_kmem_size
Definition: kern_malloc.c:187
time_uptime
volatile time_t time_uptime
Definition: kern_tc.c:95
vfs_knllock
static void vfs_knllock(void *arg)
Definition: vfs_subr.c:4515
vfs_unmountall
void vfs_unmountall(void)
Definition: vfs_subr.c:3554
bufobj_invalbuf
int bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo)
Definition: vfs_subr.c:1264
bremfree
void bremfree(struct buf *bp)
Definition: vfs_bio.c:846
microtime
void microtime(struct timeval *tvp)
Definition: kern_tc.c:220
vop_deleteextattr_post
void vop_deleteextattr_post(void *ap, int rc)
Definition: vfs_subr.c:4291
vfs_read_dirent
int vfs_read_dirent(struct vop_readdir_args *ap, struct dirent *dp, off_t off)
Definition: vfs_subr.c:4665
vfs_mark_atime
void vfs_mark_atime(struct vnode *vp, struct ucred *cred)
Definition: vfs_subr.c:4701
vop_link_post
void vop_link_post(void *ap, int rc)
Definition: vfs_subr.c:4300
mtx_destroy
void mtx_destroy(struct mtx *m)
Definition: kern_mutex.c:884
filt_vfsread
static int filt_vfsread(struct knote *kn, long hint)
Definition: vfs_subr.c:4598
vop_lock_pre
void vop_lock_pre(void *ap)
Definition: vfs_subr.c:4234
vnodes_created
static u_long vnodes_created
Definition: vfs_subr.c:126
vop_mknod_post
void vop_mknod_post(void *ap, int rc)
Definition: vfs_subr.c:4320
sync_inactive
static int sync_inactive(struct vop_inactive_args *)
Definition: vfs_subr.c:3846
vfs_msync
void vfs_msync(struct mount *mp, int flags)
Definition: vfs_subr.c:3598
__mnt_vnode_next_active
struct vnode * __mnt_vnode_next_active(struct vnode **mvp, struct mount *mp)
Definition: vfs_subr.c:4919
knlist_init_mtx
void knlist_init_mtx(struct knlist *knl, struct mtx *lock)
Definition: kern_event.c:1995
WI_MPSAFEQ
#define WI_MPSAFEQ
Definition: vfs_subr.c:97
getnewvnode_drop_reserve
void getnewvnode_drop_reserve(void)
Definition: vfs_subr.c:1038
mnt_vnode_next_active
static struct vnode * mnt_vnode_next_active(struct vnode **mvp, struct mount *mp)
Definition: vfs_subr.c:4870
SYNCER_SHUTDOWN_SPEEDUP
#define SYNCER_SHUTDOWN_SPEEDUP
syncer_shutdown
static void syncer_shutdown(void *arg, int howto)
Definition: vfs_subr.c:2071
vop_rmdir_post
void vop_rmdir_post(void *ap, int rc)
Definition: vfs_subr.c:4361
vfs_knl_assert_locked
static void vfs_knl_assert_locked(void *arg)
Definition: vfs_subr.c:4531
vop_rename_pre
void vop_rename_pre(void *ap)
Definition: vfs_subr.c:4175
speedup_syncer
int speedup_syncer(void)
Definition: vfs_subr.c:2051
brelvp
void brelvp(struct buf *bp)
Definition: vfs_subr.c:1760
numvnodes
static unsigned long numvnodes
Definition: vfs_subr.c:121
lockinit
void lockinit(struct lock *lk, int pri, const char *wmesg, int timo, int flags)
Definition: kern_lock.c:371
vn_start_secondary_write
int vn_start_secondary_write(struct vnode *vp, struct mount **mpp, int flags)
Definition: vfs_vnops.c:1537
hz
int hz
Definition: subr_param.c:84
count
int * count
Definition: cpufreq_if.m:63
WI_GIANTQ
#define WI_GIANTQ
Definition: vfs_subr.c:98
v_decr_usecount
static void v_decr_usecount(struct vnode *)
Definition: vfs_subr.c:2212