#include <sys/cdefs.h>
#include "opt_compat.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/capability.h>
#include <sys/eventhandler.h>
#include <sys/sysproto.h>
#include <sys/filedesc.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/kthread.h>
#include <sys/fcntl.h>
#include <sys/file.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/unistd.h>
#include <sys/posix4.h>
#include <sys/proc.h>
#include <sys/resourcevar.h>
#include <sys/signalvar.h>
#include <sys/protosw.h>
#include <sys/sema.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/syscall.h>
#include <sys/sysent.h>
#include <sys/sysctl.h>
#include <sys/sx.h>
#include <sys/taskqueue.h>
#include <sys/vnode.h>
#include <sys/conf.h>
#include <sys/event.h>
#include <sys/mount.h>
#include <machine/atomic.h>
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <vm/uma.h>
#include <sys/aio.h>
#include "opt_vfs_aio.h"

Include dependency graph for vfs_aio.c:

Data Structures
struct	oaiocb

struct	aiocblist

struct	aiothreadlist

struct	aioliojob

struct	kaioinfo

struct	aiocb_ops

Macros
#define	JOBST_NULL 0

#define	JOBST_JOBQSOCK 1

#define	JOBST_JOBQGLOBAL 2

#define	JOBST_JOBRUNNING 3

#define	JOBST_JOBFINISHED 4

#define	JOBST_JOBQBUF 5

#define	JOBST_JOBQSYNC 6

#define	MAX_AIO_PER_PROC 32

#define	MAX_AIO_QUEUE_PER_PROC 256 /* Bigger than AIO_LISTIO_MAX */

#define	MAX_AIO_PROCS 32

#define	MAX_AIO_QUEUE 1024 /* Bigger than AIO_LISTIO_MAX */

#define	TARGET_AIO_PROCS 4

#define	MAX_BUF_AIO 16

#define	AIOD_TIMEOUT_DEFAULT (10 * hz)

#define	AIOD_LIFETIME_DEFAULT (30 * hz)

#define	AIOCBLIST_DONE 0x01

#define	AIOCBLIST_BUFDONE 0x02

#define	AIOCBLIST_RUNDOWN 0x04

#define	AIOCBLIST_CHECKSYNC 0x08

#define	AIOP_FREE 0x1 /* proc on free queue */

#define	LIOJ_SIGNAL 0x1 /* signal on all done (lio) */

#define	LIOJ_SIGNAL_POSTED 0x2 /* signal has been posted */

#define	LIOJ_KEVENT_POSTED 0x4 /* kevent triggered */

#define	AIO_LOCK(ki) mtx_lock(&(ki)->kaio_mtx)

#define	AIO_UNLOCK(ki) mtx_unlock(&(ki)->kaio_mtx)

#define	AIO_LOCK_ASSERT(ki, f) mtx_assert(&(ki)->kaio_mtx, (f))

#define	AIO_MTX(ki) (&(ki)->kaio_mtx)

#define	KAIO_RUNDOWN 0x1 /* process is being run down */

#define	KAIO_WAKEUP 0x2 /* wakeup process when there is a significant event */

#define	DONE_BUF 1

#define	DONE_QUEUE 2

Typedefs
typedef struct oaiocb	oaiocb_t

Functions
	__FBSDID ("$BSDSUniX$")

	FEATURE (aio,"Asynchronous I/O")

static	MALLOC_DEFINE (M_LIO,"lio","listio aio control block list")

static	SYSCTL_NODE (_vfs, OID_AUTO, aio, CTLFLAG_RW, 0,"Async IO management")

	SYSCTL_INT (_vfs_aio, OID_AUTO, max_aio_procs, CTLFLAG_RW,&max_aio_procs, 0,"Maximum number of kernel threads to use for handling async IO ")

	SYSCTL_INT (_vfs_aio, OID_AUTO, num_aio_procs, CTLFLAG_RD,&num_aio_procs, 0,"Number of presently active kernel threads for async IO")

	SYSCTL_INT (_vfs_aio, OID_AUTO, target_aio_procs, CTLFLAG_RW,&target_aio_procs, 0,"Preferred number of ready kernel threads for async IO")

	SYSCTL_INT (_vfs_aio, OID_AUTO, max_aio_queue, CTLFLAG_RW,&max_queue_count, 0,"Maximum number of aio requests to queue, globally")

	SYSCTL_INT (_vfs_aio, OID_AUTO, num_queue_count, CTLFLAG_RD,&num_queue_count, 0,"Number of queued aio requests")

	SYSCTL_INT (_vfs_aio, OID_AUTO, num_buf_aio, CTLFLAG_RD,&num_buf_aio, 0,"Number of aio requests presently handled by the buf subsystem")

	SYSCTL_INT (_vfs_aio, OID_AUTO, aiod_timeout, CTLFLAG_RW,&aiod_timeout, 0,"Timeout value for synchronous aio operations")

	SYSCTL_INT (_vfs_aio, OID_AUTO, aiod_lifetime, CTLFLAG_RW,&aiod_lifetime, 0,"Maximum lifetime for idle aiod")

	SYSCTL_INT (_vfs_aio, OID_AUTO, unloadable, CTLFLAG_RW,&unloadable, 0,"Allow unload of aio (not recommended)")

	SYSCTL_INT (_vfs_aio, OID_AUTO, max_aio_per_proc, CTLFLAG_RW,&max_aio_per_proc, 0,"Maximum active aio requests per process (stored in the process)")

	SYSCTL_INT (_vfs_aio, OID_AUTO, max_aio_queue_per_proc, CTLFLAG_RW,&max_aio_queue_per_proc, 0,"Maximum queued aio requests per process (stored in the process)")

	SYSCTL_INT (_vfs_aio, OID_AUTO, max_buf_aio, CTLFLAG_RW,&max_buf_aio, 0,"Maximum buf aio requests per process (stored in the process)")

static	TAILQ_HEAD (aiothreadlist)

	TASKQUEUE_DEFINE_THREAD (aiod_bio)

static int	aio_modload (struct module module, int cmd, void arg)

	DECLARE_MODULE (aio, aio_mod, SI_SUB_VFS, SI_ORDER_ANY)

	MODULE_VERSION (aio, 1)

static int	aio_onceonly (void)

static int	aio_unload (void)

void	aio_init_aioinfo (struct proc *p)

static int	aio_sendsig (struct proc p, struct sigevent sigev, ksiginfo_t *ksi)

static int	aio_free_entry (struct aiocblist *aiocbe)

static void	aio_proc_rundown_exec (void arg, struct proc p, struct image_params *imgp __unused)

static void	aio_proc_rundown (void arg, struct proc p)

static struct aiocblist *	aio_selectjob (struct aiothreadlist *aiop)

static int	aio_fsync_vnode (struct thread td, struct vnode vp)

static void	aio_process (struct aiocblist *aiocbe)

static void	aio_bio_done_notify (struct proc userp, struct aiocblist aiocbe, int type)

static void	aio_daemon (void *_id)

static int	aio_newproc (int *start)

static int	aio_qphysio (struct proc p, struct aiocblist aiocbe)

static void	aio_swake_cb (struct socket so, struct sockbuf sb)

static int	convert_old_sigevent (struct osigevent osig, struct sigevent nsig)

static int	aiocb_copyin_old_sigevent (struct aiocb ujob, struct aiocb kjob)

static int	aiocb_copyin (struct aiocb ujob, struct aiocb kjob)

static long	aiocb_fetch_status (struct aiocb *ujob)

static long	aiocb_fetch_error (struct aiocb *ujob)

static int	aiocb_store_status (struct aiocb *ujob, long status)

static int	aiocb_store_error (struct aiocb *ujob, long error)

static int	aiocb_store_kernelinfo (struct aiocb *ujob, long jobref)

static int	aiocb_store_aiocb (struct aiocb *ujobp, struct aiocb ujob)

int	aio_aqueue (struct thread td, struct aiocb job, struct aioliojob lj, int type, struct aiocb_ops ops)

static void	aio_kick_nowait (struct proc *userp)

static int	aio_kick (struct proc *userp)

static void	aio_kick_helper (void *context, int pending)

static int	kern_aio_return (struct thread td, struct aiocb uaiocb, struct aiocb_ops *ops)

int	sys_aio_return (struct thread td, struct aio_return_args uap)

static int	kern_aio_suspend (struct thread td, int njoblist, struct aiocb ujoblist, struct timespec ts)

int	sys_aio_suspend (struct thread td, struct aio_suspend_args uap)

int	sys_aio_cancel (struct thread td, struct aio_cancel_args uap)

static int	kern_aio_error (struct thread td, struct aiocb aiocbp, struct aiocb_ops *ops)

int	sys_aio_error (struct thread td, struct aio_error_args uap)

int	sys_oaio_read (struct thread td, struct oaio_read_args uap)

int	sys_aio_read (struct thread td, struct aio_read_args uap)

int	sys_oaio_write (struct thread td, struct oaio_write_args uap)

int	sys_aio_write (struct thread td, struct aio_write_args uap)

static int	kern_lio_listio (struct thread td, int mode, struct aiocb const uacb_list, struct aiocb acb_list, int nent, struct sigevent sig, struct aiocb_ops *ops)

int	sys_olio_listio (struct thread td, struct olio_listio_args uap)

int	sys_lio_listio (struct thread td, struct lio_listio_args uap)

static void	aio_physwakeup (struct buf *bp)

static void	biohelper (void *context, int pending)

static int	kern_aio_waitcomplete (struct thread td, struct aiocb aiocbp, struct timespec ts, struct aiocb_ops *ops)

int	sys_aio_waitcomplete (struct thread td, struct aio_waitcomplete_args uap)

static int	kern_aio_fsync (struct thread td, int op, struct aiocb aiocbp, struct aiocb_ops *ops)

int	sys_aio_fsync (struct thread td, struct aio_fsync_args uap)

static int	filt_aioattach (struct knote *kn)

static void	filt_aiodetach (struct knote *kn)

static int	filt_aio (struct knote *kn, long hint)

static int	filt_lioattach (struct knote *kn)

static void	filt_liodetach (struct knote *kn)

static int	filt_lio (struct knote *kn, long hint)

Variables
static u_long	jobrefid

static uint64_t	jobseqno

static int	max_aio_procs = MAX_AIO_PROCS

static int	num_aio_procs = 0

static int	target_aio_procs = TARGET_AIO_PROCS

static int	max_queue_count = MAX_AIO_QUEUE

static int	num_queue_count = 0

static int	num_buf_aio = 0

static int	num_aio_resv_start = 0

static int	aiod_timeout

static int	aiod_lifetime

static int	unloadable = 0

static int	max_aio_per_proc = MAX_AIO_PER_PROC

static int	max_aio_queue_per_proc = MAX_AIO_QUEUE_PER_PROC

static int	max_buf_aio = MAX_BUF_AIO

static struct filterops	lio_filtops

static eventhandler_tag	exit_tag

static eventhandler_tag	exec_tag

static moduledata_t	aio_mod

static struct syscall_helper_data	aio_syscalls []

static struct aiocb_ops	aiocb_ops

static struct aiocb_ops	aiocb_ops_osigevent

Macro Definition Documentation

#define AIO_LOCK ( ki ) mtx_lock(&(ki)->kaio_mtx)

Definition at line 308 of file vfs_aio.c.

Referenced by aio_aqueue(), aio_daemon(), aio_free_entry(), aio_proc_rundown(), aio_qphysio(), biohelper(), kern_aio_error(), kern_aio_return(), kern_aio_suspend(), kern_aio_waitcomplete(), kern_lio_listio(), and sys_aio_cancel().

#define AIO_LOCK_ASSERT	(	ki,
		f
	)	mtx_assert(&(ki)->kaio_mtx, (f))

Definition at line 310 of file vfs_aio.c.

Referenced by aio_bio_done_notify(), and aio_free_entry().

#define AIO_MTX ( ki ) (&(ki)->kaio_mtx)

Definition at line 311 of file vfs_aio.c.

Referenced by aio_aqueue(), aio_proc_rundown(), kern_aio_suspend(), kern_aio_waitcomplete(), and kern_lio_listio().

#define AIO_UNLOCK ( ki ) mtx_unlock(&(ki)->kaio_mtx)

Definition at line 309 of file vfs_aio.c.

Referenced by aio_aqueue(), aio_daemon(), aio_free_entry(), aio_proc_rundown(), aio_qphysio(), biohelper(), kern_aio_error(), kern_aio_return(), kern_aio_suspend(), kern_aio_waitcomplete(), kern_lio_listio(), and sys_aio_cancel().

#define AIOCBLIST_BUFDONE 0x02

Definition at line 252 of file vfs_aio.c.

Referenced by aio_bio_done_notify().

#define AIOCBLIST_CHECKSYNC 0x08

Definition at line 254 of file vfs_aio.c.

Referenced by aio_aqueue(), and aio_bio_done_notify().

#define AIOCBLIST_DONE 0x01

Definition at line 251 of file vfs_aio.c.

Referenced by aio_bio_done_notify().

#define AIOCBLIST_RUNDOWN 0x04

Definition at line 253 of file vfs_aio.c.

#define AIOD_LIFETIME_DEFAULT (30 * hz)

Definition at line 122 of file vfs_aio.c.

Referenced by aio_onceonly().

#define AIOD_TIMEOUT_DEFAULT (10 * hz)

Definition at line 118 of file vfs_aio.c.

Referenced by aio_onceonly().

#define AIOP_FREE 0x1 /* proc on free queue */

Definition at line 259 of file vfs_aio.c.

Referenced by aio_daemon(), aio_kick(), and aio_kick_nowait().

#define DONE_BUF 1

Referenced by biohelper().

#define DONE_QUEUE 2

Referenced by aio_bio_done_notify(), aio_daemon(), aio_proc_rundown(), and sys_aio_cancel().

#define JOBST_JOBFINISHED 4

Definition at line 89 of file vfs_aio.c.

Referenced by aio_bio_done_notify(), aio_free_entry(), aio_proc_rundown(), filt_aio(), kern_aio_error(), kern_aio_return(), kern_aio_suspend(), and kern_aio_waitcomplete().

#define JOBST_JOBQBUF 5

Definition at line 90 of file vfs_aio.c.

Referenced by aio_qphysio().

#define JOBST_JOBQGLOBAL 2

Definition at line 87 of file vfs_aio.c.

Referenced by aio_aqueue(), aio_bio_done_notify(), aio_proc_rundown(), and sys_aio_cancel().

#define JOBST_JOBQSOCK 1

Definition at line 86 of file vfs_aio.c.

Referenced by aio_aqueue(), aio_proc_rundown(), aio_swake_cb(), and sys_aio_cancel().

#define JOBST_JOBQSYNC 6

Definition at line 91 of file vfs_aio.c.

Referenced by aio_aqueue(), aio_proc_rundown(), and sys_aio_cancel().

#define JOBST_JOBRUNNING 3

Definition at line 88 of file vfs_aio.c.

Referenced by aio_selectjob().

#define JOBST_NULL 0

Definition at line 85 of file vfs_aio.c.

Referenced by aio_free_entry().

#define KAIO_RUNDOWN 0x1 /* process is being run down */

Definition at line 313 of file vfs_aio.c.

Referenced by aio_bio_done_notify(), and aio_proc_rundown().

#define KAIO_WAKEUP 0x2 /* wakeup process when there is a significant event */

Definition at line 314 of file vfs_aio.c.

Referenced by aio_bio_done_notify(), aio_proc_rundown(), kern_aio_suspend(), kern_aio_waitcomplete(), and kern_lio_listio().

#define LIOJ_KEVENT_POSTED 0x4 /* kevent triggered */

Definition at line 282 of file vfs_aio.c.

Referenced by aio_bio_done_notify(), filt_lio(), and kern_lio_listio().

#define LIOJ_SIGNAL 0x1 /* signal on all done (lio) */

Definition at line 280 of file vfs_aio.c.

Referenced by aio_bio_done_notify(), and kern_lio_listio().

#define LIOJ_SIGNAL_POSTED 0x2 /* signal has been posted */

Definition at line 281 of file vfs_aio.c.

Referenced by aio_bio_done_notify(), and kern_lio_listio().

#define MAX_AIO_PER_PROC 32

Definition at line 94 of file vfs_aio.c.

#define MAX_AIO_PROCS 32

Definition at line 102 of file vfs_aio.c.

#define MAX_AIO_QUEUE 1024 /* Bigger than AIO_LISTIO_MAX */

Definition at line 106 of file vfs_aio.c.

Referenced by aio_onceonly().

#define MAX_AIO_QUEUE_PER_PROC 256 /* Bigger than AIO_LISTIO_MAX */

Definition at line 98 of file vfs_aio.c.

#define MAX_BUF_AIO 16

Definition at line 114 of file vfs_aio.c.

#define TARGET_AIO_PROCS 4

Definition at line 110 of file vfs_aio.c.

Typedef Documentation

typedef struct oaiocb oaiocb_t

Function Documentation

__FBSDID ( "$BSDSUniX$" )

int aio_aqueue	(	struct thread *	td,
		struct aiocb *	job,
		struct aioliojob *	lj,
		int	type,
		struct aiocb_ops *	ops
	)

Definition at line 1524 of file vfs_aio.c.

References aio_init_aioinfo(), aio_kick_nowait(), AIO_LOCK, AIO_MTX, aio_qphysio(), AIO_UNLOCK, AIOCBLIST_CHECKSYNC, aiocb_ops::copyin, crhold(), fd, fget(), fget_read(), fget_write(), jobrefid, jobseqno, JOBST_JOBQGLOBAL, JOBST_JOBQSOCK, JOBST_JOBQSYNC, kaioinfo::kaio_count, kaioinfo::kaio_qallowed_count, knlist_init_mtx(), kqfd_register(), aioliojob::lioj_count, max_queue_count, num_queue_count, aiocb_ops::store_error, aiocb_ops::store_kernelinfo, aiocb_ops::store_status, and type.

Referenced by kern_aio_fsync(), kern_lio_listio(), sys_aio_read(), sys_aio_write(), sys_oaio_read(), and sys_oaio_write().

Here is the call graph for this function:

Here is the caller graph for this function:

static void aio_bio_done_notify	(	struct proc *	userp,
		struct aiocblist *	aiocbe,
		int	type
	)

static

Definition at line 959 of file vfs_aio.c.

References aio_kick_nowait(), AIO_LOCK_ASSERT, aio_sendsig(), AIOCBLIST_BUFDONE, AIOCBLIST_CHECKSYNC, AIOCBLIST_DONE, DONE_QUEUE, JOBST_JOBFINISHED, JOBST_JOBQGLOBAL, kaioinfo::kaio_flags, KAIO_RUNDOWN, KAIO_WAKEUP, aioliojob::lioj_count, aioliojob::lioj_finished_count, aioliojob::lioj_flags, LIOJ_KEVENT_POSTED, aioliojob::lioj_signal, LIOJ_SIGNAL, LIOJ_SIGNAL_POSTED, and wakeup().

Referenced by aio_daemon(), aio_proc_rundown(), biohelper(), and sys_aio_cancel().

Here is the call graph for this function:

Here is the caller graph for this function:

static void aio_daemon ( void * _id )

static

Definition at line 1033 of file vfs_aio.c.

References aio_bio_done_notify(), AIO_LOCK, aio_process(), aio_selectjob(), AIO_UNLOCK, aiod_lifetime, AIOP_FREE, aiothreadlist::aiothreadflags, DONE_QUEUE, free_unr(), kaioinfo::kaio_active_count, kproc_exit(), num_aio_procs, panic(), printf(), sys_setsid(), and target_aio_procs.

Referenced by aio_newproc().

Here is the call graph for this function:

Here is the caller graph for this function:

static int aio_free_entry ( struct aiocblist * aiocbe )

static

Definition at line 635 of file vfs_aio.c.

References AIO_LOCK, AIO_LOCK_ASSERT, AIO_UNLOCK, crfree(), JOBST_JOBFINISHED, JOBST_NULL, kaioinfo::kaio_count, aioliojob::lioj_count, aioliojob::lioj_finished_count, num_queue_count, and sigqueue_take().

Referenced by aio_proc_rundown(), kern_aio_return(), and kern_aio_waitcomplete().

Here is the call graph for this function:

Here is the caller graph for this function:

static int aio_fsync_vnode	(	struct thread *	td,
		struct vnode *	vp
	)

static

Definition at line 835 of file vfs_aio.c.

References vn_finished_write(), and vn_start_write().

Referenced by aio_process().

Here is the call graph for this function:

Here is the caller graph for this function:

void aio_init_aioinfo ( struct proc * p )

Definition at line 575 of file vfs_aio.c.

References aio_kick_helper(), aio_newproc(), kaioinfo::kaio_active_count, kaioinfo::kaio_ballowed_count, kaioinfo::kaio_buffer_count, kaioinfo::kaio_count, kaioinfo::kaio_flags, kaioinfo::kaio_maxactive_count, kaioinfo::kaio_mtx, kaioinfo::kaio_qallowed_count, max_aio_per_proc, max_aio_procs, max_aio_queue_per_proc, max_buf_aio, mtx_destroy(), mtx_init(), num_aio_procs, and target_aio_procs.

Referenced by aio_aqueue(), kern_aio_fsync(), kern_aio_waitcomplete(), and kern_lio_listio().

Here is the call graph for this function:

Here is the caller graph for this function:

static int aio_kick ( struct proc * userp )

static

Definition at line 1801 of file vfs_aio.c.

References aio_newproc(), AIOP_FREE, aiothreadlist::aiothreadflags, kaioinfo::kaio_active_count, kaioinfo::kaio_maxactive_count, max_aio_procs, num_aio_procs, num_aio_resv_start, and wakeup().

Referenced by aio_kick_helper().

Here is the call graph for this function:

Here is the caller graph for this function:

static void aio_kick_helper	(	void *	context,
		int	pending
	)

static

Definition at line 1831 of file vfs_aio.c.

References aio_kick().

Referenced by aio_init_aioinfo().

Here is the call graph for this function:

Here is the caller graph for this function:

static void aio_kick_nowait ( struct proc * userp )

static

Definition at line 1783 of file vfs_aio.c.

References AIOP_FREE, aiothreadlist::aiothreadflags, kaioinfo::kaio_active_count, kaioinfo::kaio_maxactive_count, max_aio_procs, num_aio_procs, num_aio_resv_start, taskqueue_enqueue(), and wakeup().

Referenced by aio_aqueue(), aio_bio_done_notify(), and aio_swake_cb().

Here is the call graph for this function:

Here is the caller graph for this function:

static int aio_modload	(	struct module *	module,
		int	cmd,
		void *	arg
	)

static

Definition at line 397 of file vfs_aio.c.

References aio_onceonly(), and aio_unload().

Here is the call graph for this function:

static int aio_newproc ( int * start )

static

Definition at line 1216 of file vfs_aio.c.

References aio_daemon(), alloc_unr(), free_unr(), kproc_create(), and num_aio_procs.

Referenced by aio_init_aioinfo(), and aio_kick().

Here is the call graph for this function:

Here is the caller graph for this function:

static int aio_onceonly ( void )

static

Definition at line 473 of file vfs_aio.c.

References aio_proc_rundown(), aio_proc_rundown_exec(), aio_swake, aio_swake_cb(), aiod_lifetime, AIOD_LIFETIME_DEFAULT, aiod_timeout, AIOD_TIMEOUT_DEFAULT, async_io_version, jobrefid, kqueue_add_filteropts(), MAX_AIO_QUEUE, mtx_init(), new_unrhdr(), p31b_setcfg(), sema_init(), and syscall_helper_register().

Referenced by aio_modload().

Here is the call graph for this function:

Here is the caller graph for this function:

static void aio_physwakeup ( struct buf * bp )

static

Definition at line 2373 of file vfs_aio.c.

References taskqueue_enqueue().

Referenced by aio_qphysio().

Here is the call graph for this function:

Here is the caller graph for this function:

static void aio_proc_rundown	(	void *	arg,
		struct proc *	p
	)

static

Definition at line 721 of file vfs_aio.c.

References aio_bio_done_notify(), aio_free_entry(), AIO_LOCK, AIO_MTX, AIO_UNLOCK, DONE_QUEUE, hz, JOBST_JOBFINISHED, JOBST_JOBQGLOBAL, JOBST_JOBQSOCK, JOBST_JOBQSYNC, kaioinfo::kaio_flags, kaioinfo::kaio_mtx, KAIO_RUNDOWN, KAIO_WAKEUP, aioliojob::lioj_count, aioliojob::lioj_finished_count, mtx_destroy(), panic(), sigqueue_take(), and taskqueue_drain().

Referenced by aio_onceonly(), and aio_proc_rundown_exec().

Here is the call graph for this function:

Here is the caller graph for this function:

static void aio_proc_rundown_exec	(	void *	arg,
		struct proc *	p,
		struct image_params *imgp	__unused
	)

static

Definition at line 712 of file vfs_aio.c.

References aio_proc_rundown().

Referenced by aio_onceonly().

Here is the call graph for this function:

Here is the caller graph for this function:

static void aio_process ( struct aiocblist * aiocbe )

static

Definition at line 868 of file vfs_aio.c.

References aio_fsync_vnode(), bwillwrite(), and kern_psignal().

Referenced by aio_daemon().

Here is the call graph for this function:

Here is the caller graph for this function:

static int aio_qphysio	(	struct proc *	p,
		struct aiocblist *	aiocbe
	)

static

Definition at line 1251 of file vfs_aio.c.

References AIO_LOCK, aio_physwakeup(), AIO_UNLOCK, biohelper(), buf, dev_relthread(), dev_strategy_csw(), devvn_refthread(), JOBST_JOBQBUF, kaioinfo::kaio_ballowed_count, kaioinfo::kaio_buffer_count, kaioinfo::kaio_count, aioliojob::lioj_count, num_buf_aio, num_queue_count, vmapbuf(), and vn_isdisk().

Referenced by aio_aqueue().

Here is the call graph for this function:

Here is the caller graph for this function:

static struct aiocblist* aio_selectjob ( struct aiothreadlist * aiop )

static

Definition at line 808 of file vfs_aio.c.

References JOBST_JOBRUNNING, kaioinfo::kaio_active_count, and kaioinfo::kaio_maxactive_count.

Referenced by aio_daemon().

Here is the caller graph for this function:

static int aio_sendsig	(	struct proc *	p,
		struct sigevent *	sigev,
		ksiginfo_t *	ksi
	)

static

Definition at line 611 of file vfs_aio.c.

References sigev_findtd(), and tdsendsignal().

Referenced by aio_bio_done_notify(), and kern_lio_listio().

Here is the call graph for this function:

Here is the caller graph for this function:

static void aio_swake_cb	(	struct socket *	so,
		struct sockbuf *	sb
	)

static

Definition at line 1380 of file vfs_aio.c.

References aio_kick_nowait(), JOBST_JOBQSOCK, and panic().

Referenced by aio_onceonly().

Here is the call graph for this function:

Here is the caller graph for this function:

static int aio_unload ( void )

static

Definition at line 524 of file vfs_aio.c.

References aio_swake, async_io_version, delete_unrhdr(), kqueue_del_filteropts(), mtx_destroy(), p31b_setcfg(), sema_destroy(), syscall_helper_unregister(), taskqueue_free(), and unloadable.

Referenced by aio_modload().

Here is the call graph for this function:

Here is the caller graph for this function:

static int aiocb_copyin	(	struct aiocb *	ujob,
		struct aiocb *	kjob
	)

static

Definition at line 1451 of file vfs_aio.c.

static int aiocb_copyin_old_sigevent	(	struct aiocb *	ujob,
		struct aiocb *	kjob
	)

static

Definition at line 1437 of file vfs_aio.c.

References oaiocb::aio_sigevent, and convert_old_sigevent().

Here is the call graph for this function:

static long aiocb_fetch_error ( struct aiocb * ujob )

static

Definition at line 1465 of file vfs_aio.c.

static long aiocb_fetch_status ( struct aiocb * ujob )

static

Definition at line 1458 of file vfs_aio.c.

static int aiocb_store_aiocb	(	struct aiocb **	ujobp,
		struct aiocb *	ujob
	)

static

Definition at line 1493 of file vfs_aio.c.

References suword.

static int aiocb_store_error	(	struct aiocb *	ujob,
		long	error
	)

static

Definition at line 1479 of file vfs_aio.c.

References suword.

static int aiocb_store_kernelinfo	(	struct aiocb *	ujob,
		long	jobref
	)

static

Definition at line 1486 of file vfs_aio.c.

References suword.

static int aiocb_store_status	(	struct aiocb *	ujob,
		long	status
	)

static

Definition at line 1472 of file vfs_aio.c.

References suword.

static void biohelper	(	void *	context,
		int	pending
	)

static

Definition at line 2385 of file vfs_aio.c.

References aio_bio_done_notify(), AIO_LOCK, AIO_UNLOCK, buf, DONE_BUF, kaioinfo::kaio_buffer_count, num_buf_aio, and vunmapbuf().

Referenced by aio_qphysio().

Here is the call graph for this function:

Here is the caller graph for this function:

static int convert_old_sigevent	(	struct osigevent *	osig,
		struct sigevent *	nsig
	)

static

Definition at line 1411 of file vfs_aio.c.

Referenced by aiocb_copyin_old_sigevent(), and sys_olio_listio().

Here is the caller graph for this function:

DECLARE_MODULE	(	aio	,
		aio_mod	,
		SI_SUB_VFS	,
		SI_ORDER_ANY
	)

FEATURE	(	aio	,
		"Asynchronous I/O"
	)

static int filt_aio	(	struct knote *	kn,
		long	hint
	)

static

Definition at line 2561 of file vfs_aio.c.

References JOBST_JOBFINISHED.

static int filt_aioattach ( struct knote * kn )

static

Definition at line 2526 of file vfs_aio.c.

References knlist_add().

Here is the call graph for this function:

static void filt_aiodetach ( struct knote * kn )

static

Definition at line 2547 of file vfs_aio.c.

References knlist_empty(), and knlist_remove().

Here is the call graph for this function:

static int filt_lio	(	struct knote *	kn,
		long	hint
	)

static

Definition at line 2609 of file vfs_aio.c.

References aioliojob::lioj_flags, and LIOJ_KEVENT_POSTED.

static int filt_lioattach ( struct knote * kn )

static

Definition at line 2574 of file vfs_aio.c.

References knlist_add().

Here is the call graph for this function:

static void filt_liodetach ( struct knote * kn )

static

Definition at line 2595 of file vfs_aio.c.

References knlist_empty(), and knlist_remove().

Here is the call graph for this function:

static int kern_aio_error	(	struct thread *	td,
		struct aiocb *	aiocbp,
		struct aiocb_ops *	ops
	)

static

Definition at line 2085 of file vfs_aio.c.

References AIO_LOCK, AIO_UNLOCK, aiocb_ops::fetch_error, aiocb_ops::fetch_status, and JOBST_JOBFINISHED.

Referenced by sys_aio_error().

Here is the caller graph for this function:

static int kern_aio_fsync	(	struct thread *	td,
		int	op,
		struct aiocb *	aiocbp,
		struct aiocb_ops *	ops
	)

static

Definition at line 2503 of file vfs_aio.c.

References aio_aqueue(), and aio_init_aioinfo().

Referenced by sys_aio_fsync().

Here is the call graph for this function:

Here is the caller graph for this function:

static int kern_aio_return	(	struct thread *	td,
		struct aiocb *	uaiocb,
		struct aiocb_ops *	ops
	)

static

Definition at line 1848 of file vfs_aio.c.

References aio_free_entry(), AIO_LOCK, AIO_UNLOCK, JOBST_JOBFINISHED, aiocb_ops::store_error, and aiocb_ops::store_status.

Referenced by sys_aio_return().

Here is the call graph for this function:

Here is the caller graph for this function:

static int kern_aio_suspend	(	struct thread *	td,
		int	njoblist,
		struct aiocb **	ujoblist,
		struct timespec *	ts
	)

static

Definition at line 1897 of file vfs_aio.c.

References AIO_LOCK, AIO_MTX, AIO_UNLOCK, itimerfix(), JOBST_JOBFINISHED, kaioinfo::kaio_flags, KAIO_WAKEUP, and tvtohz().

Referenced by sys_aio_suspend().

Here is the call graph for this function:

Here is the caller graph for this function:

static int kern_aio_waitcomplete	(	struct thread *	td,
		struct aiocb **	aiocbp,
		struct timespec *	ts,
		struct aiocb_ops *	ops
	)

static

Definition at line 2420 of file vfs_aio.c.

References aio_free_entry(), aio_init_aioinfo(), AIO_LOCK, AIO_MTX, AIO_UNLOCK, itimerfix(), JOBST_JOBFINISHED, kaioinfo::kaio_flags, KAIO_WAKEUP, aiocb_ops::store_aiocb, aiocb_ops::store_error, aiocb_ops::store_status, and tvtohz().

Referenced by sys_aio_waitcomplete().

Here is the call graph for this function:

Here is the caller graph for this function:

static int kern_lio_listio	(	struct thread *	td,
		int	mode,
		struct aiocb const	uacb_list,
		struct aiocb **	acb_list,
		int	nent,
		struct sigevent *	sig,
		struct aiocb_ops *	ops
	)

static

Definition at line 2165 of file vfs_aio.c.

References aio_aqueue(), aio_init_aioinfo(), AIO_LOCK, AIO_MTX, aio_sendsig(), AIO_UNLOCK, kaioinfo::kaio_flags, KAIO_WAKEUP, knlist_init_mtx(), kqfd_register(), aioliojob::lioj_count, aioliojob::lioj_finished_count, aioliojob::lioj_flags, LIOJ_KEVENT_POSTED, aioliojob::lioj_signal, LIOJ_SIGNAL, LIOJ_SIGNAL_POSTED, and sigqueue_take().

Referenced by sys_lio_listio(), and sys_olio_listio().

Here is the call graph for this function:

Here is the caller graph for this function:

static MALLOC_DEFINE	(	M_LIO	,
		"lio"	,
		"listio aio control block list"
	)

static

MODULE_VERSION	(	aio	,
		1
	)

int sys_aio_cancel	(	struct thread *	td,
		struct aio_cancel_args *	uap
	)

Definition at line 1986 of file vfs_aio.c.

References aio_bio_done_notify(), AIO_LOCK, AIO_UNLOCK, DONE_QUEUE, fget(), JOBST_JOBQGLOBAL, JOBST_JOBQSOCK, JOBST_JOBQSYNC, and vn_isdisk().

Here is the call graph for this function:

int sys_aio_error	(	struct thread *	td,
		struct aio_error_args *	uap
	)

Definition at line 2126 of file vfs_aio.c.

References kern_aio_error().

Here is the call graph for this function:

int sys_aio_fsync	(	struct thread *	td,
		struct aio_fsync_args *	uap
	)

Definition at line 2518 of file vfs_aio.c.

References kern_aio_fsync().

Here is the call graph for this function:

int sys_aio_read	(	struct thread *	td,
		struct aio_read_args *	uap
	)

Definition at line 2142 of file vfs_aio.c.

References aio_aqueue().

Here is the call graph for this function:

int sys_aio_return	(	struct thread *	td,
		struct aio_return_args *	uap
	)

Definition at line 1887 of file vfs_aio.c.

References kern_aio_return().

Here is the call graph for this function:

int sys_aio_suspend	(	struct thread *	td,
		struct aio_suspend_args *	uap
	)

Definition at line 1956 of file vfs_aio.c.

References kern_aio_suspend(), and ts.

Here is the call graph for this function:

int sys_aio_waitcomplete	(	struct thread *	td,
		struct aio_waitcomplete_args *	uap
	)

Definition at line 2485 of file vfs_aio.c.

References kern_aio_waitcomplete(), and ts.

Here is the call graph for this function:

int sys_aio_write	(	struct thread *	td,
		struct aio_write_args *	uap
	)

Definition at line 2158 of file vfs_aio.c.

References aio_aqueue().

Here is the call graph for this function:

int sys_lio_listio	(	struct thread *	td,
		struct lio_listio_args *	uap
	)

Definition at line 2338 of file vfs_aio.c.

References free(), kern_lio_listio(), and malloc().

Here is the call graph for this function:

int sys_oaio_read	(	struct thread *	td,
		struct oaio_read_args *	uap
	)

Definition at line 2134 of file vfs_aio.c.

References aio_aqueue().

Here is the call graph for this function:

int sys_oaio_write	(	struct thread *	td,
		struct oaio_write_args *	uap
	)

Definition at line 2150 of file vfs_aio.c.

References aio_aqueue().

Here is the call graph for this function:

int sys_olio_listio	(	struct thread *	td,
		struct olio_listio_args *	uap
	)

Definition at line 2301 of file vfs_aio.c.

References convert_old_sigevent(), free(), kern_lio_listio(), and malloc().

Here is the call graph for this function:

SYSCTL_INT	(	_vfs_aio	,
		OID_AUTO	,
		max_aio_procs	,
		CTLFLAG_RW	,
		&	max_aio_procs,
		0	,
		"Maximum number of kernel threads to use for handling async IO "
	)

SYSCTL_INT	(	_vfs_aio	,
		OID_AUTO	,
		num_aio_procs	,
		CTLFLAG_RD	,
		&	num_aio_procs,
		0	,
		"Number of presently active kernel threads for async IO"
	)

SYSCTL_INT	(	_vfs_aio	,
		OID_AUTO	,
		target_aio_procs	,
		CTLFLAG_RW	,
		&	target_aio_procs,
		0	,
		"Preferred number of ready kernel threads for async IO"
	)

SYSCTL_INT	(	_vfs_aio	,
		OID_AUTO	,
		max_aio_queue	,
		CTLFLAG_RW	,
		&	max_queue_count,
		0	,
		"Maximum number of aio requests to	queue,
		globally"
	)

SYSCTL_INT	(	_vfs_aio	,
		OID_AUTO	,
		num_queue_count	,
		CTLFLAG_RD	,
		&	num_queue_count,
		0	,
		"Number of queued aio requests"
	)

SYSCTL_INT	(	_vfs_aio	,
		OID_AUTO	,
		num_buf_aio	,
		CTLFLAG_RD	,
		&	num_buf_aio,
		0	,
		"Number of aio requests presently handled by the buf subsystem"
	)

SYSCTL_INT	(	_vfs_aio	,
		OID_AUTO	,
		aiod_timeout	,
		CTLFLAG_RW	,
		&	aiod_timeout,
		0	,
		"Timeout value for synchronous aio operations"
	)

SYSCTL_INT	(	_vfs_aio	,
		OID_AUTO	,
		aiod_lifetime	,
		CTLFLAG_RW	,
		&	aiod_lifetime,
		0	,
		"Maximum lifetime for idle aiod"
	)

SYSCTL_INT	(	_vfs_aio	,
		OID_AUTO	,
		unloadable	,
		CTLFLAG_RW	,
		&	unloadable,
		0	,
		"Allow unload of aio (not recommended)"
	)

SYSCTL_INT	(	_vfs_aio	,
		OID_AUTO	,
		max_aio_per_proc	,
		CTLFLAG_RW	,
		&	max_aio_per_proc,
		0	,
		"Maximum active aio requests per process (stored in the process)"
	)

SYSCTL_INT	(	_vfs_aio	,
		OID_AUTO	,
		max_aio_queue_per_proc	,
		CTLFLAG_RW	,
		&	max_aio_queue_per_proc,
		0	,
		"Maximum queued aio requests per process (stored in the process)"
	)

SYSCTL_INT	(	_vfs_aio	,
		OID_AUTO	,
		max_buf_aio	,
		CTLFLAG_RW	,
		&	max_buf_aio,
		0	,
		"Maximum buf aio requests per process (stored in the process)"
	)

static SYSCTL_NODE	(	_vfs	,
		OID_AUTO	,
		aio	,
		CTLFLAG_RW	,
		0	,
		"Async IO management"
	)

static

static TAILQ_HEAD ( aiothreadlist )

static

Definition at line 330 of file vfs_aio.c.

TASKQUEUE_DEFINE_THREAD ( aiod_bio )

Variable Documentation

moduledata_t aio_mod

static

Initial value:

= {
        "aio",
        &aio_modload,
        NULL
}

Definition at line 417 of file vfs_aio.c.

struct syscall_helper_data aio_syscalls[]

static

Initial value:

= {
        SYSCALL_INIT_HELPER(aio_cancel),
        SYSCALL_INIT_HELPER(aio_error),
        SYSCALL_INIT_HELPER(aio_fsync),
        SYSCALL_INIT_HELPER(aio_read),
        SYSCALL_INIT_HELPER(aio_return),
        SYSCALL_INIT_HELPER(aio_suspend),
        SYSCALL_INIT_HELPER(aio_waitcomplete),
        SYSCALL_INIT_HELPER(aio_write),
        SYSCALL_INIT_HELPER(lio_listio),
        SYSCALL_INIT_HELPER(oaio_read),
        SYSCALL_INIT_HELPER(oaio_write),
        SYSCALL_INIT_HELPER(olio_listio),
        SYSCALL_INIT_LAST
}

Definition at line 423 of file vfs_aio.c.

struct aiocb_ops aiocb_ops

static

Initial value:

= {
        .copyin = aiocb_copyin,
        .fetch_status = aiocb_fetch_status,
        .fetch_error = aiocb_fetch_error,
        .store_status = aiocb_store_status,
        .store_error = aiocb_store_error,
        .store_kernelinfo = aiocb_store_kernelinfo,
        .store_aiocb = aiocb_store_aiocb,
}

Definition at line 1499 of file vfs_aio.c.

struct aiocb_ops aiocb_ops_osigevent

static

Initial value:

= {
        .copyin = aiocb_copyin_old_sigevent,
        .fetch_status = aiocb_fetch_status,
        .fetch_error = aiocb_fetch_error,
        .store_status = aiocb_store_status,
        .store_error = aiocb_store_error,
        .store_kernelinfo = aiocb_store_kernelinfo,
        .store_aiocb = aiocb_store_aiocb,
}

Definition at line 1509 of file vfs_aio.c.

int aiod_lifetime

static

Definition at line 169 of file vfs_aio.c.

Referenced by aio_daemon(), and aio_onceonly().

int aiod_timeout

static

Definition at line 165 of file vfs_aio.c.

Referenced by aio_onceonly().

eventhandler_tag exec_tag

static

Definition at line 389 of file vfs_aio.c.

eventhandler_tag exit_tag

static

Definition at line 389 of file vfs_aio.c.

u_long jobrefid

static

Definition at line 78 of file vfs_aio.c.

Referenced by aio_aqueue(), and aio_onceonly().

uint64_t jobseqno

static

Definition at line 83 of file vfs_aio.c.

Referenced by aio_aqueue().

struct filterops lio_filtops

static

Initial value:

= {
        .f_isfd = 0,
        .f_attach = filt_lioattach,
        .f_detach = filt_liodetach,
        .f_event = filt_lio
}

Definition at line 382 of file vfs_aio.c.

int max_aio_per_proc = MAX_AIO_PER_PROC

static

Definition at line 178 of file vfs_aio.c.

Referenced by aio_init_aioinfo().

int max_aio_procs = MAX_AIO_PROCS

static

Definition at line 131 of file vfs_aio.c.

Referenced by aio_init_aioinfo(), aio_kick(), and aio_kick_nowait().

int max_aio_queue_per_proc = MAX_AIO_QUEUE_PER_PROC

static

Definition at line 182 of file vfs_aio.c.

Referenced by aio_init_aioinfo().

int max_buf_aio = MAX_BUF_AIO

static

Definition at line 187 of file vfs_aio.c.

Referenced by aio_init_aioinfo().

int max_queue_count = MAX_AIO_QUEUE

static

Definition at line 149 of file vfs_aio.c.

Referenced by aio_aqueue().

int num_aio_procs = 0

static

Definition at line 136 of file vfs_aio.c.

Referenced by aio_daemon(), aio_init_aioinfo(), aio_kick(), aio_kick_nowait(), and aio_newproc().

int num_aio_resv_start = 0

static

Definition at line 163 of file vfs_aio.c.

Referenced by aio_kick(), and aio_kick_nowait().

int num_buf_aio = 0

static

Definition at line 157 of file vfs_aio.c.

Referenced by aio_qphysio(), and biohelper().

int num_queue_count = 0

static

Definition at line 153 of file vfs_aio.c.

Referenced by aio_aqueue(), aio_free_entry(), and aio_qphysio().

int target_aio_procs = TARGET_AIO_PROCS

static

Definition at line 145 of file vfs_aio.c.

Referenced by aio_daemon(), and aio_init_aioinfo().

int unloadable = 0

static

Definition at line 173 of file vfs_aio.c.

Referenced by aio_unload().

Data Structures

Macros

Typedefs

Functions

Variables

Macro Definition Documentation

Typedef Documentation

Function Documentation

Variable Documentation