#include <sys/cdefs.h>
#include "opt_compat.h"
#include "opt_umtx_profiling.h"
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/sysent.h>
#include <sys/systm.h>
#include <sys/sysproto.h>
#include <sys/syscallsubr.h>
#include <sys/eventhandler.h>
#include <sys/umtx.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <machine/cpu.h>

Include dependency graph for kern_umtx.c:

Data Structures
struct	umtx_pi

struct	umtx_q

struct	umtxq_queue

struct	umtxq_chain

Macros
#define	_UMUTEX_TRY 1

#define	_UMUTEX_WAIT 2

#define	UQF_UMTXQ 0x0001

#define	UMTX_SHARED_QUEUE 0

#define	UMTX_EXCLUSIVE_QUEUE 1

#define	UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED)

#define	UMTXQ_BUSY_ASSERT(uc) KASSERT(&(uc)->uc_busy, ("umtx chain is not busy"))

#define	UPRI(td)

#define	GOLDEN_RATIO_PRIME 2654404609U

#define	UMTX_CHAINS 512

#define	UMTX_SHIFTS (__WORD_BIT - 9)

#define	GET_SHARE(flags) (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE)

#define	BUSY_SPINS 200

#define	umtxq_signal(key, nwake) umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE)

#define	umtxq_insert(uq) umtxq_insert_queue((uq), UMTX_SHARED_QUEUE)

#define	umtxq_remove(uq) umtxq_remove_queue((uq), UMTX_SHARED_QUEUE)

#define	BATCH_SIZE 128

Typedefs
typedef int(*	_umtx_op_func) (struct thread td, struct _umtx_op_args uap)

Functions
	__FBSDID ("$BSDSUniX$")

	TAILQ_HEAD (umtxq_head, umtx_q)

	LIST_HEAD (umtxq_list, umtxq_queue)

static	MALLOC_DEFINE (M_UMTX,"umtx","UMTX queue memory")

static	SYSCTL_NODE (_debug, OID_AUTO, umtx, CTLFLAG_RW, 0,"umtx debug")

	SYSCTL_INT (_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD,&umtx_pi_allocated, 0,"Allocated umtx_pi")

static void	umtxq_sysinit (void *)

static void	umtxq_hash (struct umtx_key *key)

static struct umtxq_chain *	umtxq_getchain (struct umtx_key *key)

static void	umtxq_lock (struct umtx_key *key)

static void	umtxq_unlock (struct umtx_key *key)

static void	umtxq_busy (struct umtx_key *key)

static void	umtxq_unbusy (struct umtx_key *key)

static void	umtxq_insert_queue (struct umtx_q *uq, int q)

static void	umtxq_remove_queue (struct umtx_q *uq, int q)

static int	umtxq_sleep (struct umtx_q uq, const char wmesg, int timo)

static int	umtxq_count (struct umtx_key *key)

static struct umtx_pi *	umtx_pi_alloc (int)

static void	umtx_pi_free (struct umtx_pi *pi)

static int	do_unlock_pp (struct thread td, struct umutex m, uint32_t flags)

static void	umtx_thread_cleanup (struct thread *td)

static void	umtx_exec_hook (void arg __unused, struct proc p __unused, struct image_params *imgp __unused)

	SYSINIT (umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL)

static void	umtxq_sysinit (void *arg __unused)

struct umtx_q *	umtxq_alloc (void)

void	umtxq_free (struct umtx_q *uq)

static struct umtxq_queue *	umtxq_queue_lookup (struct umtx_key *key, int q)

static int	umtxq_count_pi (struct umtx_key key, struct umtx_q *first)

static int	umtxq_check_susp (struct thread *td)

static int	umtxq_signal_queue (struct umtx_key *key, int n_wake, int q)

static void	umtxq_signal_thread (struct umtx_q *uq)

int	umtx_key_get (void addr, int type, int share, struct umtx_key key)

void	umtx_key_release (struct umtx_key *key)

static int	_do_lock_umtx (struct thread td, struct umtx umtx, u_long id, int timo)

static int	do_lock_umtx (struct thread td, struct umtx umtx, u_long id, struct timespec *timeout)

static int	do_unlock_umtx (struct thread td, struct umtx umtx, u_long id)

static int	do_wait (struct thread td, void addr, u_long id, struct timespec *timeout, int compat32, int is_private)

int	kern_umtx_wake (struct thread td, void uaddr, int n_wake, int is_private)

static int	_do_lock_normal (struct thread td, struct umutex m, uint32_t flags, int timo, int mode)

static int	do_unlock_normal (struct thread td, struct umutex m, uint32_t flags)

static int	do_wake_umutex (struct thread td, struct umutex m)

static int	do_wake2_umutex (struct thread td, struct umutex m, uint32_t flags)

static int	umtx_pi_adjust_thread (struct umtx_pi pi, struct thread td)

static void	umtx_propagate_priority (struct thread *td)

static void	umtx_repropagate_priority (struct umtx_pi *pi)

static void	umtx_pi_setowner (struct umtx_pi pi, struct thread owner)

static int	umtx_pi_claim (struct umtx_pi pi, struct thread owner)

void	umtx_pi_adjust (struct thread *td, u_char oldpri)

static int	umtxq_sleep_pi (struct umtx_q uq, struct umtx_pi pi, uint32_t owner, const char *wmesg, int timo)

static void	umtx_pi_ref (struct umtx_pi *pi)

static void	umtx_pi_unref (struct umtx_pi *pi)

static struct umtx_pi *	umtx_pi_lookup (struct umtx_key *key)

static void	umtx_pi_insert (struct umtx_pi *pi)

static int	_do_lock_pi (struct thread td, struct umutex m, uint32_t flags, int timo, int try)

static int	do_unlock_pi (struct thread td, struct umutex m, uint32_t flags)

static int	_do_lock_pp (struct thread td, struct umutex m, uint32_t flags, int timo, int try)

static int	do_set_ceiling (struct thread td, struct umutex m, uint32_t ceiling, uint32_t *old_ceiling)

static int	_do_lock_umutex (struct thread td, struct umutex m, int flags, int timo, int mode)

static int	do_lock_umutex (struct thread td, struct umutex m, struct timespec *timeout, int mode)

static int	do_unlock_umutex (struct thread td, struct umutex m)

static int	do_cv_wait (struct thread td, struct ucond cv, struct umutex m, struct timespec timeout, u_long wflags)

static int	do_cv_signal (struct thread td, struct ucond cv)

static int	do_cv_broadcast (struct thread td, struct ucond cv)

static int	do_rw_rdlock (struct thread td, struct urwlock rwlock, long fflag, int timo)

static int	do_rw_rdlock2 (struct thread td, void obj, long val, struct timespec *timeout)

static int	do_rw_wrlock (struct thread td, struct urwlock rwlock, int timo)

static int	do_rw_wrlock2 (struct thread td, void obj, struct timespec *timeout)

static int	do_rw_unlock (struct thread td, struct urwlock rwlock)

static int	do_sem_wait (struct thread td, struct _usem sem, struct timespec *timeout)

static int	do_sem_wake (struct thread td, struct _usem sem)

int	sys__umtx_lock (struct thread td, struct _umtx_lock_args uap)

int	sys__umtx_unlock (struct thread td, struct _umtx_unlock_args uap)

int	umtx_copyin_timeout (const void addr, struct timespec tsp)

static int	__umtx_op_lock_umtx (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_unlock_umtx (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_wait (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_wait_uint (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_wait_uint_private (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_wake (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_nwake_private (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_wake_private (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_lock_umutex (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_trylock_umutex (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_wait_umutex (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_wake_umutex (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_unlock_umutex (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_set_ceiling (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_cv_wait (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_cv_signal (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_cv_broadcast (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_rw_rdlock (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_rw_wrlock (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_rw_unlock (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_sem_wait (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_sem_wake (struct thread td, struct _umtx_op_args uap)

static int	__umtx_op_wake2_umutex (struct thread td, struct _umtx_op_args uap)

int	sys__umtx_op (struct thread td, struct _umtx_op_args uap)

void	umtx_thread_init (struct thread *td)

void	umtx_thread_fini (struct thread *td)

void	umtx_thread_alloc (struct thread *td)

void	umtx_thread_exit (struct thread *td)

Variables
static uma_zone_t	umtx_pi_zone

static struct umtxq_chain	umtxq_chains [2][UMTX_CHAINS]

static int	umtx_pi_allocated

static struct mtx	umtx_lock

static _umtx_op_func	op_table []

Macro Definition Documentation

#define _UMUTEX_TRY 1

Definition at line 64 of file kern_umtx.c.

Referenced by __umtx_op_trylock_umutex(), and _do_lock_normal().

#define _UMUTEX_WAIT 2

Definition at line 65 of file kern_umtx.c.

Referenced by __umtx_op_wait_umutex(), _do_lock_normal(), and do_lock_umutex().

#define BATCH_SIZE 128

Definition at line 3266 of file kern_umtx.c.

Referenced by __umtx_op_nwake_private().

#define BUSY_SPINS 200

Definition at line 188 of file kern_umtx.c.

Referenced by umtxq_busy().

#define GET_SHARE ( flags ) (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE)

Definition at line 185 of file kern_umtx.c.

Referenced by _do_lock_normal(), _do_lock_pi(), _do_lock_pp(), do_cv_broadcast(), do_cv_signal(), do_cv_wait(), do_rw_rdlock(), do_rw_unlock(), do_rw_wrlock(), do_sem_wait(), do_sem_wake(), do_set_ceiling(), do_unlock_normal(), do_unlock_pi(), do_unlock_pp(), do_wake2_umutex(), and do_wake_umutex().

#define GOLDEN_RATIO_PRIME 2654404609U

Definition at line 181 of file kern_umtx.c.

Referenced by umtxq_hash().

#define UMTX_CHAINS 512

Definition at line 182 of file kern_umtx.c.

Referenced by umtxq_sysinit().

#define UMTX_EXCLUSIVE_QUEUE 1

Definition at line 146 of file kern_umtx.c.

Referenced by do_rw_unlock(), and do_rw_wrlock().

#define UMTX_SHARED_QUEUE 0

Definition at line 145 of file kern_umtx.c.

Referenced by do_rw_unlock(), do_rw_wrlock(), umtxq_count(), and umtxq_count_pi().

#define UMTX_SHIFTS (__WORD_BIT - 9)

Definition at line 183 of file kern_umtx.c.

Referenced by umtxq_hash().

#define UMTXQ_BUSY_ASSERT ( uc ) KASSERT(&(uc)->uc_busy, ("umtx chain is not busy"))

Definition at line 166 of file kern_umtx.c.

Referenced by umtxq_sleep_pi().

#define umtxq_insert ( uq ) umtxq_insert_queue((uq), UMTX_SHARED_QUEUE)

Definition at line 225 of file kern_umtx.c.

Referenced by _do_lock_normal(), _do_lock_pp(), _do_lock_umtx(), do_cv_wait(), do_rw_rdlock(), do_sem_wait(), do_set_ceiling(), do_wait(), and umtxq_sleep_pi().

#define UMTXQ_LOCKED_ASSERT ( uc ) mtx_assert(&(uc)->uc_lock, MA_OWNED)

Definition at line 165 of file kern_umtx.c.

Referenced by umtx_pi_insert(), umtx_pi_lookup(), umtx_pi_ref(), umtx_pi_unref(), umtxq_count(), umtxq_count_pi(), umtxq_insert_queue(), umtxq_queue_lookup(), umtxq_remove_queue(), umtxq_signal_queue(), umtxq_signal_thread(), umtxq_sleep(), and umtxq_sleep_pi().

#define umtxq_remove ( uq ) umtxq_remove_queue((uq), UMTX_SHARED_QUEUE)

Definition at line 226 of file kern_umtx.c.

Referenced by _do_lock_normal(), _do_lock_pp(), _do_lock_umtx(), do_cv_wait(), do_rw_rdlock(), do_sem_wait(), do_set_ceiling(), do_wait(), umtxq_signal_thread(), and umtxq_sleep_pi().

#define umtxq_signal	(	key,
		nwake
	)	umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE)

Definition at line 224 of file kern_umtx.c.

Referenced by do_cv_broadcast(), do_cv_signal(), do_sem_wake(), do_set_ceiling(), do_unlock_normal(), do_unlock_pp(), do_unlock_umtx(), do_wake2_umutex(), do_wake_umutex(), and kern_umtx_wake().

#define UPRI ( td )

Value:

(((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\
                          (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\
                         PRI_MAX_TIMESHARE : (td)->td_user_pri)

Definition at line 177 of file kern_umtx.c.

Referenced by _do_lock_pp(), do_unlock_pi(), do_unlock_pp(), umtx_pi_adjust_thread(), umtx_pi_claim(), umtx_propagate_priority(), umtx_repropagate_priority(), and umtxq_sleep_pi().

#define UQF_UMTXQ 0x0001

Referenced by do_cv_wait(), do_sem_wait(), do_unlock_pi(), do_wait(), umtxq_insert_queue(), umtxq_remove_queue(), umtxq_sleep(), and umtxq_sleep_pi().

Typedef Documentation

typedef int(* _umtx_op_func) (struct thread *td, struct _umtx_op_args *uap)

Definition at line 3461 of file kern_umtx.c.

Function Documentation

__FBSDID ( "$BSDSUniX$" )

static int __umtx_op_cv_broadcast	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3383 of file kern_umtx.c.

References do_cv_broadcast().

Here is the call graph for this function:

static int __umtx_op_cv_signal	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3377 of file kern_umtx.c.

References do_cv_signal().

Here is the call graph for this function:

static int __umtx_op_cv_wait	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3359 of file kern_umtx.c.

References do_cv_wait(), timeout(), ts, and umtx_copyin_timeout().

Here is the call graph for this function:

static int __umtx_op_lock_umtx	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3186 of file kern_umtx.c.

References do_lock_umtx(), timeout(), ts, and umtx_copyin_timeout().

Here is the call graph for this function:

static int __umtx_op_lock_umutex	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3299 of file kern_umtx.c.

References do_lock_umutex(), timeout(), ts, and umtx_copyin_timeout().

Here is the call graph for this function:

static int __umtx_op_nwake_private	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3268 of file kern_umtx.c.

References BATCH_SIZE, count, and kern_umtx_wake().

Here is the call graph for this function:

static int __umtx_op_rw_rdlock	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3389 of file kern_umtx.c.

References do_rw_rdlock(), do_rw_rdlock2(), and umtx_copyin_timeout().

Here is the call graph for this function:

static int __umtx_op_rw_unlock	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3426 of file kern_umtx.c.

References do_rw_unlock().

Here is the call graph for this function:

static int __umtx_op_rw_wrlock	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3407 of file kern_umtx.c.

References do_rw_wrlock(), do_rw_wrlock2(), and umtx_copyin_timeout().

Here is the call graph for this function:

static int __umtx_op_sem_wait	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3432 of file kern_umtx.c.

References do_sem_wait(), timeout(), ts, and umtx_copyin_timeout().

Here is the call graph for this function:

static int __umtx_op_sem_wake	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3450 of file kern_umtx.c.

References do_sem_wake().

Here is the call graph for this function:

static int __umtx_op_set_ceiling	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3353 of file kern_umtx.c.

References do_set_ceiling().

Here is the call graph for this function:

static int __umtx_op_trylock_umutex	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3317 of file kern_umtx.c.

References _UMUTEX_TRY, and do_lock_umutex().

Here is the call graph for this function:

static int __umtx_op_unlock_umtx	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3204 of file kern_umtx.c.

References do_unlock_umtx().

Here is the call graph for this function:

static int __umtx_op_unlock_umutex	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3347 of file kern_umtx.c.

References do_unlock_umutex().

Here is the call graph for this function:

static int __umtx_op_wait	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3210 of file kern_umtx.c.

References do_wait(), timeout(), ts, and umtx_copyin_timeout().

Here is the call graph for this function:

static int __umtx_op_wait_uint	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3227 of file kern_umtx.c.

References do_wait(), timeout(), ts, and umtx_copyin_timeout().

Here is the call graph for this function:

static int __umtx_op_wait_uint_private	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3244 of file kern_umtx.c.

References do_wait(), timeout(), ts, and umtx_copyin_timeout().

Here is the call graph for this function:

static int __umtx_op_wait_umutex	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3323 of file kern_umtx.c.

References _UMUTEX_WAIT, do_lock_umutex(), timeout(), ts, and umtx_copyin_timeout().

Here is the call graph for this function:

static int __umtx_op_wake	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3261 of file kern_umtx.c.

References kern_umtx_wake().

Here is the call graph for this function:

static int __umtx_op_wake2_umutex	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3456 of file kern_umtx.c.

References do_wake2_umutex().

Here is the call graph for this function:

static int __umtx_op_wake_private	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3293 of file kern_umtx.c.

References kern_umtx_wake().

Here is the call graph for this function:

static int __umtx_op_wake_umutex	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

static

Definition at line 3341 of file kern_umtx.c.

References do_wake_umutex().

Here is the call graph for this function:

static int _do_lock_normal	(	struct thread *	td,
		struct umutex *	m,
		uint32_t	flags,
		int	timo,
		int	mode
	)

static

Definition at line 1155 of file kern_umtx.c.

References _UMUTEX_TRY, _UMUTEX_WAIT, GET_SHARE, umtx_key_get(), umtx_key_release(), umtxq_busy(), umtxq_check_susp(), umtxq_insert, umtxq_lock(), umtxq_remove, umtxq_sleep(), umtxq_unbusy(), and umtxq_unlock().

Referenced by _do_lock_umutex().

Here is the call graph for this function:

Here is the caller graph for this function:

static int _do_lock_pi	(	struct thread *	td,
		struct umutex *	m,
		uint32_t	flags,
		int	timo,
		int	try
	)

static

Definition at line 1829 of file kern_umtx.c.

References GET_SHARE, umtx_key_get(), umtx_key_release(), umtx_pi_alloc(), umtx_pi_claim(), umtx_pi_free(), umtx_pi_insert(), umtx_pi_lookup(), umtx_pi_ref(), umtx_pi_unref(), umtxq_busy(), umtxq_check_susp(), umtxq_lock(), umtxq_sleep_pi(), umtxq_unbusy(), and umtxq_unlock().

Referenced by _do_lock_umutex().

Here is the call graph for this function:

Here is the caller graph for this function:

static int _do_lock_pp	(	struct thread *	td,
		struct umutex *	m,
		uint32_t	flags,
		int	timo,
		int	try
	)

static

Definition at line 2086 of file kern_umtx.c.

References GET_SHARE, priv_check(), sched_lend_user_prio(), umtx_key_get(), umtx_key_release(), umtx_lock, umtxq_busy(), umtxq_insert, umtxq_lock(), umtxq_remove, umtxq_sleep(), umtxq_unbusy(), umtxq_unlock(), and UPRI.

Referenced by _do_lock_umutex().

Here is the call graph for this function:

Here is the caller graph for this function:

static int _do_lock_umtx	(	struct thread *	td,
		struct umtx *	umtx,
		u_long	id,
		int	timo
	)

static

Definition at line 656 of file kern_umtx.c.

References umtx_key_get(), umtx_key_release(), umtxq_busy(), umtxq_check_susp(), umtxq_insert, umtxq_lock(), umtxq_remove, umtxq_sleep(), umtxq_unbusy(), and umtxq_unlock().

Referenced by do_lock_umtx(), and sys__umtx_lock().

Here is the call graph for this function:

Here is the caller graph for this function:

static int _do_lock_umutex	(	struct thread *	td,
		struct umutex *	m,
		int	flags,
		int	timo,
		int	mode
	)

static

Definition at line 2379 of file kern_umtx.c.

References _do_lock_normal(), _do_lock_pi(), and _do_lock_pp().

Referenced by do_lock_umutex().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_cv_broadcast	(	struct thread *	td,
		struct ucond *	cv
	)

static

Definition at line 2600 of file kern_umtx.c.

References GET_SHARE, umtx_key_get(), umtx_key_release(), umtxq_busy(), umtxq_lock(), umtxq_signal, umtxq_unbusy(), and umtxq_unlock().

Referenced by __umtx_op_cv_broadcast().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_cv_signal	(	struct thread *	td,
		struct ucond *	cv
	)

static

Definition at line 2574 of file kern_umtx.c.

References GET_SHARE, umtx_key_get(), umtx_key_release(), umtxq_busy(), umtxq_count(), umtxq_lock(), umtxq_signal, umtxq_unbusy(), and umtxq_unlock().

Referenced by __umtx_op_cv_signal().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_cv_wait	(	struct thread *	td,
		struct ucond *	cv,
		struct umutex *	m,
		struct timespec *	timeout,
		u_long	wflags
	)

static

Definition at line 2463 of file kern_umtx.c.

References do_unlock_umutex(), GET_SHARE, kern_clock_gettime(), timeout(), tvtohz(), umtx_key_get(), umtx_key_release(), umtxq_busy(), umtxq_insert, umtxq_lock(), umtxq_remove, umtxq_sleep(), umtxq_unbusy(), umtxq_unlock(), and UQF_UMTXQ.

Referenced by __umtx_op_cv_wait().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_lock_umtx	(	struct thread *	td,
		struct umtx *	umtx,
		u_long	id,
		struct timespec *	timeout
	)

static

Definition at line 760 of file kern_umtx.c.

References _do_lock_umtx(), getnanouptime(), ts, and tvtohz().

Referenced by __umtx_op_lock_umtx().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_lock_umutex	(	struct thread *	td,
		struct umutex *	m,
		struct timespec *	timeout,
		int	mode
	)

static

Definition at line 2397 of file kern_umtx.c.

References _do_lock_umutex(), _UMUTEX_WAIT, getnanouptime(), ts, and tvtohz().

Referenced by __umtx_op_lock_umutex(), __umtx_op_trylock_umutex(), and __umtx_op_wait_umutex().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_rw_rdlock	(	struct thread *	td,
		struct urwlock *	rwlock,
		long	fflag,
		int	timo
	)

static

Definition at line 2626 of file kern_umtx.c.

References GET_SHARE, umtx_key_get(), umtx_key_release(), umtxq_busy(), umtxq_check_susp(), umtxq_insert, umtxq_lock(), umtxq_remove, umtxq_sleep(), umtxq_unbusy(), and umtxq_unlock().

Referenced by __umtx_op_rw_rdlock(), and do_rw_rdlock2().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_rw_rdlock2	(	struct thread *	td,
		void *	obj,
		long	val,
		struct timespec *	timeout
	)

static

Definition at line 2765 of file kern_umtx.c.

References do_rw_rdlock(), getnanouptime(), ts, and tvtohz().

Referenced by __umtx_op_rw_rdlock().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_rw_unlock	(	struct thread *	td,
		struct urwlock *	rwlock
	)

static

Definition at line 2965 of file kern_umtx.c.

References count, GET_SHARE, UMTX_EXCLUSIVE_QUEUE, umtx_key_get(), umtx_key_release(), UMTX_SHARED_QUEUE, umtxq_busy(), umtxq_check_susp(), umtxq_lock(), umtxq_signal_queue(), umtxq_unbusy(), and umtxq_unlock().

Referenced by __umtx_op_rw_unlock().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_rw_wrlock	(	struct thread *	td,
		struct urwlock *	rwlock,
		int	timo
	)

static

Definition at line 2793 of file kern_umtx.c.

References GET_SHARE, UMTX_EXCLUSIVE_QUEUE, umtx_key_get(), umtx_key_release(), UMTX_SHARED_QUEUE, umtxq_busy(), umtxq_check_susp(), umtxq_insert_queue(), umtxq_lock(), umtxq_remove_queue(), umtxq_signal_queue(), umtxq_sleep(), umtxq_unbusy(), and umtxq_unlock().

Referenced by __umtx_op_rw_wrlock(), and do_rw_wrlock2().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_rw_wrlock2	(	struct thread *	td,
		void *	obj,
		struct timespec *	timeout
	)

static

Definition at line 2937 of file kern_umtx.c.

References do_rw_wrlock(), getnanouptime(), ts, and tvtohz().

Referenced by __umtx_op_rw_wrlock().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_sem_wait	(	struct thread *	td,
		struct _usem *	sem,
		struct timespec *	timeout
	)

static

Definition at line 3057 of file kern_umtx.c.

References count, GET_SHARE, getnanouptime(), tvtohz(), umtx_key_get(), umtx_key_release(), umtxq_busy(), umtxq_insert, umtxq_lock(), umtxq_remove, umtxq_sleep(), umtxq_unbusy(), umtxq_unlock(), and UQF_UMTXQ.

Referenced by __umtx_op_sem_wait().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_sem_wake	(	struct thread *	td,
		struct _usem *	sem
	)

static

Definition at line 3131 of file kern_umtx.c.

References GET_SHARE, umtx_key_get(), umtx_key_release(), umtxq_busy(), umtxq_count(), umtxq_lock(), umtxq_signal, umtxq_unbusy(), and umtxq_unlock().

Referenced by __umtx_op_sem_wake().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_set_ceiling	(	struct thread *	td,
		struct umutex *	m,
		uint32_t	ceiling,
		uint32_t *	old_ceiling
	)

static

Definition at line 2299 of file kern_umtx.c.

References GET_SHARE, umtx_key_get(), umtx_key_release(), umtxq_busy(), umtxq_insert, umtxq_lock(), umtxq_remove, umtxq_signal, umtxq_sleep(), umtxq_unbusy(), and umtxq_unlock().

Referenced by __umtx_op_set_ceiling().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_unlock_normal	(	struct thread *	td,
		struct umutex *	m,
		uint32_t	flags
	)

static

Definition at line 1277 of file kern_umtx.c.

References count, GET_SHARE, umtx_key_get(), umtx_key_release(), umtxq_busy(), umtxq_count(), umtxq_lock(), umtxq_signal, umtxq_unbusy(), and umtxq_unlock().

Referenced by do_unlock_umutex().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_unlock_pi	(	struct thread *	td,
		struct umutex *	m,
		uint32_t	flags
	)

static

Definition at line 1986 of file kern_umtx.c.

References count, GET_SHARE, umtx_pi::pi_owner, sched_lend_user_prio(), umtx_key_get(), umtx_key_release(), umtx_lock, umtxq_busy(), umtxq_count_pi(), umtxq_lock(), umtxq_signal_thread(), umtxq_unbusy(), umtxq_unlock(), UPRI, and UQF_UMTXQ.

Referenced by do_unlock_umutex().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_unlock_pp	(	struct thread *	td,
		struct umutex *	m,
		uint32_t	flags
	)

static

Definition at line 2216 of file kern_umtx.c.

References GET_SHARE, priv_check(), sched_lend_user_prio(), umtx_key_get(), umtx_key_release(), umtx_lock, umtxq_busy(), umtxq_lock(), umtxq_signal, umtxq_unbusy(), umtxq_unlock(), and UPRI.

Referenced by do_unlock_umutex().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_unlock_umtx	(	struct thread *	td,
		struct umtx *	umtx,
		u_long	id
	)

static

Definition at line 800 of file kern_umtx.c.

References count, umtx_key_get(), umtx_key_release(), umtxq_busy(), umtxq_count(), umtxq_lock(), umtxq_signal, umtxq_unbusy(), and umtxq_unlock().

Referenced by __umtx_op_unlock_umtx(), and sys__umtx_unlock().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_unlock_umutex	(	struct thread *	td,
		struct umutex *	m
	)

static

Definition at line 2442 of file kern_umtx.c.

References do_unlock_normal(), do_unlock_pi(), and do_unlock_pp().

Referenced by __umtx_op_unlock_umutex(), and do_cv_wait().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_wait	(	struct thread *	td,
		void *	addr,
		u_long	id,
		struct timespec *	timeout,
		int	compat32,
		int	is_private
	)

static

Definition at line 1068 of file kern_umtx.c.

References getnanouptime(), ts, tvtohz(), umtx_key_get(), umtx_key_release(), umtxq_insert, umtxq_lock(), umtxq_remove, umtxq_sleep(), umtxq_unlock(), and UQF_UMTXQ.

Referenced by __umtx_op_wait(), __umtx_op_wait_uint(), and __umtx_op_wait_uint_private().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_wake2_umutex	(	struct thread *	td,
		struct umutex *	m,
		uint32_t	flags
	)

static

Definition at line 1381 of file kern_umtx.c.

References count, GET_SHARE, type, umtx_key_get(), umtx_key_release(), umtxq_busy(), umtxq_check_susp(), umtxq_count(), umtxq_lock(), umtxq_signal, umtxq_unbusy(), and umtxq_unlock().

Referenced by __umtx_op_wake2_umutex().

Here is the call graph for this function:

Here is the caller graph for this function:

static int do_wake_umutex	(	struct thread *	td,
		struct umutex *	m
	)

static

Definition at line 1338 of file kern_umtx.c.

References count, GET_SHARE, umtx_key_get(), umtx_key_release(), umtxq_busy(), umtxq_count(), umtxq_lock(), umtxq_signal, umtxq_unbusy(), and umtxq_unlock().

Referenced by __umtx_op_wake_umutex().

Here is the call graph for this function:

Here is the caller graph for this function:

int kern_umtx_wake	(	struct thread *	td,
		void *	uaddr,
		int	n_wake,
		int	is_private
	)

Definition at line 1136 of file kern_umtx.c.

References umtx_key_get(), umtx_key_release(), umtxq_lock(), umtxq_signal, and umtxq_unlock().

Referenced by __umtx_op_nwake_private(), __umtx_op_wake(), __umtx_op_wake_private(), and sys_thr_exit().

Here is the call graph for this function:

Here is the caller graph for this function:

LIST_HEAD	(	umtxq_list	,
		umtxq_queue
	)

static MALLOC_DEFINE	(	M_UMTX	,
		"umtx"	,
		"UMTX queue memory"
	)

static

int sys__umtx_lock	(	struct thread *	td,
		struct _umtx_lock_args *	uap
	)

Definition at line 3157 of file kern_umtx.c.

References _do_lock_umtx().

Here is the call graph for this function:

int sys__umtx_op	(	struct thread *	td,
		struct _umtx_op_args *	uap
	)

Definition at line 3490 of file kern_umtx.c.

int sys__umtx_unlock	(	struct thread *	td,
		struct _umtx_unlock_args *	uap
	)

Definition at line 3164 of file kern_umtx.c.

References do_unlock_umtx().

Here is the call graph for this function:

SYSCTL_INT	(	_debug_umtx	,
		OID_AUTO	,
		umtx_pi_allocated	,
		CTLFLAG_RD	,
		&	umtx_pi_allocated,
		0	,
		"Allocated umtx_pi"
	)

static SYSCTL_NODE	(	_debug	,
		OID_AUTO	,
		umtx	,
		CTLFLAG_RW	,
		0	,
		"umtx debug"
	)

static

SYSINIT	(	umtx	,
		SI_SUB_EVENTHANDLER+	1,
		SI_ORDER_MIDDLE	,
		umtxq_sysinit	,
		NULL
	)

TAILQ_HEAD	(	umtxq_head	,
		umtx_q
	)

int umtx_copyin_timeout	(	const void *	addr,
		struct timespec *	tsp
	)

inline

Definition at line 3171 of file kern_umtx.c.

Referenced by __umtx_op_cv_wait(), __umtx_op_lock_umtx(), __umtx_op_lock_umutex(), __umtx_op_rw_rdlock(), __umtx_op_rw_wrlock(), __umtx_op_sem_wait(), __umtx_op_wait(), __umtx_op_wait_uint(), __umtx_op_wait_uint_private(), __umtx_op_wait_umutex(), and sys_thr_suspend().

Here is the caller graph for this function:

static void umtx_exec_hook	(	void *arg	__unused,
		struct proc *p	__unused,
		struct image_params *imgp	__unused
	)

static

Definition at line 3800 of file kern_umtx.c.

References umtx_thread_cleanup().

Referenced by umtxq_sysinit().

Here is the call graph for this function:

Here is the caller graph for this function:

int umtx_key_get	(	void *	addr,
		int	type,
		int	share,
		struct umtx_key *	key
	)

Definition at line 600 of file kern_umtx.c.

References type, and umtxq_hash().

Referenced by _do_lock_normal(), _do_lock_pi(), _do_lock_pp(), _do_lock_umtx(), do_cv_broadcast(), do_cv_signal(), do_cv_wait(), do_rw_rdlock(), do_rw_unlock(), do_rw_wrlock(), do_sem_wait(), do_sem_wake(), do_set_ceiling(), do_unlock_normal(), do_unlock_pi(), do_unlock_pp(), do_unlock_umtx(), do_wait(), do_wake2_umutex(), do_wake_umutex(), and kern_umtx_wake().

Here is the call graph for this function:

Here is the caller graph for this function:

void umtx_key_release ( struct umtx_key * key )

Definition at line 646 of file kern_umtx.c.

Referenced by _do_lock_normal(), _do_lock_pi(), _do_lock_pp(), _do_lock_umtx(), do_cv_broadcast(), do_cv_signal(), do_cv_wait(), do_rw_rdlock(), do_rw_unlock(), do_rw_wrlock(), do_sem_wait(), do_sem_wake(), do_set_ceiling(), do_unlock_normal(), do_unlock_pi(), do_unlock_pp(), do_unlock_umtx(), do_wait(), do_wake2_umutex(), do_wake_umutex(), and kern_umtx_wake().

Here is the caller graph for this function:

void umtx_pi_adjust	(	struct thread *	td,
		u_char	oldpri
	)

Definition at line 1662 of file kern_umtx.c.

References umtx_lock, umtx_pi_adjust_thread(), and umtx_repropagate_priority().

Referenced by rtp_to_pri().

Here is the call graph for this function:

Here is the caller graph for this function:

static int umtx_pi_adjust_thread	(	struct umtx_pi *	pi,
		struct thread *	td
	)

static

Definition at line 1482 of file kern_umtx.c.

References umtx_lock, and UPRI.

Referenced by umtx_pi_adjust(), umtx_propagate_priority(), and umtx_repropagate_priority().

Here is the caller graph for this function:

static struct umtx_pi * umtx_pi_alloc ( int flags )

static

Definition at line 1460 of file kern_umtx.c.

Referenced by _do_lock_pi().

Here is the caller graph for this function:

static int umtx_pi_claim	(	struct umtx_pi *	pi,
		struct thread *	owner
	)

static

Definition at line 1624 of file kern_umtx.c.

References umtx_pi::pi_owner, sched_lend_user_prio(), umtx_lock, umtx_pi_setowner(), and UPRI.

Referenced by _do_lock_pi().

Here is the call graph for this function:

Here is the caller graph for this function:

static void umtx_pi_free ( struct umtx_pi * pi )

inlinestatic

Definition at line 1471 of file kern_umtx.c.

Referenced by _do_lock_pi(), and umtx_pi_unref().

Here is the caller graph for this function:

static void umtx_pi_insert ( struct umtx_pi * pi )

inlinestatic

Definition at line 1816 of file kern_umtx.c.

References umtxq_getchain(), and UMTXQ_LOCKED_ASSERT.

Referenced by _do_lock_pi().

Here is the call graph for this function:

Here is the caller graph for this function:

static struct umtx_pi* umtx_pi_lookup ( struct umtx_key * key )

static

Definition at line 1796 of file kern_umtx.c.

References umtxq_getchain(), and UMTXQ_LOCKED_ASSERT.

Referenced by _do_lock_pi().

Here is the call graph for this function:

Here is the caller graph for this function:

static void umtx_pi_ref ( struct umtx_pi * pi )

static

Definition at line 1756 of file kern_umtx.c.

References umtx_pi::pi_refcount, umtxq_getchain(), and UMTXQ_LOCKED_ASSERT.

Referenced by _do_lock_pi().

Here is the call graph for this function:

Here is the caller graph for this function:

static void umtx_pi_setowner	(	struct umtx_pi *	pi,
		struct thread *	owner
	)

static

Definition at line 1608 of file kern_umtx.c.

References panic(), umtx_pi::pi_owner, and umtx_lock.

Referenced by umtx_pi_claim(), and umtxq_sleep_pi().

Here is the call graph for this function:

Here is the caller graph for this function:

static void umtx_pi_unref ( struct umtx_pi * pi )

static

Definition at line 1770 of file kern_umtx.c.

References umtx_pi::pi_owner, umtx_pi::pi_refcount, umtx_lock, umtx_pi_free(), umtxq_getchain(), and UMTXQ_LOCKED_ASSERT.

Referenced by _do_lock_pi().

Here is the call graph for this function:

Here is the caller graph for this function:

static void umtx_propagate_priority ( struct thread * td )

static

Definition at line 1527 of file kern_umtx.c.

References umtx_pi::pi_owner, sched_lend_user_prio(), umtx_lock, umtx_pi_adjust_thread(), and UPRI.

Referenced by umtxq_sleep_pi().

Here is the call graph for this function:

Here is the caller graph for this function:

static void umtx_repropagate_priority ( struct umtx_pi * pi )

static

Definition at line 1574 of file kern_umtx.c.

References umtx_pi::pi_owner, sched_lend_user_prio(), umtx_lock, umtx_pi_adjust_thread(), and UPRI.

Referenced by umtx_pi_adjust(), and umtxq_sleep_pi().

Here is the call graph for this function:

Here is the caller graph for this function:

void umtx_thread_alloc ( struct thread * td )

Definition at line 3783 of file kern_umtx.c.

Referenced by thread_ctor().

Here is the caller graph for this function:

static void umtx_thread_cleanup ( struct thread * td )

static

Definition at line 3819 of file kern_umtx.c.

References umtx_pi::pi_owner, sched_lend_user_prio(), and umtx_lock.

Referenced by umtx_exec_hook(), and umtx_thread_exit().

Here is the call graph for this function:

Here is the caller graph for this function:

void umtx_thread_exit ( struct thread * td )

Definition at line 3810 of file kern_umtx.c.

References umtx_thread_cleanup().

Referenced by thread_exit().

Here is the call graph for this function:

Here is the caller graph for this function:

void umtx_thread_fini ( struct thread * td )

Definition at line 3774 of file kern_umtx.c.

References umtxq_free().

Referenced by thread_fini().

Here is the call graph for this function:

Here is the caller graph for this function:

void umtx_thread_init ( struct thread * td )

Definition at line 3767 of file kern_umtx.c.

References umtxq_alloc().

Referenced by thread_init().

Here is the call graph for this function:

Here is the caller graph for this function:

struct umtx_q* umtxq_alloc ( void )

Definition at line 283 of file kern_umtx.c.

References malloc().

Referenced by umtx_thread_init().

Here is the call graph for this function:

Here is the caller graph for this function:

static void umtxq_busy ( struct umtx_key * key )

inlinestatic

Definition at line 347 of file kern_umtx.c.

References BUSY_SPINS, count, smp_cpus, umtxq_chain::uc_lock, umtxq_getchain(), umtxq_lock(), and umtxq_unlock().

Referenced by _do_lock_normal(), _do_lock_pi(), _do_lock_pp(), _do_lock_umtx(), do_cv_broadcast(), do_cv_signal(), do_cv_wait(), do_rw_rdlock(), do_rw_unlock(), do_rw_wrlock(), do_sem_wait(), do_sem_wake(), do_set_ceiling(), do_unlock_normal(), do_unlock_pi(), do_unlock_pp(), do_unlock_umtx(), do_wake2_umutex(), and do_wake_umutex().

Here is the call graph for this function:

Here is the caller graph for this function:

static int umtxq_check_susp ( struct thread * td )

static

Definition at line 509 of file kern_umtx.c.

Referenced by _do_lock_normal(), _do_lock_pi(), _do_lock_umtx(), do_rw_rdlock(), do_rw_unlock(), do_rw_wrlock(), and do_wake2_umutex().

Here is the caller graph for this function:

static int umtxq_count ( struct umtx_key * key )

static

Definition at line 474 of file kern_umtx.c.

References UMTX_SHARED_QUEUE, umtxq_getchain(), UMTXQ_LOCKED_ASSERT, and umtxq_queue_lookup().

Referenced by do_cv_signal(), do_sem_wake(), do_unlock_normal(), do_unlock_umtx(), do_wake2_umutex(), and do_wake_umutex().

Here is the call graph for this function:

Here is the caller graph for this function:

static int umtxq_count_pi	(	struct umtx_key *	key,
		struct umtx_q **	first
	)

static

Definition at line 492 of file kern_umtx.c.

References umtxq_queue::head, UMTX_SHARED_QUEUE, umtxq_getchain(), UMTXQ_LOCKED_ASSERT, and umtxq_queue_lookup().

Referenced by do_unlock_pi().

Here is the call graph for this function:

Here is the caller graph for this function:

void umtxq_free ( struct umtx_q * uq )

Definition at line 296 of file kern_umtx.c.

References free().

Referenced by umtx_thread_fini().

Here is the call graph for this function:

Here is the caller graph for this function:

static struct umtxq_chain * umtxq_getchain ( struct umtx_key * key )

static

Definition at line 311 of file kern_umtx.c.

References umtxq_chains.

Referenced by umtx_pi_insert(), umtx_pi_lookup(), umtx_pi_ref(), umtx_pi_unref(), umtxq_busy(), umtxq_count(), umtxq_count_pi(), umtxq_insert_queue(), umtxq_lock(), umtxq_queue_lookup(), umtxq_remove_queue(), umtxq_signal_queue(), umtxq_signal_thread(), umtxq_sleep(), umtxq_sleep_pi(), umtxq_unbusy(), and umtxq_unlock().

Here is the caller graph for this function:

static void umtxq_hash ( struct umtx_key * key )

inlinestatic

Definition at line 304 of file kern_umtx.c.

References GOLDEN_RATIO_PRIME, and UMTX_SHIFTS.

Referenced by umtx_key_get().

Here is the caller graph for this function:

static void umtxq_insert_queue	(	struct umtx_q *	uq,
		int	q
	)

inlinestatic

Definition at line 407 of file kern_umtx.c.

References umtxq_queue::head, umtxq_queue::key, umtxq_chain::uc_queue, umtxq_getchain(), UMTXQ_LOCKED_ASSERT, umtxq_queue_lookup(), and UQF_UMTXQ.

Referenced by do_rw_wrlock().

Here is the call graph for this function:

Here is the caller graph for this function:

static void umtxq_lock ( struct umtx_key * key )

inlinestatic

Definition at line 322 of file kern_umtx.c.

References umtxq_chain::uc_lock, and umtxq_getchain().

Referenced by _do_lock_normal(), _do_lock_pi(), _do_lock_pp(), _do_lock_umtx(), do_cv_broadcast(), do_cv_signal(), do_cv_wait(), do_rw_rdlock(), do_rw_unlock(), do_rw_wrlock(), do_sem_wait(), do_sem_wake(), do_set_ceiling(), do_unlock_normal(), do_unlock_pi(), do_unlock_pp(), do_unlock_umtx(), do_wait(), do_wake2_umutex(), do_wake_umutex(), kern_umtx_wake(), and umtxq_busy().

Here is the call graph for this function:

Here is the caller graph for this function:

static struct umtxq_queue* umtxq_queue_lookup	(	struct umtx_key *	key,
		int	q
	)

static

Definition at line 391 of file kern_umtx.c.

References umtxq_queue::key, umtxq_chain::uc_queue, umtxq_getchain(), and UMTXQ_LOCKED_ASSERT.

Referenced by umtxq_count(), umtxq_count_pi(), umtxq_insert_queue(), and umtxq_signal_queue().

Here is the call graph for this function:

Here is the caller graph for this function:

static void umtxq_remove_queue	(	struct umtx_q *	uq,
		int	q
	)

inlinestatic

Definition at line 441 of file kern_umtx.c.

References umtxq_queue::head, umtxq_getchain(), UMTXQ_LOCKED_ASSERT, and UQF_UMTXQ.

Referenced by do_rw_wrlock(), and umtxq_signal_queue().

Here is the call graph for this function:

Here is the caller graph for this function:

static int umtxq_signal_queue	(	struct umtx_key *	key,
		int	n_wake,
		int	q
	)

static

Definition at line 539 of file kern_umtx.c.

References umtxq_queue::head, umtxq_getchain(), UMTXQ_LOCKED_ASSERT, umtxq_queue_lookup(), umtxq_remove_queue(), and wakeup().

Referenced by do_rw_unlock(), and do_rw_wrlock().

Here is the call graph for this function:

Here is the caller graph for this function:

static void umtxq_signal_thread ( struct umtx_q * uq )

inlinestatic

Definition at line 566 of file kern_umtx.c.

References umtxq_getchain(), UMTXQ_LOCKED_ASSERT, umtxq_remove, and wakeup().

Referenced by do_unlock_pi().

Here is the call graph for this function:

Here is the caller graph for this function:

static int umtxq_sleep	(	struct umtx_q *	uq,
		const char *	wmesg,
		int	timo
	)

inlinestatic

Definition at line 581 of file kern_umtx.c.

References umtxq_chain::uc_lock, umtxq_getchain(), UMTXQ_LOCKED_ASSERT, and UQF_UMTXQ.

Referenced by _do_lock_normal(), _do_lock_pp(), _do_lock_umtx(), do_cv_wait(), do_rw_rdlock(), do_rw_wrlock(), do_sem_wait(), do_set_ceiling(), and do_wait().

Here is the call graph for this function:

Here is the caller graph for this function:

static int umtxq_sleep_pi	(	struct umtx_q *	uq,
		struct umtx_pi *	pi,
		uint32_t	owner,
		const char *	wmesg,
		int	timo
	)

static

Definition at line 1684 of file kern_umtx.c.

References umtx_pi::pi_owner, tdfind(), umtxq_chain::uc_lock, umtx_lock, umtx_pi_setowner(), umtx_propagate_priority(), umtx_repropagate_priority(), UMTXQ_BUSY_ASSERT, umtxq_getchain(), umtxq_insert, UMTXQ_LOCKED_ASSERT, umtxq_remove, umtxq_unbusy(), umtxq_unlock(), UPRI, and UQF_UMTXQ.

Referenced by _do_lock_pi().

Here is the call graph for this function:

Here is the caller graph for this function:

static void umtxq_sysinit ( void * )

static

static void umtxq_sysinit ( void *arg __unused )

static

Definition at line 252 of file kern_umtx.c.

References mtx_init(), UMTX_CHAINS, umtx_exec_hook(), umtx_lock, and umtxq_chains.

Here is the call graph for this function:

static void umtxq_unbusy ( struct umtx_key * key )

inlinestatic

Definition at line 378 of file kern_umtx.c.

References umtxq_chain::uc_lock, umtxq_getchain(), and wakeup_one().

Referenced by _do_lock_normal(), _do_lock_pi(), _do_lock_pp(), _do_lock_umtx(), do_cv_broadcast(), do_cv_signal(), do_cv_wait(), do_rw_rdlock(), do_rw_unlock(), do_rw_wrlock(), do_sem_wait(), do_sem_wake(), do_set_ceiling(), do_unlock_normal(), do_unlock_pi(), do_unlock_pp(), do_unlock_umtx(), do_wake2_umutex(), do_wake_umutex(), and umtxq_sleep_pi().

Here is the call graph for this function:

Here is the caller graph for this function:

static void umtxq_unlock ( struct umtx_key * key )

inlinestatic

Definition at line 334 of file kern_umtx.c.

References umtxq_chain::uc_lock, and umtxq_getchain().

Referenced by _do_lock_normal(), _do_lock_pi(), _do_lock_pp(), _do_lock_umtx(), do_cv_broadcast(), do_cv_signal(), do_cv_wait(), do_rw_rdlock(), do_rw_unlock(), do_rw_wrlock(), do_sem_wait(), do_sem_wake(), do_set_ceiling(), do_unlock_normal(), do_unlock_pi(), do_unlock_pp(), do_unlock_umtx(), do_wait(), do_wake2_umutex(), do_wake_umutex(), kern_umtx_wake(), umtxq_busy(), and umtxq_sleep_pi().

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

_umtx_op_func op_table[]

static

Initial value:

= {
        __umtx_op_lock_umtx,            
        __umtx_op_unlock_umtx,          
        __umtx_op_wait,                 
        __umtx_op_wake,                 
        __umtx_op_trylock_umutex,       
        __umtx_op_lock_umutex,          
        __umtx_op_unlock_umutex,        
        __umtx_op_set_ceiling,          
        __umtx_op_cv_wait,              
        __umtx_op_cv_signal,            
        __umtx_op_cv_broadcast,         
        __umtx_op_wait_uint,            
        __umtx_op_rw_rdlock,            
        __umtx_op_rw_wrlock,            
        __umtx_op_rw_unlock,            
        __umtx_op_wait_uint_private,    
        __umtx_op_wake_private,         
        __umtx_op_wait_umutex,          
        __umtx_op_wake_umutex,          
        __umtx_op_sem_wait,             
        __umtx_op_sem_wake,             
        __umtx_op_nwake_private,        
        __umtx_op_wake2_umutex          
}

Definition at line 3463 of file kern_umtx.c.

struct mtx umtx_lock

static

Definition at line 228 of file kern_umtx.c.

Referenced by _do_lock_pp(), do_unlock_pi(), do_unlock_pp(), umtx_pi_adjust(), umtx_pi_adjust_thread(), umtx_pi_claim(), umtx_pi_setowner(), umtx_pi_unref(), umtx_propagate_priority(), umtx_repropagate_priority(), umtx_thread_cleanup(), umtxq_sleep_pi(), and umtxq_sysinit().

int umtx_pi_allocated

static

Definition at line 193 of file kern_umtx.c.

uma_zone_t umtx_pi_zone

static

Definition at line 190 of file kern_umtx.c.

struct umtxq_chain umtxq_chains[2][UMTX_CHAINS]

static

Definition at line 191 of file kern_umtx.c.

Referenced by umtxq_getchain(), and umtxq_sysinit().

Data Structures

Macros

Typedefs

Functions

Variables

Macro Definition Documentation

Typedef Documentation

Function Documentation

Variable Documentation