subr_turnstile.c File Reference

#include <sys/cdefs.h>
#include "opt_ddb.h"
#include "opt_kdtrace.h"
#include "opt_turnstile_profiling.h"
#include "opt_sched.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/queue.h>
#include <sys/sched.h>
#include <sys/sdt.h>
#include <sys/sysctl.h>
#include <sys/turnstile.h>
#include <vm/uma.h>

Include dependency graph for subr_turnstile.c:

Go to the source code of this file.

Data Structures
struct	turnstile
struct	turnstile_chain

Macros
#define	TC_TABLESIZE   128 /* Must be power of 2. */
#define	TC_MASK   (TC_TABLESIZE - 1)
#define	TC_SHIFT   8
#define	TC_HASH(lock)   (((uintptr_t)(lock) >> TC_SHIFT) & TC_MASK)
#define	TC_LOOKUP(lock)   &turnstile_chains[TC_HASH(lock)]

Functions
	__FBSDID ("$BSDSUniX$")
static void	init_turnstile0 (void *dummy)
static void	propagate_priority (struct thread *td)
static int	turnstile_adjust_thread (struct turnstile *ts, struct thread *td)
static struct thread *	turnstile_first_waiter (struct turnstile *ts)
static void	turnstile_setowner (struct turnstile *ts, struct thread *owner)
static int	turnstile_init (void *mem, int size, int flags)
static void	turnstile_fini (void *mem, int size)
	SDT_PROVIDER_DECLARE (sched)
	SDT_PROBE_DEFINE (sched,,, sleep)
	SDT_PROBE_DEFINE2 (sched,,, wakeup,"struct thread *","struct proc *")
void	init_turnstiles (void)
	SYSINIT (turnstile0, SI_SUB_LOCK, SI_ORDER_ANY, init_turnstile0, NULL)
void	turnstile_adjust (struct thread *td, u_char oldpri)
struct turnstile *	turnstile_alloc (void)
void	turnstile_free (struct turnstile *ts)
void	turnstile_chain_lock (struct lock_object *lock)
struct turnstile *	turnstile_trywait (struct lock_object *lock)
void	turnstile_cancel (struct turnstile *ts)
struct turnstile *	turnstile_lookup (struct lock_object *lock)
void	turnstile_chain_unlock (struct lock_object *lock)
void	turnstile_claim (struct turnstile *ts)
void	turnstile_wait (struct turnstile *ts, struct thread *owner, int queue)
int	turnstile_signal (struct turnstile *ts, int queue)
void	turnstile_broadcast (struct turnstile *ts, int queue)
void	turnstile_unpend (struct turnstile *ts, int owner_type)
void	turnstile_disown (struct turnstile *ts)
struct thread *	turnstile_head (struct turnstile *ts, int queue)
int	turnstile_empty (struct turnstile *ts, int queue)

Variables
static struct mtx	td_contested_lock
static struct turnstile_chain	turnstile_chains [TC_TABLESIZE]
static uma_zone_t	turnstile_zone

Macro Definition Documentation

#define TC_HASH

(

lock

)

(((uintptr_t)(lock) >> TC_SHIFT) & TC_MASK)

Definition at line 98 of file subr_turnstile.c.

#define TC_LOOKUP

(

lock

)

&turnstile_chains[TC_HASH(lock)]

Definition at line 99 of file subr_turnstile.c.

Referenced by turnstile_broadcast(), turnstile_cancel(), turnstile_chain_lock(), turnstile_chain_unlock(), turnstile_claim(), turnstile_lookup(), turnstile_signal(), turnstile_trywait(), and turnstile_wait().

#define TC_MASK   (TC_TABLESIZE - 1)

Definition at line 96 of file subr_turnstile.c.

#define TC_SHIFT   8

Definition at line 97 of file subr_turnstile.c.

#define TC_TABLESIZE   128 /* Must be power of 2. */

Definition at line 95 of file subr_turnstile.c.

Referenced by init_turnstiles().

Function Documentation

__FBSDID

(

"$BSDSUniX$"

)

static void init_turnstile0 ( void *  dummy )

static

Definition at line 401 of file subr_turnstile.c.

References turnstile_alloc(), turnstile_fini(), and turnstile_init().

Here is the call graph for this function:

void init_turnstiles

(

void

)

Definition at line 361 of file subr_turnstile.c.

References mtx_init(), TC_TABLESIZE, td_contested_lock, and turnstile_chains.

Referenced by mutex_init().

Here is the call graph for this function:

Here is the caller graph for this function:

static void propagate_priority ( struct thread *  td )

static

Definition at line 183 of file subr_turnstile.c.

References kdb_backtrace_thread(), panic(), printf(), sched_lend_prio(), ts, turnstile::ts_lock, and turnstile_adjust_thread().

Referenced by turnstile_adjust(), and turnstile_wait().

Here is the call graph for this function:

Here is the caller graph for this function:

SDT_PROBE_DEFINE	(	sched	,
		sleep
	)

SDT_PROBE_DEFINE2	(	sched	,
		wakeup	,
		"struct thread *"	,
		"struct proc *"
	)

SDT_PROVIDER_DECLARE

(

sched

)

SYSINIT	(	turnstile0	,
		SI_SUB_LOCK	,
		SI_ORDER_ANY	,
		init_turnstile0	,
		NULL
	)

void turnstile_adjust	(	struct thread *	td,
		u_char	oldpri
	)

Definition at line 421 of file subr_turnstile.c.

References propagate_priority(), ts, turnstile::ts_blocked, turnstile::ts_lock, and turnstile_adjust_thread().

Referenced by sched_prio().

Here is the call graph for this function:

Here is the caller graph for this function:

static int turnstile_adjust_thread ( struct turnstile *  ts, struct thread *  td  )

static

Definition at line 291 of file subr_turnstile.c.

References td_contested_lock, turnstile::ts_blocked, and turnstile::ts_lock.

Referenced by propagate_priority(), and turnstile_adjust().

Here is the caller graph for this function:

struct turnstile* turnstile_alloc

(

void

)

Definition at line 518 of file subr_turnstile.c.

Referenced by init_turnstile0(), and thread_init().

Here is the caller graph for this function:

void turnstile_broadcast	(	struct turnstile *	ts,
		int	queue
	)

Definition at line 817 of file subr_turnstile.c.

References TC_LOOKUP, td_contested_lock, ts, turnstile::ts_blocked, turnstile::ts_lock, and turnstile::ts_pending.

Referenced by _mtx_unlock_sleep(), _rw_downgrade(), _rw_runlock(), and _rw_wunlock_hard().

Here is the caller graph for this function:

void turnstile_cancel

(

struct turnstile *

ts

)

Definition at line 570 of file subr_turnstile.c.

References TC_LOOKUP, and turnstile::ts_lock.

Referenced by _mtx_lock_sleep(), _rw_rlock(), _rw_try_upgrade(), and _rw_wlock_hard().

Here is the caller graph for this function:

void turnstile_chain_lock

(

struct lock_object *

lock

)

Definition at line 538 of file subr_turnstile.c.

References TC_LOOKUP.

Referenced by _mtx_unlock_sleep(), _rm_unlock_hard(), _rw_downgrade(), _rw_runlock(), and _rw_wunlock_hard().

Here is the caller graph for this function:

void turnstile_chain_unlock

(

struct lock_object *

lock

)

Definition at line 610 of file subr_turnstile.c.

References TC_LOOKUP.

Referenced by _mtx_unlock_sleep(), _rm_unlock_hard(), _rw_downgrade(), _rw_runlock(), and _rw_wunlock_hard().

Here is the caller graph for this function:

void turnstile_claim

(

struct turnstile *

ts

)

Definition at line 639 of file subr_turnstile.c.

References sched_lend_prio(), TC_LOOKUP, td_contested_lock, turnstile::ts_lock, turnstile_first_waiter(), and turnstile_setowner().

Referenced by _rw_try_upgrade(), and _rw_wlock_hard().

Here is the call graph for this function:

Here is the caller graph for this function:

void turnstile_disown

(

struct turnstile *

ts

)

Definition at line 952 of file subr_turnstile.c.

References sched_unlend_prio(), td_contested_lock, turnstile::ts_blocked, turnstile::ts_lock, turnstile::ts_pending, and turnstile_first_waiter().

Referenced by _rw_downgrade().

Here is the call graph for this function:

Here is the caller graph for this function:

int turnstile_empty	(	struct turnstile *	ts,
		int	queue
	)

Definition at line 1014 of file subr_turnstile.c.

References turnstile::ts_blocked, and turnstile::ts_lock.

static void turnstile_fini ( void *  mem, int  size  )

static

Definition at line 506 of file subr_turnstile.c.

References mtx_destroy(), ts, and turnstile::ts_lock.

Referenced by init_turnstile0().

Here is the call graph for this function:

Here is the caller graph for this function:

static struct thread * turnstile_first_waiter ( struct turnstile *  ts )

static

Definition at line 623 of file subr_turnstile.c.

References turnstile::ts_blocked.

Referenced by turnstile_claim(), turnstile_disown(), and turnstile_unpend().

Here is the caller graph for this function:

void turnstile_free

(

struct turnstile *

ts

)

Definition at line 528 of file subr_turnstile.c.

Referenced by thread_fini().

Here is the caller graph for this function:

struct thread* turnstile_head	(	struct turnstile *	ts,
		int	queue
	)

Definition at line 999 of file subr_turnstile.c.

References turnstile::ts_blocked, and turnstile::ts_lock.

static int turnstile_init ( void *  mem, int  size, int  flags  )

static

Definition at line 491 of file subr_turnstile.c.

References mtx_init(), ts, turnstile::ts_blocked, turnstile::ts_lock, and turnstile::ts_pending.

Referenced by init_turnstile0().

Here is the call graph for this function:

Here is the caller graph for this function:

struct turnstile* turnstile_lookup

(

struct lock_object *

lock

)

Definition at line 591 of file subr_turnstile.c.

References TC_LOOKUP, ts, and turnstile::ts_lock.

Referenced by _mtx_unlock_sleep(), _rm_unlock_hard(), _rw_downgrade(), _rw_runlock(), and _rw_wunlock_hard().

Here is the caller graph for this function:

static void turnstile_setowner ( struct turnstile *  ts, struct thread *  owner  )

static

Definition at line 456 of file subr_turnstile.c.

References td_contested_lock.

Referenced by turnstile_claim(), and turnstile_wait().

Here is the caller graph for this function:

int turnstile_signal	(	struct turnstile *	ts,
		int	queue
	)

Definition at line 766 of file subr_turnstile.c.

References empty, TC_LOOKUP, td_contested_lock, ts, turnstile::ts_blocked, turnstile::ts_lock, and turnstile::ts_pending.

Referenced by _rm_unlock_hard().

Here is the caller graph for this function:

struct turnstile* turnstile_trywait

(

struct lock_object *

lock

)

Definition at line 547 of file subr_turnstile.c.

References TC_LOOKUP, ts, and turnstile::ts_lock.

Referenced by _mtx_lock_sleep(), _rm_wlock(), _rw_rlock(), _rw_try_upgrade(), and _rw_wlock_hard().

Here is the caller graph for this function:

void turnstile_unpend	(	struct turnstile *	ts,
		int	owner_type
	)

Definition at line 867 of file subr_turnstile.c.

References sched_add(), sched_unlend_prio(), TAILQ_HEAD(), td_contested_lock, turnstile::ts_blocked, turnstile::ts_lock, turnstile::ts_pending, turnstile_first_waiter(), and wakeup().

Referenced by _mtx_unlock_sleep(), _rm_unlock_hard(), _rw_downgrade(), _rw_runlock(), and _rw_wunlock_hard().

Here is the call graph for this function:

Here is the caller graph for this function:

void turnstile_wait	(	struct turnstile *	ts,
		struct thread *	owner,
		int	queue
	)

Definition at line 676 of file subr_turnstile.c.

References mi_switch(), propagate_priority(), TC_LOOKUP, td_contested_lock, thread_lock_set(), ticks, ts, turnstile::ts_blocked, turnstile::ts_lock, turnstile::ts_pending, and turnstile_setowner().

Referenced by _mtx_lock_sleep(), _rm_wlock(), _rw_rlock(), and _rw_wlock_hard().

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

struct mtx td_contested_lock

static

Definition at line 150 of file subr_turnstile.c.

Referenced by init_turnstiles(), turnstile_adjust_thread(), turnstile_broadcast(), turnstile_claim(), turnstile_disown(), turnstile_setowner(), turnstile_signal(), turnstile_unpend(), and turnstile_wait().

struct turnstile_chain turnstile_chains[TC_TABLESIZE]

static

Definition at line 151 of file subr_turnstile.c.

Referenced by init_turnstiles().

uma_zone_t turnstile_zone

static

Definition at line 152 of file subr_turnstile.c.