sched_ule.c File Reference

#include <sys/cdefs.h>
#include "opt_hwpmc_hooks.h"
#include "opt_kdtrace.h"
#include "opt_sched.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/resource.h>
#include <sys/resourcevar.h>
#include <sys/sched.h>
#include <sys/sdt.h>
#include <sys/smp.h>
#include <sys/sx.h>
#include <sys/sysctl.h>
#include <sys/sysproto.h>
#include <sys/turnstile.h>
#include <sys/umtx.h>
#include <sys/vmmeter.h>
#include <sys/cpuset.h>
#include <sys/sbuf.h>
#include <machine/cpu.h>
#include <machine/smp.h>

Include dependency graph for sched_ule.c:

Go to the source code of this file.

Data Structures
struct	td_sched
struct	tdq

Macros
#define	KTR_ULE   0
#define	TS_NAME_LEN   (MAXCOMLEN + sizeof(" td ") + sizeof(__XSTRING(UINT_MAX)))
#define	TDQ_NAME_LEN   (sizeof("sched lock ") + sizeof(__XSTRING(MAXCPU)))
#define	TDQ_LOADNAME_LEN   (sizeof("CPU ") + sizeof(__XSTRING(MAXCPU)) - 1 + sizeof(" load"))
#define	TSF_BOUND   0x0001 /* Thread can not migrate. */
#define	TSF_XFERABLE   0x0002 /* Thread was added as transferable. */
#define	THREAD_CAN_MIGRATE(td)   ((td)->td_pinned == 0)
#define	THREAD_CAN_SCHED(td, cpu)   CPU_ISSET((cpu), &(td)->td_cpuset->cs_mask)
#define	PRI_TIMESHARE_RANGE   (PRI_MAX_TIMESHARE - PRI_MIN_TIMESHARE + 1)
#define	PRI_INTERACT_RANGE   ((PRI_TIMESHARE_RANGE - SCHED_PRI_NRESV) / 2)
#define	PRI_BATCH_RANGE   (PRI_TIMESHARE_RANGE - PRI_INTERACT_RANGE)
#define	PRI_MIN_INTERACT   PRI_MIN_TIMESHARE
#define	PRI_MAX_INTERACT   (PRI_MIN_TIMESHARE + PRI_INTERACT_RANGE - 1)
#define	PRI_MIN_BATCH   (PRI_MIN_TIMESHARE + PRI_INTERACT_RANGE)
#define	PRI_MAX_BATCH   PRI_MAX_TIMESHARE
#define	SCHED_TICK_SECS   10
#define	SCHED_TICK_TARG   (hz * SCHED_TICK_SECS)
#define	SCHED_TICK_MAX   (SCHED_TICK_TARG + hz)
#define	SCHED_TICK_SHIFT   10
#define	SCHED_TICK_HZ(ts)   ((ts)->ts_ticks >> SCHED_TICK_SHIFT)
#define	SCHED_TICK_TOTAL(ts)   (max((ts)->ts_ltick - (ts)->ts_ftick, hz))
#define	SCHED_PRI_NRESV   (PRIO_MAX - PRIO_MIN)
#define	SCHED_PRI_NHALF   (SCHED_PRI_NRESV / 2)
#define	SCHED_PRI_MIN   (PRI_MIN_BATCH + SCHED_PRI_NHALF)
#define	SCHED_PRI_MAX   (PRI_MAX_BATCH - SCHED_PRI_NHALF)
#define	SCHED_PRI_RANGE   (SCHED_PRI_MAX - SCHED_PRI_MIN + 1)
#define	SCHED_PRI_TICKS(ts)
#define	SCHED_PRI_NICE(nice)   (nice)
#define	SCHED_SLP_RUN_MAX   ((hz * 5) << SCHED_TICK_SHIFT)
#define	SCHED_SLP_RUN_FORK   ((hz / 2) << SCHED_TICK_SHIFT)
#define	SCHED_INTERACT_MAX   (100)
#define	SCHED_INTERACT_HALF   (SCHED_INTERACT_MAX / 2)
#define	SCHED_INTERACT_THRESH   (30)
#define	TDF_SLICEEND   TDF_SCHED2 /* Thread time slice is over. */
#define	TDQ_RUNNING   1
#define	TDQ_IDLE   2
#define	TDQ_ID(x)   (0)
#define	TDQ_SELF()   (&tdq_cpu)
#define	TDQ_CPU(x)   (&tdq_cpu)
#define	TDQ_LOCK_ASSERT(t, type)   mtx_assert(TDQ_LOCKPTR((t)), (type))
#define	TDQ_LOCK(t)   mtx_lock_spin(TDQ_LOCKPTR((t)))
#define	TDQ_LOCK_FLAGS(t, f)   mtx_lock_spin_flags(TDQ_LOCKPTR((t)), (f))
#define	TDQ_UNLOCK(t)   mtx_unlock_spin(TDQ_LOCKPTR((t)))
#define	TDQ_LOCKPTR(t)   (&(t)->tdq_lock)
#define	TDQ_IDLESPIN(tdq)   1

Functions
	__FBSDID ("$BSDSUniX$")
struct tdq	__aligned (64)
static void	sched_priority (struct thread *)
static void	sched_thread_priority (struct thread *, u_char)
static int	sched_interact_score (struct thread *)
static void	sched_interact_update (struct thread *)
static void	sched_interact_fork (struct thread *)
static void	sched_pctcpu_update (struct td_sched *, int)
static struct thread *	tdq_choose (struct tdq *)
static void	tdq_setup (struct tdq *)
static void	tdq_load_add (struct tdq *, struct thread *)
static void	tdq_load_rem (struct tdq *, struct thread *)
static __inline void	tdq_runq_add (struct tdq *, struct thread *, int)
static __inline void	tdq_runq_rem (struct tdq *, struct thread *)
static int	sched_shouldpreempt (int, int, int)
void	tdq_print (int cpu)
static void	runq_print (struct runq *rq)
static void	tdq_add (struct tdq *, struct thread *, int)
static void	sched_setup (void *dummy)
	SYSINIT (sched_setup, SI_SUB_RUN_QUEUE, SI_ORDER_FIRST, sched_setup, NULL)
static void	sched_initticks (void *dummy)
	SYSINIT (sched_initticks, SI_SUB_CLOCKS, SI_ORDER_THIRD, sched_initticks, NULL)
	SDT_PROVIDER_DEFINE (sched)
	SDT_PROBE_DEFINE3 (sched,,, change__pri,"struct thread *","struct proc *","uint8_t")
	SDT_PROBE_DEFINE3 (sched,,, dequeue,"struct thread *","struct proc *","void *")
	SDT_PROBE_DEFINE4 (sched,,, enqueue,"struct thread *","struct proc *","void *","int")
	SDT_PROBE_DEFINE4 (sched,,, lend__pri,"struct thread *","struct proc *","uint8_t","struct thread *")
	SDT_PROBE_DEFINE2 (sched,,, load__change,"int","int")
	SDT_PROBE_DEFINE2 (sched,,, off__cpu,"struct thread *","struct proc *")
	SDT_PROBE_DEFINE (sched,,, on__cpu)
	SDT_PROBE_DEFINE (sched,,, remain__cpu)
	SDT_PROBE_DEFINE2 (sched,,, surrender,"struct thread *","struct proc *")
static void	tdq_setlowpri (struct tdq *tdq, struct thread *ctd)
void	schedinit (void)
int	sched_rr_interval (void)
void	sched_lend_prio (struct thread *td, u_char prio)
void	sched_unlend_prio (struct thread *td, u_char prio)
void	sched_prio (struct thread *td, u_char prio)
void	sched_user_prio (struct thread *td, u_char prio)
void	sched_lend_user_prio (struct thread *td, u_char prio)
static struct mtx *	sched_switch_migrate (struct tdq *tdq, struct thread *td, int flags)
static void	thread_unblock_switch (struct thread *td, struct mtx *mtx)
void	sched_switch (struct thread *td, struct thread *newtd, int flags)
void	sched_nice (struct proc *p, int nice)
void	sched_sleep (struct thread *td, int prio)
void	sched_wakeup (struct thread *td)
void	sched_fork (struct thread *td, struct thread *child)
void	sched_fork_thread (struct thread *td, struct thread *child)
void	sched_class (struct thread *td, int class)
void	sched_exit (struct proc *p, struct thread *child)
void	sched_exit_thread (struct thread *td, struct thread *child)
void	sched_preempt (struct thread *td)
void	sched_userret (struct thread *td)
void	sched_clock (struct thread *td)
void	sched_tick (int cnt)
int	sched_runnable (void)
struct thread *	sched_choose (void)
static void	sched_setpreempt (struct thread *td)
void	sched_add (struct thread *td, int flags)
void	sched_rem (struct thread *td)
fixpt_t	sched_pctcpu (struct thread *td)
void	sched_affinity (struct thread *td)
void	sched_bind (struct thread *td, int cpu)
void	sched_unbind (struct thread *td)
int	sched_is_bound (struct thread *td)
void	sched_relinquish (struct thread *td)
int	sched_load (void)
int	sched_sizeof_proc (void)
int	sched_sizeof_thread (void)
void	sched_idletd (void *dummy)
void	sched_throw (struct thread *td)
void	sched_fork_exit (struct thread *td)
char *	sched_tdname (struct thread *td)
static int	sysctl_kern_quantum (SYSCTL_HANDLER_ARGS)
	SYSCTL_NODE (_kern, OID_AUTO, sched, CTLFLAG_RW, 0,"Scheduler")
	SYSCTL_STRING (_kern_sched, OID_AUTO, name, CTLFLAG_RD,"ULE", 0,"Scheduler name")
	SYSCTL_PROC (_kern_sched, OID_AUTO, quantum, CTLTYPE_INT|CTLFLAG_RW, NULL, 0, sysctl_kern_quantum,"I","Quantum for timeshare threads in microseconds")
	SYSCTL_INT (_kern_sched, OID_AUTO, slice, CTLFLAG_RW,&sched_slice, 0,"Quantum for timeshare threads in stathz ticks")
	SYSCTL_INT (_kern_sched, OID_AUTO, interact, CTLFLAG_RW,&sched_interact, 0,"Interactivity score threshold")
	SYSCTL_INT (_kern_sched, OID_AUTO, preempt_thresh, CTLFLAG_RW,&preempt_thresh, 0,"Maximal (lowest) priority for preemption")
	SYSCTL_INT (_kern_sched, OID_AUTO, static_boost, CTLFLAG_RW,&static_boost, 0,"Assign static kernel priorities to sleeping threads")
	SYSCTL_INT (_kern_sched, OID_AUTO, idlespins, CTLFLAG_RW,&sched_idlespins, 0,"Number of times idle thread will spin waiting for new work")
	SYSCTL_INT (_kern_sched, OID_AUTO, idlespinthresh, CTLFLAG_RW,&sched_idlespinthresh, 0,"Threshold before we will permit idle thread spinning")
	SYSCTL_INT (_kern, OID_AUTO, ccpu, CTLFLAG_RD,&ccpu, 0,"")

Variables
static struct td_sched	td_sched0
static int	sched_interact = SCHED_INTERACT_THRESH
static int	realstathz = 127
static int	tickincr = 8 << SCHED_TICK_SHIFT
static int	sched_slice = 12
static int	preempt_thresh = 0
static int	static_boost = PRI_MIN_BATCH
static int	sched_idlespins = 10000
static int	sched_idlespinthresh = -1
struct mtx	tdq_lock
struct cpu_group *	tdq_cg
volatile int	tdq_load
volatile int	tdq_cpu_idle
int	tdq_sysload
int	tdq_transferable
short	tdq_switchcnt
short	tdq_oldswitchcnt
u_char	tdq_lowpri
u_char	tdq_ipipending
u_char	tdq_idx
u_char	tdq_ridx
struct runq	tdq_realtime
struct runq	tdq_timeshare
struct runq	tdq_idle
char	tdq_name [TDQ_NAME_LEN]
static struct tdq	tdq_cpu
static int	ccpu = 0

Macro Definition Documentation

#define KTR_ULE   0

Definition at line 84 of file sched_ule.c.

#define PRI_BATCH_RANGE   (PRI_TIMESHARE_RANGE - PRI_INTERACT_RANGE)

Definition at line 128 of file sched_ule.c.

Referenced by tdq_runq_add().

#define PRI_INTERACT_RANGE   ((PRI_TIMESHARE_RANGE - SCHED_PRI_NRESV) / 2)

Definition at line 127 of file sched_ule.c.

#define PRI_MAX_BATCH   PRI_MAX_TIMESHARE

Definition at line 133 of file sched_ule.c.

Referenced by sched_priority(), and tdq_runq_add().

#define PRI_MAX_INTERACT   (PRI_MIN_TIMESHARE + PRI_INTERACT_RANGE - 1)

Definition at line 131 of file sched_ule.c.

Referenced by sched_priority(), and sched_shouldpreempt().

#define PRI_MIN_BATCH   (PRI_MIN_TIMESHARE + PRI_INTERACT_RANGE)

Definition at line 132 of file sched_ule.c.

Referenced by sched_priority(), tdq_choose(), and tdq_runq_add().

#define PRI_MIN_INTERACT   PRI_MIN_TIMESHARE

Definition at line 130 of file sched_ule.c.

Referenced by sched_priority().

#define PRI_TIMESHARE_RANGE   (PRI_MAX_TIMESHARE - PRI_MIN_TIMESHARE + 1)

Definition at line 126 of file sched_ule.c.

#define SCHED_INTERACT_HALF   (SCHED_INTERACT_MAX / 2)

Definition at line 189 of file sched_ule.c.

Referenced by sched_interact_score().

#define SCHED_INTERACT_MAX   (100)

Definition at line 188 of file sched_ule.c.

#define SCHED_INTERACT_THRESH   (30)

Definition at line 190 of file sched_ule.c.

#define SCHED_PRI_MAX   (PRI_MAX_BATCH - SCHED_PRI_NHALF)

Definition at line 167 of file sched_ule.c.

#define SCHED_PRI_MIN   (PRI_MIN_BATCH + SCHED_PRI_NHALF)

Definition at line 166 of file sched_ule.c.

Referenced by sched_priority().

#define SCHED_PRI_NHALF   (SCHED_PRI_NRESV / 2)

Definition at line 165 of file sched_ule.c.

#define SCHED_PRI_NICE

(

nice

)

(nice)

Definition at line 172 of file sched_ule.c.

Referenced by sched_priority().

#define SCHED_PRI_NRESV   (PRIO_MAX - PRIO_MIN)

Definition at line 164 of file sched_ule.c.

#define SCHED_PRI_RANGE   (SCHED_PRI_MAX - SCHED_PRI_MIN + 1)

Definition at line 168 of file sched_ule.c.

Referenced by sched_priority().

#define SCHED_PRI_TICKS

(

ts

)

Value:

(SCHED_TICK_HZ((ts)) /                                          \
    (roundup(SCHED_TICK_TOTAL((ts)), SCHED_PRI_RANGE) / SCHED_PRI_RANGE))

Definition at line 169 of file sched_ule.c.

Referenced by sched_priority().

#define SCHED_SLP_RUN_FORK   ((hz / 2) << SCHED_TICK_SHIFT)

Definition at line 187 of file sched_ule.c.

Referenced by sched_interact_fork().

#define SCHED_SLP_RUN_MAX   ((hz * 5) << SCHED_TICK_SHIFT)

Definition at line 186 of file sched_ule.c.

Referenced by sched_interact_update().

#define SCHED_TICK_HZ

(

ts

)

((ts)->ts_ticks >> SCHED_TICK_SHIFT)

Definition at line 149 of file sched_ule.c.

Referenced by sched_pctcpu().

#define SCHED_TICK_MAX   (SCHED_TICK_TARG + hz)

Definition at line 147 of file sched_ule.c.

Referenced by sched_pctcpu_update().

#define SCHED_TICK_SECS   10

Definition at line 145 of file sched_ule.c.

Referenced by sched_pctcpu().

#define SCHED_TICK_SHIFT   10

Definition at line 148 of file sched_ule.c.

Referenced by sched_initticks(), sched_pctcpu_update(), and sched_wakeup().

#define SCHED_TICK_TARG   (hz * SCHED_TICK_SECS)

Definition at line 146 of file sched_ule.c.

Referenced by sched_pctcpu_update().

#define SCHED_TICK_TOTAL

(

ts

)

(max((ts)->ts_ltick - (ts)->ts_ftick, hz))

Definition at line 150 of file sched_ule.c.

#define TDF_SLICEEND   TDF_SCHED2 /* Thread time slice is over. */

Definition at line 193 of file sched_ule.c.

Referenced by sched_clock(), and sched_switch().

#define TDQ_CPU

(

x

)

(&tdq_cpu)

Definition at line 283 of file sched_ule.c.

Referenced by sched_fork_exit(), sched_load(), sched_rem(), sched_switch(), sched_switch_migrate(), sched_thread_priority(), and tdq_print().

#define TDQ_ID

(

x

)

(0)

Definition at line 281 of file sched_ule.c.

Referenced by tdq_load_add(), tdq_load_rem(), tdq_print(), tdq_setlowpri(), and tdq_setup().

#define TDQ_IDLE   2

Definition at line 250 of file sched_ule.c.

#define TDQ_IDLESPIN

(

tdq

)

1

Definition at line 2563 of file sched_ule.c.

Referenced by sched_idletd().

#define TDQ_LOADNAME_LEN   (sizeof("CPU ") + sizeof(__XSTRING(MAXCPU)) - 1 + sizeof(" load"))

Definition at line 88 of file sched_ule.c.

#define TDQ_LOCK

(

t

)

mtx_lock_spin(TDQ_LOCKPTR((t)))

Definition at line 287 of file sched_ule.c.

Referenced by sched_add(), sched_setup(), sched_switch(), and sched_throw().

#define TDQ_LOCK_ASSERT	(		t,
			type
	)		mtx_assert(TDQ_LOCKPTR((t)), (type))

Definition at line 286 of file sched_ule.c.

Referenced by sched_choose(), sched_fork_exit(), sched_preempt(), sched_rem(), sched_switch(), tdq_add(), tdq_choose(), tdq_load_add(), tdq_load_rem(), tdq_runq_add(), tdq_runq_rem(), and tdq_setlowpri().

#define TDQ_LOCK_FLAGS	(		t,
			f
	)		mtx_lock_spin_flags(TDQ_LOCKPTR((t)), (f))

Definition at line 288 of file sched_ule.c.

#define TDQ_LOCKPTR

(

t

)

(&(t)->tdq_lock)

Definition at line 290 of file sched_ule.c.

Referenced by sched_add(), sched_fork_exit(), sched_fork_thread(), sched_rem(), sched_setup(), sched_switch(), sched_switch_migrate(), sched_throw(), and tdq_print().

#define TDQ_NAME_LEN   (sizeof("sched lock ") + sizeof(__XSTRING(MAXCPU)))

Definition at line 87 of file sched_ule.c.

#define TDQ_RUNNING   1

Definition at line 249 of file sched_ule.c.

#define TDQ_SELF

(

)

(&tdq_cpu)

Definition at line 282 of file sched_ule.c.

Referenced by sched_add(), sched_choose(), sched_clock(), sched_fork_thread(), sched_idletd(), sched_load(), sched_preempt(), sched_runnable(), sched_setup(), sched_throw(), and sched_userret().

#define TDQ_UNLOCK

(

t

)

mtx_unlock_spin(TDQ_LOCKPTR((t)))

Definition at line 289 of file sched_ule.c.

Referenced by sched_setup(), and sched_switch_migrate().

#define THREAD_CAN_MIGRATE

(

td

)

((td)->td_pinned == 0)

Definition at line 115 of file sched_ule.c.

Referenced by sched_bind(), sched_switch(), and tdq_runq_add().

#define THREAD_CAN_SCHED	(		td,
			cpu
	)		CPU_ISSET((cpu), &(td)->td_cpuset->cs_mask)

Definition at line 116 of file sched_ule.c.

Referenced by sched_affinity(), and sched_switch().

#define TS_NAME_LEN   (MAXCOMLEN + sizeof(" td ") + sizeof(__XSTRING(UINT_MAX)))

Definition at line 86 of file sched_ule.c.

#define TSF_BOUND   0x0001 /* Thread can not migrate. */

Definition at line 110 of file sched_ule.c.

Referenced by sched_bind(), sched_is_bound(), sched_switch(), and sched_unbind().

#define TSF_XFERABLE   0x0002 /* Thread was added as transferable. */

Definition at line 111 of file sched_ule.c.

Referenced by tdq_runq_add(), and tdq_runq_rem().

Function Documentation

struct tdq __aligned

(

64

)

__FBSDID

(

"$BSDSUniX$"

)

static void runq_print ( struct runq *  rq )

static

Definition at line 356 of file sched_ule.c.

References printf().

Referenced by tdq_print().

Here is the call graph for this function:

Here is the caller graph for this function:

void sched_add	(	struct thread *	td,
		int	flags
	)

Definition at line 2339 of file sched_ule.c.

References sched_priority(), sched_setpreempt(), sched_tdname(), tdq_add(), TDQ_LOCK, TDQ_LOCKPTR, TDQ_SELF, and thread_lock_set().

Referenced by sched_affinity(), sched_thread_priority(), and sched_wakeup().

Here is the call graph for this function:

Here is the caller graph for this function:

void sched_affinity

(

struct thread *

td

)

Definition at line 2443 of file sched_ule.c.

References sched_add(), sched_rem(), THREAD_CAN_SCHED, ts, and td_sched::ts_cpu.

Here is the call graph for this function:

void sched_bind	(	struct thread *	td,
		int	cpu
	)

Definition at line 2474 of file sched_ule.c.

References mi_switch(), sched_unbind(), THREAD_CAN_MIGRATE, ts, td_sched::ts_cpu, td_sched::ts_flags, and TSF_BOUND.

Here is the call graph for this function:

struct thread* sched_choose

(

void

)

Definition at line 2269 of file sched_ule.c.

References tdq_choose(), TDQ_LOCK_ASSERT, tdq::tdq_lowpri, tdq_runq_rem(), and TDQ_SELF.

Here is the call graph for this function:

void sched_class	(	struct thread *	td,
		int	class
	)

Definition at line 2069 of file sched_ule.c.

void sched_clock

(

struct thread *

td

)

Definition at line 2174 of file sched_ule.c.

References sched_interact_update(), sched_pctcpu_update(), sched_priority(), sched_slice, TDF_SLICEEND, tdq::tdq_idx, tdq::tdq_load, tdq::tdq_oldswitchcnt, tdq::tdq_ridx, TDQ_SELF, tdq::tdq_switchcnt, tdq::tdq_timeshare, tickincr, ts, and td_sched::ts_slice.

Here is the call graph for this function:

void sched_exit	(	struct proc *	p,
		struct thread *	child
	)

Definition at line 2082 of file sched_ule.c.

References sched_exit_thread(), and sched_tdname().

Here is the call graph for this function:

void sched_exit_thread	(	struct thread *	td,
		struct thread *	child
	)

Definition at line 2100 of file sched_ule.c.

References sched_interact_update(), sched_priority(), and sched_tdname().

Referenced by sched_exit().

Here is the call graph for this function:

Here is the caller graph for this function:

void sched_fork	(	struct thread *	td,
		struct thread *	child
	)

Definition at line 2008 of file sched_ule.c.

References sched_fork_thread(), sched_interact_fork(), sched_interact_update(), sched_pctcpu_update(), sched_priority(), and tickincr.

Here is the call graph for this function:

void sched_fork_exit

(

struct thread *

td

)

Definition at line 2675 of file sched_ule.c.

References TDQ_CPU, TDQ_LOCK_ASSERT, TDQ_LOCKPTR, and ts.

void sched_fork_thread	(	struct thread *	td,
		struct thread *	child
	)

Definition at line 2027 of file sched_ule.c.

References cpuset_ref(), TDQ_LOCKPTR, TDQ_SELF, ts, td_sched::ts_cpu, td_sched::ts_flags, td_sched::ts_ftick, td_sched::ts_ltick, td_sched::ts_runtime, td_sched::ts_slice, td_sched::ts_slptime, and td_sched::ts_ticks.

Referenced by sched_fork().

Here is the call graph for this function:

Here is the caller graph for this function:

void sched_idletd

(

void *

dummy

)

Definition at line 2570 of file sched_ule.c.

References Giant, mi_switch(), sched_idlespins, sched_idlespinthresh, tdq::tdq_cpu_idle, TDQ_IDLESPIN, tdq::tdq_load, tdq::tdq_oldswitchcnt, TDQ_SELF, and tdq::tdq_switchcnt.

Here is the call graph for this function:

static void sched_initticks ( void *  dummy )

static

Definition at line 1372 of file sched_ule.c.

References hogticks, hz, realstathz, sched_idlespinthresh, sched_slice, SCHED_TICK_SHIFT, stathz, and tickincr.

static void sched_interact_fork ( struct thread *  td )

static

Definition at line 1551 of file sched_ule.c.

References SCHED_SLP_RUN_FORK.

Referenced by sched_fork().

Here is the caller graph for this function:

static int sched_interact_score ( struct thread *  td )

static

Definition at line 1414 of file sched_ule.c.

References sched_interact, SCHED_INTERACT_HALF, ts, td_sched::ts_runtime, and td_sched::ts_slptime.

Referenced by sched_priority().

Here is the caller graph for this function:

static void sched_interact_update ( struct thread *  td )

static

Definition at line 1506 of file sched_ule.c.

References SCHED_SLP_RUN_MAX, ts, td_sched::ts_runtime, and td_sched::ts_slptime.

Referenced by sched_clock(), sched_exit_thread(), sched_fork(), and sched_wakeup().

Here is the caller graph for this function:

int sched_is_bound

(

struct thread *

td

)

Definition at line 2511 of file sched_ule.c.

References TSF_BOUND.

void sched_lend_prio	(	struct thread *	td,
		u_char	prio
	)

Definition at line 1679 of file sched_ule.c.

References sched_thread_priority().

Referenced by sched_unlend_prio().

Here is the call graph for this function:

Here is the caller graph for this function:

void sched_lend_user_prio	(	struct thread *	td,
		u_char	prio
	)

Definition at line 1755 of file sched_ule.c.

References sched_prio().

Here is the call graph for this function:

int sched_load

(

void

)

Definition at line 2532 of file sched_ule.c.

References TDQ_CPU, TDQ_SELF, and tdq_sysload.

void sched_nice	(	struct proc *	p,
		int	nice
	)

Definition at line 1939 of file sched_ule.c.

References sched_prio(), and sched_priority().

Here is the call graph for this function:

fixpt_t sched_pctcpu

(

struct thread *

td

)

Definition at line 2415 of file sched_ule.c.

References hz, sched_pctcpu_update(), SCHED_TICK_HZ, SCHED_TICK_SECS, ts, and td_sched::ts_ticks.

Here is the call graph for this function:

static void sched_pctcpu_update ( struct td_sched *  ts, int  run  )

static

Definition at line 1601 of file sched_ule.c.

References SCHED_TICK_MAX, SCHED_TICK_SHIFT, SCHED_TICK_TARG, ticks, td_sched::ts_ftick, td_sched::ts_ltick, and td_sched::ts_ticks.

Referenced by sched_clock(), sched_fork(), sched_pctcpu(), sched_switch(), and sched_wakeup().

Here is the caller graph for this function:

void sched_preempt

(

struct thread *

td

)

Definition at line 2118 of file sched_ule.c.

References mi_switch(), tdq::tdq_ipipending, TDQ_LOCK_ASSERT, tdq::tdq_lowpri, and TDQ_SELF.

Here is the call graph for this function:

void sched_prio	(	struct thread *	td,
		u_char	prio
	)

Definition at line 1715 of file sched_ule.c.

References sched_thread_priority(), and turnstile_adjust().

Referenced by sched_lend_user_prio(), sched_nice(), and sched_sleep().

Here is the call graph for this function:

Here is the caller graph for this function:

static void sched_priority ( struct thread *  td )

static

Definition at line 1454 of file sched_ule.c.

References PRI_MAX_BATCH, PRI_MAX_INTERACT, PRI_MIN_BATCH, PRI_MIN_INTERACT, sched_interact, sched_interact_score(), SCHED_PRI_MIN, SCHED_PRI_NICE, SCHED_PRI_RANGE, SCHED_PRI_TICKS, and sched_user_prio().

Referenced by sched_add(), sched_clock(), sched_exit_thread(), sched_fork(), and sched_nice().

Here is the call graph for this function:

Here is the caller graph for this function:

void sched_relinquish

(

struct thread *

td

)

Definition at line 2521 of file sched_ule.c.

References mi_switch().

Here is the call graph for this function:

void sched_rem

(

struct thread *

td

)

Definition at line 2392 of file sched_ule.c.

References sched_tdname(), TDQ_CPU, tdq_load_rem(), TDQ_LOCK_ASSERT, TDQ_LOCKPTR, tdq::tdq_lowpri, tdq_runq_rem(), and tdq_setlowpri().

Referenced by sched_affinity(), and sched_thread_priority().

Here is the call graph for this function:

Here is the caller graph for this function:

int sched_rr_interval

(

void

)

Definition at line 1587 of file sched_ule.c.

References hz, realstathz, and sched_slice.

int sched_runnable

(

void

)

Definition at line 2244 of file sched_ule.c.

References tdq::tdq_load, and TDQ_SELF.

static void sched_setpreempt ( struct thread *  td )

inlinestatic

Definition at line 2291 of file sched_ule.c.

References panicstr, and sched_shouldpreempt().

Referenced by sched_add().

Here is the call graph for this function:

Here is the caller graph for this function:

static void sched_setup ( void *  dummy )

static

Definition at line 1348 of file sched_ule.c.

References tdq_load_add(), TDQ_LOCK, TDQ_LOCKPTR, tdq::tdq_lowpri, TDQ_SELF, tdq_setup(), and TDQ_UNLOCK.

Here is the call graph for this function:

static int sched_shouldpreempt ( int  pri, int  cpri, int  remote  )

inlinestatic

Definition at line 409 of file sched_ule.c.

References preempt_thresh, and PRI_MAX_INTERACT.

Referenced by sched_setpreempt().

Here is the caller graph for this function:

int sched_sizeof_proc

(

void

)

Definition at line 2548 of file sched_ule.c.

int sched_sizeof_thread

(

void

)

Definition at line 2554 of file sched_ule.c.

void sched_sleep	(	struct thread *	td,
		int	prio
	)

Definition at line 1958 of file sched_ule.c.

References sched_prio(), static_boost, and ticks.

Here is the call graph for this function:

void sched_switch	(	struct thread *	td,
		struct thread *	newtd,
		int	flags
	)

Definition at line 1822 of file sched_ule.c.

References choosethread(), sched_pctcpu_update(), sched_switch_migrate(), TDF_SLICEEND, TDQ_CPU, tdq_load_rem(), TDQ_LOCK, TDQ_LOCK_ASSERT, TDQ_LOCKPTR, tdq_runq_add(), tdq::tdq_switchcnt, THREAD_CAN_MIGRATE, THREAD_CAN_SCHED, thread_lock_block(), thread_unblock_switch(), ticks, ts, td_sched::ts_cpu, td_sched::ts_flags, td_sched::ts_rltick, and TSF_BOUND.

Here is the call graph for this function:

static struct mtx* sched_switch_migrate ( struct tdq *  tdq, struct thread *  td, int  flags  )

static

Definition at line 1772 of file sched_ule.c.

References tdq_add(), TDQ_CPU, tdq_load_rem(), TDQ_LOCKPTR, TDQ_UNLOCK, and thread_lock_block().

Referenced by sched_switch().

Here is the call graph for this function:

Here is the caller graph for this function:

char* sched_tdname

(

struct thread *

td

)

Definition at line 2701 of file sched_ule.c.

References snprintf(), and ts.

Referenced by sched_add(), sched_exit(), sched_exit_thread(), sched_rem(), and sched_thread_priority().

Here is the call graph for this function:

Here is the caller graph for this function:

static void sched_thread_priority ( struct thread *  td, u_char  prio  )

static

Definition at line 1624 of file sched_ule.c.

References sched_add(), sched_rem(), sched_tdname(), TDQ_CPU, tdq_lowpri, tdq::tdq_lowpri, tdq_setlowpri(), ts, and td_sched::ts_cpu.

Referenced by sched_lend_prio(), sched_prio(), and sched_unlend_prio().

Here is the call graph for this function:

Here is the caller graph for this function:

void sched_throw

(

struct thread *

td

)

Definition at line 2645 of file sched_ule.c.

References choosethread(), cpu_ticks, tdq_load_rem(), TDQ_LOCK, TDQ_LOCKPTR, TDQ_SELF, and ticks.

Here is the call graph for this function:

void sched_tick

(

int

cnt

)

Definition at line 2234 of file sched_ule.c.

void sched_unbind

(

struct thread *

td

)

Definition at line 2497 of file sched_ule.c.

References ts, td_sched::ts_flags, and TSF_BOUND.

Referenced by sched_bind().

Here is the caller graph for this function:

void sched_unlend_prio	(	struct thread *	td,
		u_char	prio
	)

Definition at line 1695 of file sched_ule.c.

References sched_lend_prio(), and sched_thread_priority().

Here is the call graph for this function:

void sched_user_prio	(	struct thread *	td,
		u_char	prio
	)

Definition at line 1745 of file sched_ule.c.

Referenced by sched_priority().

Here is the caller graph for this function:

void sched_userret

(

struct thread *

td

)

Definition at line 2147 of file sched_ule.c.

References TDQ_SELF, and tdq_setlowpri().

Here is the call graph for this function:

void sched_wakeup

(

struct thread *

td

)

Definition at line 1979 of file sched_ule.c.

References sched_add(), sched_interact_update(), sched_pctcpu_update(), sched_slice, SCHED_TICK_SHIFT, ticks, ts, td_sched::ts_slice, and td_sched::ts_slptime.

Here is the call graph for this function:

void schedinit

(

void

)

Definition at line 1568 of file sched_ule.c.

References proc0, sched_slice, td_sched0, ticks, td_sched::ts_ftick, td_sched::ts_ltick, and td_sched::ts_slice.

SDT_PROBE_DEFINE	(	sched	,
		on__cpu
	)

SDT_PROBE_DEFINE	(	sched	,
		remain__cpu
	)

SDT_PROBE_DEFINE2	(	sched	,
		load__change	,
		"int"	,
		"int"
	)

SDT_PROBE_DEFINE2	(	sched	,
		off__cpu	,
		"struct thread *"	,
		"struct proc *"
	)

SDT_PROBE_DEFINE2	(	sched	,
		surrender	,
		"struct thread *"	,
		"struct proc *"
	)

SDT_PROBE_DEFINE3	(	sched	,
		change__pri	,
		"struct thread *"	,
		"struct proc *"	,
		"uint8_t"
	)

SDT_PROBE_DEFINE3	(	sched	,
		dequeue	,
		"struct thread *"	,
		"struct proc *"	,
		"void *"
	)

SDT_PROBE_DEFINE4	(	sched	,
		enqueue	,
		"struct thread *"	,
		"struct proc *"	,
		"void *"	,
		"int"
	)

SDT_PROBE_DEFINE4	(	sched	,
		lend__pri	,
		"struct thread *"	,
		"struct proc *"	,
		"uint8_t"	,
		"struct thread *"
	)

SDT_PROVIDER_DEFINE

(

sched

)

SYSCTL_INT	(	_kern_sched	,
		OID_AUTO	,
		slice	,
		CTLFLAG_RW	,
		&	sched_slice,
		0	,
		"Quantum for timeshare threads in stathz ticks"
	)

SYSCTL_INT	(	_kern_sched	,
		OID_AUTO	,
		interact	,
		CTLFLAG_RW	,
		&	sched_interact,
		0	,
		"Interactivity score threshold"
	)

SYSCTL_INT	(	_kern_sched	,
		OID_AUTO	,
		preempt_thresh	,
		CTLFLAG_RW	,
		&	preempt_thresh,
		0	,
		"Maximal (lowest) priority for preemption"
	)

SYSCTL_INT	(	_kern_sched	,
		OID_AUTO	,
		static_boost	,
		CTLFLAG_RW	,
		&	static_boost,
		0	,
		"Assign static kernel priorities to sleeping threads"
	)

SYSCTL_INT	(	_kern_sched	,
		OID_AUTO	,
		idlespins	,
		CTLFLAG_RW	,
		&	sched_idlespins,
		0	,
		"Number of times idle thread will spin waiting for new work"
	)

SYSCTL_INT	(	_kern_sched	,
		OID_AUTO	,
		idlespinthresh	,
		CTLFLAG_RW	,
		&	sched_idlespinthresh,
		0	,
		"Threshold before we will permit idle thread spinning"
	)

SYSCTL_INT	(	_kern	,
		OID_AUTO	,
		ccpu	,
		CTLFLAG_RD	,
		&	ccpu,
		0	,
		""
	)

static int sysctl_kern_quantum ( SYSCTL_HANDLER_ARGS  )

static

Definition at line 2809 of file sched_ule.c.

References hogticks, hz, realstathz, sched_slice, and sysctl_handle_int().

Here is the call graph for this function:

SYSCTL_NODE	(	_kern	,
		OID_AUTO	,
		sched	,
		CTLFLAG_RW	,
		0	,
		"Scheduler"
	)

SYSCTL_PROC	(	_kern_sched	,
		OID_AUTO	,
		quantum	,
		CTLTYPE_INT|	CTLFLAG_RW,
		NULL	,
		0	,
		sysctl_kern_quantum	,
		"I"	,
		"Quantum for timeshare threads in microseconds"
	)

SYSCTL_STRING	(	_kern_sched	,
		OID_AUTO	,
		name	,
		CTLFLAG_RD	,
		"ULE"	,
		0	,
		"Scheduler name"
	)

SYSINIT	(	sched_setup	,
		SI_SUB_RUN_QUEUE	,
		SI_ORDER_FIRST	,
		sched_setup	,
		NULL
	)

SYSINIT	(	sched_initticks	,
		SI_SUB_CLOCKS	,
		SI_ORDER_THIRD	,
		sched_initticks	,
		NULL
	)

void tdq_add ( struct tdq *  tdq, struct thread *  td, int  flags  )

static

Definition at line 2317 of file sched_ule.c.

References tdq_load_add(), TDQ_LOCK_ASSERT, tdq::tdq_lowpri, and tdq_runq_add().

Referenced by sched_add(), and sched_switch_migrate().

Here is the call graph for this function:

Here is the caller graph for this function:

static struct thread * tdq_choose ( struct tdq *  tdq )

static

Definition at line 1275 of file sched_ule.c.

References PRI_MIN_BATCH, runq_choose(), runq_choose_from(), tdq::tdq_idle, TDQ_LOCK_ASSERT, tdq::tdq_realtime, tdq::tdq_ridx, and tdq::tdq_timeshare.

Referenced by sched_choose(), and tdq_setlowpri().

Here is the call graph for this function:

Here is the caller graph for this function:

static void tdq_load_add ( struct tdq *  tdq, struct thread *  td  )

static

Definition at line 524 of file sched_ule.c.

References TDQ_ID, tdq::tdq_load, TDQ_LOCK_ASSERT, and tdq::tdq_sysload.

Referenced by sched_setup(), and tdq_add().

Here is the caller graph for this function:

static void tdq_load_rem ( struct tdq *  tdq, struct thread *  td  )

static

Definition at line 542 of file sched_ule.c.

References TDQ_ID, tdq::tdq_load, TDQ_LOCK_ASSERT, and tdq::tdq_sysload.

Referenced by sched_rem(), sched_switch(), sched_switch_migrate(), and sched_throw().

Here is the caller graph for this function:

void tdq_print

(

int

cpu

)

Definition at line 384 of file sched_ule.c.

References printf(), runq_print(), TDQ_CPU, TDQ_ID, tdq::tdq_idle, tdq::tdq_idx, tdq::tdq_load, TDQ_LOCKPTR, tdq::tdq_lowpri, tdq::tdq_name, tdq::tdq_oldswitchcnt, tdq::tdq_realtime, tdq::tdq_ridx, tdq::tdq_switchcnt, tdq::tdq_timeshare, and tdq::tdq_transferable.

Here is the call graph for this function:

static __inline void tdq_runq_add ( struct tdq *  tdq, struct thread *  td, int  flags  )

static

Definition at line 447 of file sched_ule.c.

References PRI_BATCH_RANGE, PRI_MAX_BATCH, PRI_MIN_BATCH, runq_add(), runq_add_pri(), tdq::tdq_idle, tdq::tdq_idx, TDQ_LOCK_ASSERT, tdq::tdq_realtime, tdq::tdq_ridx, tdq::tdq_timeshare, tdq::tdq_transferable, THREAD_CAN_MIGRATE, ts, td_sched::ts_flags, td_sched::ts_runq, and TSF_XFERABLE.

Referenced by sched_switch(), and tdq_add().

Here is the call graph for this function:

Here is the caller graph for this function:

static __inline void tdq_runq_rem ( struct tdq *  tdq, struct thread *  td  )

static

Definition at line 498 of file sched_ule.c.

References runq_remove(), runq_remove_idx(), tdq::tdq_idx, TDQ_LOCK_ASSERT, tdq::tdq_ridx, tdq::tdq_timeshare, tdq::tdq_transferable, ts, td_sched::ts_flags, td_sched::ts_runq, and TSF_XFERABLE.

Referenced by sched_choose(), and sched_rem().

Here is the call graph for this function:

Here is the caller graph for this function:

static void tdq_setlowpri ( struct tdq *  tdq, struct thread *  ctd  )

static

Definition at line 562 of file sched_ule.c.

References pcpu_find(), tdq_choose(), TDQ_ID, TDQ_LOCK_ASSERT, and tdq::tdq_lowpri.

Referenced by sched_rem(), sched_thread_priority(), and sched_userret().

Here is the call graph for this function:

Here is the caller graph for this function:

static void tdq_setup ( struct tdq *  tdq )

static

Definition at line 1305 of file sched_ule.c.

References bootverbose, mtx_init(), printf(), runq_init(), snprintf(), TDQ_ID, tdq::tdq_idle, tdq::tdq_lock, tdq::tdq_name, tdq::tdq_realtime, and tdq::tdq_timeshare.

Referenced by sched_setup().

Here is the call graph for this function:

Here is the caller graph for this function:

static void thread_unblock_switch ( struct thread *  td, struct mtx *  mtx  )

inlinestatic

Definition at line 1809 of file sched_ule.c.

Referenced by sched_switch().

Here is the caller graph for this function:

Variable Documentation

int ccpu = 0

static

Definition at line 2864 of file sched_ule.c.

int preempt_thresh = 0

static

Definition at line 214 of file sched_ule.c.

Referenced by sched_shouldpreempt().

int realstathz = 127

static

Definition at line 204 of file sched_ule.c.

Referenced by sched_initticks(), sched_rr_interval(), and sysctl_kern_quantum().

int sched_idlespins = 10000

static

Definition at line 217 of file sched_ule.c.

Referenced by sched_idletd().

int sched_idlespinthresh = -1

static

Definition at line 218 of file sched_ule.c.

Referenced by sched_idletd(), and sched_initticks().

int sched_interact = SCHED_INTERACT_THRESH

static

Definition at line 203 of file sched_ule.c.

Referenced by sched_interact_score(), and sched_priority().

int sched_slice = 12

static

Definition at line 206 of file sched_ule.c.

Referenced by sched_clock(), sched_initticks(), sched_rr_interval(), sched_wakeup(), schedinit(), and sysctl_kern_quantum().

int static_boost = PRI_MIN_BATCH

static

Definition at line 216 of file sched_ule.c.

Referenced by sched_sleep().

struct td_sched td_sched0

static

Definition at line 113 of file sched_ule.c.

Referenced by schedinit().

struct cpu_group* tdq_cg

Definition at line 221 of file sched_ule.c.

struct tdq tdq_cpu

static

Definition at line 279 of file sched_ule.c.

volatile int tdq_cpu_idle

Definition at line 223 of file sched_ule.c.

struct runq tdq_idle

Definition at line 234 of file sched_ule.c.

u_char tdq_idx

Definition at line 230 of file sched_ule.c.

u_char tdq_ipipending

Definition at line 229 of file sched_ule.c.

volatile int tdq_load

Definition at line 222 of file sched_ule.c.

struct mtx tdq_lock

Definition at line 220 of file sched_ule.c.

u_char tdq_lowpri

Definition at line 228 of file sched_ule.c.

Referenced by sched_thread_priority().

char tdq_name[TDQ_NAME_LEN]

Definition at line 235 of file sched_ule.c.

short tdq_oldswitchcnt

Definition at line 227 of file sched_ule.c.

struct runq tdq_realtime

Definition at line 232 of file sched_ule.c.

u_char tdq_ridx

Definition at line 231 of file sched_ule.c.

short tdq_switchcnt

Definition at line 226 of file sched_ule.c.

int tdq_sysload

Definition at line 224 of file sched_ule.c.

Referenced by sched_load().

struct runq tdq_timeshare

Definition at line 233 of file sched_ule.c.

int tdq_transferable

Definition at line 225 of file sched_ule.c.

int tickincr = 8 << SCHED_TICK_SHIFT

static

Definition at line 205 of file sched_ule.c.

Referenced by sched_clock(), sched_fork(), and sched_initticks().