1 | /* $NetBSD: sleepq.h,v 1.24 2015/02/08 19:39:09 christos Exp $ */ |
2 | |
3 | /*- |
4 | * Copyright (c) 2002, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc. |
5 | * All rights reserved. |
6 | * |
7 | * This code is derived from software contributed to The NetBSD Foundation |
8 | * by Jason R. Thorpe and Andrew Doran. |
9 | * |
10 | * Redistribution and use in source and binary forms, with or without |
11 | * modification, are permitted provided that the following conditions |
12 | * are met: |
13 | * 1. Redistributions of source code must retain the above copyright |
14 | * notice, this list of conditions and the following disclaimer. |
15 | * 2. Redistributions in binary form must reproduce the above copyright |
16 | * notice, this list of conditions and the following disclaimer in the |
17 | * documentation and/or other materials provided with the distribution. |
18 | * |
19 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
20 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
21 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
23 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
24 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
25 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
26 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
27 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
28 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
29 | * POSSIBILITY OF SUCH DAMAGE. |
30 | */ |
31 | |
32 | #ifndef _SYS_SLEEPQ_H_ |
33 | #define _SYS_SLEEPQ_H_ |
34 | |
35 | #include <sys/lwp.h> |
36 | #include <sys/mutex.h> |
37 | #include <sys/pool.h> |
38 | #include <sys/queue.h> |
39 | #include <sys/sched.h> |
40 | #include <sys/syncobj.h> |
41 | |
42 | /* |
43 | * Generic sleep queues. |
44 | */ |
45 | |
46 | #define SLEEPTAB_HASH_SHIFT 7 |
47 | #define SLEEPTAB_HASH_SIZE (1 << SLEEPTAB_HASH_SHIFT) |
48 | #define SLEEPTAB_HASH_MASK (SLEEPTAB_HASH_SIZE - 1) |
49 | #define SLEEPTAB_HASH(wchan) (((uintptr_t)(wchan) >> 8) & SLEEPTAB_HASH_MASK) |
50 | |
51 | TAILQ_HEAD(sleepq, lwp); |
52 | |
53 | typedef struct sleepq sleepq_t; |
54 | |
55 | typedef struct sleeptab { |
56 | struct { |
57 | kmutex_t *st_mutex; |
58 | sleepq_t st_queue; |
59 | } st_queues[SLEEPTAB_HASH_SIZE]; |
60 | } sleeptab_t; |
61 | |
62 | void sleepq_init(sleepq_t *); |
63 | void sleepq_remove(sleepq_t *, lwp_t *); |
64 | void sleepq_enqueue(sleepq_t *, wchan_t, const char *, struct syncobj *); |
65 | void sleepq_unsleep(lwp_t *, bool); |
66 | void sleepq_timeout(void *); |
67 | void sleepq_wake(sleepq_t *, wchan_t, u_int, kmutex_t *); |
68 | int sleepq_abort(kmutex_t *, int); |
69 | void sleepq_changepri(lwp_t *, pri_t); |
70 | void sleepq_lendpri(lwp_t *, pri_t); |
71 | int sleepq_block(int, bool); |
72 | |
73 | void sleeptab_init(sleeptab_t *); |
74 | |
75 | extern sleeptab_t sleeptab; |
76 | |
77 | #ifdef _KERNEL |
78 | /* |
79 | * Return non-zero if it is unsafe to sleep. |
80 | * |
81 | * XXX This only exists because panic() is broken. |
82 | */ |
83 | static inline bool |
84 | sleepq_dontsleep(lwp_t *l) |
85 | { |
86 | extern int cold; |
87 | |
88 | return cold || (doing_shutdown && (panicstr || CURCPU_IDLE_P())); |
89 | } |
90 | |
91 | /* |
92 | * Find the correct sleep queue for the specified wait channel. This |
93 | * acquires and holds the per-queue interlock. |
94 | */ |
95 | static inline sleepq_t * |
96 | sleeptab_lookup(sleeptab_t *st, wchan_t wchan, kmutex_t **mp) |
97 | { |
98 | sleepq_t *sq; |
99 | |
100 | sq = &st->st_queues[SLEEPTAB_HASH(wchan)].st_queue; |
101 | *mp = st->st_queues[SLEEPTAB_HASH(wchan)].st_mutex; |
102 | mutex_spin_enter(*mp); |
103 | return sq; |
104 | } |
105 | |
106 | static inline kmutex_t * |
107 | sleepq_hashlock(wchan_t wchan) |
108 | { |
109 | kmutex_t *mp; |
110 | |
111 | mp = sleeptab.st_queues[SLEEPTAB_HASH(wchan)].st_mutex; |
112 | mutex_spin_enter(mp); |
113 | return mp; |
114 | } |
115 | |
116 | /* |
117 | * Prepare to block on a sleep queue, after which any interlock can be |
118 | * safely released. |
119 | */ |
120 | static inline void |
121 | sleepq_enter(sleepq_t *sq, lwp_t *l, kmutex_t *mp) |
122 | { |
123 | |
124 | /* |
125 | * Acquire the per-LWP mutex and lend it ours sleep queue lock. |
126 | * Once interlocked, we can release the kernel lock. |
127 | */ |
128 | lwp_lock(l); |
129 | lwp_unlock_to(l, mp); |
130 | KERNEL_UNLOCK_ALL(NULL, &l->l_biglocks); |
131 | } |
132 | #endif |
133 | |
134 | /* |
135 | * Turnstiles, specialized sleep queues for use by kernel locks. |
136 | */ |
137 | |
138 | typedef struct turnstile { |
139 | LIST_ENTRY(turnstile) ts_chain; /* link on hash chain */ |
140 | struct turnstile *ts_free; /* turnstile free list */ |
141 | wchan_t ts_obj; /* lock object */ |
142 | sleepq_t ts_sleepq[2]; /* sleep queues */ |
143 | u_int ts_waiters[2]; /* count of waiters */ |
144 | |
145 | /* priority inheritance */ |
146 | pri_t ts_eprio; |
147 | lwp_t *ts_inheritor; |
148 | SLIST_ENTRY(turnstile) ts_pichain; |
149 | } turnstile_t; |
150 | |
151 | typedef struct tschain { |
152 | kmutex_t *tc_mutex; /* mutex on structs & queues */ |
153 | LIST_HEAD(, turnstile) tc_chain; /* turnstile chain */ |
154 | } tschain_t; |
155 | |
156 | #define TS_READER_Q 0 /* reader sleep queue */ |
157 | #define TS_WRITER_Q 1 /* writer sleep queue */ |
158 | |
159 | #define TS_WAITERS(ts, q) \ |
160 | (ts)->ts_waiters[(q)] |
161 | |
162 | #define TS_ALL_WAITERS(ts) \ |
163 | ((ts)->ts_waiters[TS_READER_Q] + \ |
164 | (ts)->ts_waiters[TS_WRITER_Q]) |
165 | |
166 | #define TS_FIRST(ts, q) (TAILQ_FIRST(&(ts)->ts_sleepq[(q)])) |
167 | |
168 | #ifdef _KERNEL |
169 | |
170 | void turnstile_init(void); |
171 | turnstile_t *turnstile_lookup(wchan_t); |
172 | void turnstile_exit(wchan_t); |
173 | void turnstile_block(turnstile_t *, int, wchan_t, syncobj_t *); |
174 | void turnstile_wakeup(turnstile_t *, int, int, lwp_t *); |
175 | void turnstile_print(volatile void *, void (*)(const char *, ...) |
176 | __printflike(1, 2)); |
177 | void turnstile_unsleep(lwp_t *, bool); |
178 | void turnstile_changepri(lwp_t *, pri_t); |
179 | |
180 | extern pool_cache_t turnstile_cache; |
181 | extern turnstile_t turnstile0; |
182 | |
183 | #endif /* _KERNEL */ |
184 | |
185 | #endif /* _SYS_SLEEPQ_H_ */ |
186 | |