1/* $NetBSD: kern_softint.c,v 1.43 2016/07/04 04:20:14 knakahara Exp $ */
2
3/*-
4 * Copyright (c) 2007, 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * Generic software interrupt framework.
34 *
35 * Overview
36 *
37 * The soft interrupt framework provides a mechanism to schedule a
38 * low priority callback that runs with thread context. It allows
39 * for dynamic registration of software interrupts, and for fair
40 * queueing and prioritization of those interrupts. The callbacks
41 * can be scheduled to run from nearly any point in the kernel: by
42 * code running with thread context, by code running from a
43 * hardware interrupt handler, and at any interrupt priority
44 * level.
45 *
46 * Priority levels
47 *
48 * Since soft interrupt dispatch can be tied to the underlying
49 * architecture's interrupt dispatch code, it can be limited
50 * both by the capabilities of the hardware and the capabilities
51 * of the interrupt dispatch code itself. The number of priority
52 * levels is restricted to four. In order of priority (lowest to
53 * highest) the levels are: clock, bio, net, serial.
54 *
55 * The names are symbolic and in isolation do not have any direct
56 * connection with a particular kind of device activity: they are
57 * only meant as a guide.
58 *
59 * The four priority levels map directly to scheduler priority
60 * levels, and where the architecture implements 'fast' software
61 * interrupts, they also map onto interrupt priorities. The
62 * interrupt priorities are intended to be hidden from machine
63 * independent code, which should use thread-safe mechanisms to
64 * synchronize with software interrupts (for example: mutexes).
65 *
66 * Capabilities
67 *
68 * Software interrupts run with limited machine context. In
69 * particular, they do not posess any address space context. They
70 * should not try to operate on user space addresses, or to use
71 * virtual memory facilities other than those noted as interrupt
72 * safe.
73 *
74 * Unlike hardware interrupts, software interrupts do have thread
75 * context. They may block on synchronization objects, sleep, and
76 * resume execution at a later time.
77 *
78 * Since software interrupts are a limited resource and run with
79 * higher priority than most other LWPs in the system, all
80 * block-and-resume activity by a software interrupt must be kept
81 * short to allow futher processing at that level to continue. By
82 * extension, code running with process context must take care to
83 * ensure that any lock that may be taken from a software interrupt
84 * can not be held for more than a short period of time.
85 *
86 * The kernel does not allow software interrupts to use facilities
87 * or perform actions that may block for a significant amount of
88 * time. This means that it's not valid for a software interrupt
89 * to sleep on condition variables or wait for resources to become
90 * available (for example, memory).
91 *
92 * Per-CPU operation
93 *
94 * If a soft interrupt is triggered on a CPU, it can only be
95 * dispatched on the same CPU. Each LWP dedicated to handling a
96 * soft interrupt is bound to its home CPU, so if the LWP blocks
97 * and needs to run again, it can only run there. Nearly all data
98 * structures used to manage software interrupts are per-CPU.
99 *
100 * The per-CPU requirement is intended to reduce "ping-pong" of
101 * cache lines between CPUs: lines occupied by data structures
102 * used to manage the soft interrupts, and lines occupied by data
103 * items being passed down to the soft interrupt. As a positive
104 * side effect, this also means that the soft interrupt dispatch
105 * code does not need to to use spinlocks to synchronize.
106 *
107 * Generic implementation
108 *
109 * A generic, low performance implementation is provided that
110 * works across all architectures, with no machine-dependent
111 * modifications needed. This implementation uses the scheduler,
112 * and so has a number of restrictions:
113 *
114 * 1) The software interrupts are not currently preemptive, so
115 * must wait for the currently executing LWP to yield the CPU.
116 * This can introduce latency.
117 *
118 * 2) An expensive context switch is required for a software
119 * interrupt to be handled.
120 *
121 * 'Fast' software interrupts
122 *
123 * If an architectures defines __HAVE_FAST_SOFTINTS, it implements
124 * the fast mechanism. Threads running either in the kernel or in
125 * userspace will be interrupted, but will not be preempted. When
126 * the soft interrupt completes execution, the interrupted LWP
127 * is resumed. Interrupt dispatch code must provide the minimum
128 * level of context necessary for the soft interrupt to block and
129 * be resumed at a later time. The machine-dependent dispatch
130 * path looks something like the following:
131 *
132 * softintr()
133 * {
134 * go to IPL_HIGH if necessary for switch;
135 * save any necessary registers in a format that can be
136 * restored by cpu_switchto if the softint blocks;
137 * arrange for cpu_switchto() to restore into the
138 * trampoline function;
139 * identify LWP to handle this interrupt;
140 * switch to the LWP's stack;
141 * switch register stacks, if necessary;
142 * assign new value of curlwp;
143 * call MI softint_dispatch, passing old curlwp and IPL
144 * to execute interrupt at;
145 * switch back to old stack;
146 * switch back to old register stack, if necessary;
147 * restore curlwp;
148 * return to interrupted LWP;
149 * }
150 *
151 * If the soft interrupt blocks, a trampoline function is returned
152 * to in the context of the interrupted LWP, as arranged for by
153 * softint():
154 *
155 * softint_ret()
156 * {
157 * unlock soft interrupt LWP;
158 * resume interrupt processing, likely returning to
159 * interrupted LWP or dispatching another, different
160 * interrupt;
161 * }
162 *
163 * Once the soft interrupt has fired (and even if it has blocked),
164 * no further soft interrupts at that level will be triggered by
165 * MI code until the soft interrupt handler has ceased execution.
166 * If a soft interrupt handler blocks and is resumed, it resumes
167 * execution as a normal LWP (kthread) and gains VM context. Only
168 * when it has completed and is ready to fire again will it
169 * interrupt other threads.
170 */
171
172#include <sys/cdefs.h>
173__KERNEL_RCSID(0, "$NetBSD: kern_softint.c,v 1.43 2016/07/04 04:20:14 knakahara Exp $");
174
175#include <sys/param.h>
176#include <sys/proc.h>
177#include <sys/intr.h>
178#include <sys/ipi.h>
179#include <sys/mutex.h>
180#include <sys/kthread.h>
181#include <sys/evcnt.h>
182#include <sys/cpu.h>
183#include <sys/xcall.h>
184#include <sys/pserialize.h>
185
186#include <net/netisr.h>
187
188#include <uvm/uvm_extern.h>
189
190/* This could overlap with signal info in struct lwp. */
191typedef struct softint {
192 SIMPLEQ_HEAD(, softhand) si_q;
193 struct lwp *si_lwp;
194 struct cpu_info *si_cpu;
195 uintptr_t si_machdep;
196 struct evcnt si_evcnt;
197 struct evcnt si_evcnt_block;
198 int si_active;
199 char si_name[8];
200 char si_name_block[8+6];
201} softint_t;
202
203typedef struct softhand {
204 SIMPLEQ_ENTRY(softhand) sh_q;
205 void (*sh_func)(void *);
206 void *sh_arg;
207 softint_t *sh_isr;
208 u_int sh_flags;
209 u_int sh_ipi_id;
210} softhand_t;
211
212typedef struct softcpu {
213 struct cpu_info *sc_cpu;
214 softint_t sc_int[SOFTINT_COUNT];
215 softhand_t sc_hand[1];
216} softcpu_t;
217
218static void softint_thread(void *);
219
220u_int softint_bytes = 8192;
221u_int softint_timing;
222static u_int softint_max;
223static kmutex_t softint_lock;
224static void *softint_netisrs[NETISR_MAX];
225
226/*
227 * softint_init_isr:
228 *
229 * Initialize a single interrupt level for a single CPU.
230 */
231static void
232softint_init_isr(softcpu_t *sc, const char *desc, pri_t pri, u_int level)
233{
234 struct cpu_info *ci;
235 softint_t *si;
236 int error;
237
238 si = &sc->sc_int[level];
239 ci = sc->sc_cpu;
240 si->si_cpu = ci;
241
242 SIMPLEQ_INIT(&si->si_q);
243
244 error = kthread_create(pri, KTHREAD_MPSAFE | KTHREAD_INTR |
245 KTHREAD_IDLE, ci, softint_thread, si, &si->si_lwp,
246 "soft%s/%u", desc, ci->ci_index);
247 if (error != 0)
248 panic("softint_init_isr: error %d", error);
249
250 snprintf(si->si_name, sizeof(si->si_name), "%s/%u", desc,
251 ci->ci_index);
252 evcnt_attach_dynamic(&si->si_evcnt, EVCNT_TYPE_MISC, NULL,
253 "softint", si->si_name);
254 snprintf(si->si_name_block, sizeof(si->si_name_block), "%s block/%u",
255 desc, ci->ci_index);
256 evcnt_attach_dynamic(&si->si_evcnt_block, EVCNT_TYPE_MISC, NULL,
257 "softint", si->si_name_block);
258
259 si->si_lwp->l_private = si;
260 softint_init_md(si->si_lwp, level, &si->si_machdep);
261}
262
263/*
264 * softint_init:
265 *
266 * Initialize per-CPU data structures. Called from mi_cpu_attach().
267 */
268void
269softint_init(struct cpu_info *ci)
270{
271 static struct cpu_info *first;
272 softcpu_t *sc, *scfirst;
273 softhand_t *sh, *shmax;
274
275 if (first == NULL) {
276 /* Boot CPU. */
277 first = ci;
278 mutex_init(&softint_lock, MUTEX_DEFAULT, IPL_NONE);
279 softint_bytes = round_page(softint_bytes);
280 softint_max = (softint_bytes - sizeof(softcpu_t)) /
281 sizeof(softhand_t);
282 }
283
284 /* Use uvm_km(9) for persistent, page-aligned allocation. */
285 sc = (softcpu_t *)uvm_km_alloc(kernel_map, softint_bytes, 0,
286 UVM_KMF_WIRED | UVM_KMF_ZERO);
287 if (sc == NULL)
288 panic("softint_init_cpu: cannot allocate memory");
289
290 ci->ci_data.cpu_softcpu = sc;
291 ci->ci_data.cpu_softints = 0;
292 sc->sc_cpu = ci;
293
294 softint_init_isr(sc, "net", PRI_SOFTNET, SOFTINT_NET);
295 softint_init_isr(sc, "bio", PRI_SOFTBIO, SOFTINT_BIO);
296 softint_init_isr(sc, "clk", PRI_SOFTCLOCK, SOFTINT_CLOCK);
297 softint_init_isr(sc, "ser", PRI_SOFTSERIAL, SOFTINT_SERIAL);
298
299 if (first != ci) {
300 mutex_enter(&softint_lock);
301 scfirst = first->ci_data.cpu_softcpu;
302 sh = sc->sc_hand;
303 memcpy(sh, scfirst->sc_hand, sizeof(*sh) * softint_max);
304 /* Update pointers for this CPU. */
305 for (shmax = sh + softint_max; sh < shmax; sh++) {
306 if (sh->sh_func == NULL)
307 continue;
308 sh->sh_isr =
309 &sc->sc_int[sh->sh_flags & SOFTINT_LVLMASK];
310 }
311 mutex_exit(&softint_lock);
312 } else {
313 /*
314 * Establish handlers for legacy net interrupts.
315 * XXX Needs to go away.
316 */
317#define DONETISR(n, f) \
318 softint_netisrs[(n)] = softint_establish(SOFTINT_NET|SOFTINT_MPSAFE,\
319 (void (*)(void *))(f), NULL)
320#include <net/netisr_dispatch.h>
321 }
322}
323
324/*
325 * softint_establish:
326 *
327 * Register a software interrupt handler.
328 */
329void *
330softint_establish(u_int flags, void (*func)(void *), void *arg)
331{
332 CPU_INFO_ITERATOR cii;
333 struct cpu_info *ci;
334 softcpu_t *sc;
335 softhand_t *sh;
336 u_int level, index;
337 u_int ipi_id = 0;
338 void *sih;
339
340 level = (flags & SOFTINT_LVLMASK);
341 KASSERT(level < SOFTINT_COUNT);
342 KASSERT((flags & SOFTINT_IMPMASK) == 0);
343
344 mutex_enter(&softint_lock);
345
346 /* Find a free slot. */
347 sc = curcpu()->ci_data.cpu_softcpu;
348 for (index = 1; index < softint_max; index++) {
349 if (sc->sc_hand[index].sh_func == NULL)
350 break;
351 }
352 if (index == softint_max) {
353 mutex_exit(&softint_lock);
354 printf("WARNING: softint_establish: table full, "
355 "increase softint_bytes\n");
356 return NULL;
357 }
358 sih = (void *)((uint8_t *)&sc->sc_hand[index] - (uint8_t *)sc);
359
360 if (flags & SOFTINT_RCPU) {
361 if ((ipi_id = ipi_register(softint_schedule, sih)) == 0) {
362 mutex_exit(&softint_lock);
363 return NULL;
364 }
365 }
366
367 /* Set up the handler on each CPU. */
368 if (ncpu < 2) {
369 /* XXX hack for machines with no CPU_INFO_FOREACH() early on */
370 sc = curcpu()->ci_data.cpu_softcpu;
371 sh = &sc->sc_hand[index];
372 sh->sh_isr = &sc->sc_int[level];
373 sh->sh_func = func;
374 sh->sh_arg = arg;
375 sh->sh_flags = flags;
376 sh->sh_ipi_id = ipi_id;
377 } else for (CPU_INFO_FOREACH(cii, ci)) {
378 sc = ci->ci_data.cpu_softcpu;
379 sh = &sc->sc_hand[index];
380 sh->sh_isr = &sc->sc_int[level];
381 sh->sh_func = func;
382 sh->sh_arg = arg;
383 sh->sh_flags = flags;
384 sh->sh_ipi_id = ipi_id;
385 }
386 mutex_exit(&softint_lock);
387
388 return sih;
389}
390
391/*
392 * softint_disestablish:
393 *
394 * Unregister a software interrupt handler. The soft interrupt could
395 * still be active at this point, but the caller commits not to try
396 * and trigger it again once this call is made. The caller must not
397 * hold any locks that could be taken from soft interrupt context,
398 * because we will wait for the softint to complete if it's still
399 * running.
400 */
401void
402softint_disestablish(void *arg)
403{
404 CPU_INFO_ITERATOR cii;
405 struct cpu_info *ci;
406 softcpu_t *sc;
407 softhand_t *sh;
408 uintptr_t offset;
409 uint64_t where;
410 u_int flags;
411
412 offset = (uintptr_t)arg;
413 KASSERTMSG(offset != 0 && offset < softint_bytes, "%"PRIuPTR" %u",
414 offset, softint_bytes);
415
416 /*
417 * Unregister an IPI handler if there is any. Note: there is
418 * no need to disable preemption here - ID is stable.
419 */
420 sc = curcpu()->ci_data.cpu_softcpu;
421 sh = (softhand_t *)((uint8_t *)sc + offset);
422 if (sh->sh_ipi_id) {
423 ipi_unregister(sh->sh_ipi_id);
424 }
425
426 /*
427 * Run a cross call so we see up to date values of sh_flags from
428 * all CPUs. Once softint_disestablish() is called, the caller
429 * commits to not trigger the interrupt and set SOFTINT_ACTIVE on
430 * it again. So, we are only looking for handler records with
431 * SOFTINT_ACTIVE already set.
432 */
433 where = xc_broadcast(0, (xcfunc_t)nullop, NULL, NULL);
434 xc_wait(where);
435
436 for (;;) {
437 /* Collect flag values from each CPU. */
438 flags = 0;
439 for (CPU_INFO_FOREACH(cii, ci)) {
440 sc = ci->ci_data.cpu_softcpu;
441 sh = (softhand_t *)((uint8_t *)sc + offset);
442 KASSERT(sh->sh_func != NULL);
443 flags |= sh->sh_flags;
444 }
445 /* Inactive on all CPUs? */
446 if ((flags & SOFTINT_ACTIVE) == 0) {
447 break;
448 }
449 /* Oops, still active. Wait for it to clear. */
450 (void)kpause("softdis", false, 1, NULL);
451 }
452
453 /* Clear the handler on each CPU. */
454 mutex_enter(&softint_lock);
455 for (CPU_INFO_FOREACH(cii, ci)) {
456 sc = ci->ci_data.cpu_softcpu;
457 sh = (softhand_t *)((uint8_t *)sc + offset);
458 KASSERT(sh->sh_func != NULL);
459 sh->sh_func = NULL;
460 }
461 mutex_exit(&softint_lock);
462}
463
464/*
465 * softint_schedule:
466 *
467 * Trigger a software interrupt. Must be called from a hardware
468 * interrupt handler, or with preemption disabled (since we are
469 * using the value of curcpu()).
470 */
471void
472softint_schedule(void *arg)
473{
474 softhand_t *sh;
475 softint_t *si;
476 uintptr_t offset;
477 int s;
478
479 KASSERT(kpreempt_disabled());
480
481 /* Find the handler record for this CPU. */
482 offset = (uintptr_t)arg;
483 KASSERTMSG(offset != 0 && offset < softint_bytes, "%"PRIuPTR" %u",
484 offset, softint_bytes);
485 sh = (softhand_t *)((uint8_t *)curcpu()->ci_data.cpu_softcpu + offset);
486
487 /* If it's already pending there's nothing to do. */
488 if ((sh->sh_flags & SOFTINT_PENDING) != 0) {
489 return;
490 }
491
492 /*
493 * Enqueue the handler into the LWP's pending list.
494 * If the LWP is completely idle, then make it run.
495 */
496 s = splhigh();
497 if ((sh->sh_flags & SOFTINT_PENDING) == 0) {
498 si = sh->sh_isr;
499 sh->sh_flags |= SOFTINT_PENDING;
500 SIMPLEQ_INSERT_TAIL(&si->si_q, sh, sh_q);
501 if (si->si_active == 0) {
502 si->si_active = 1;
503 softint_trigger(si->si_machdep);
504 }
505 }
506 splx(s);
507}
508
509/*
510 * softint_schedule_cpu:
511 *
512 * Trigger a software interrupt on a target CPU. This invokes
513 * softint_schedule() for the local CPU or send an IPI to invoke
514 * this routine on the remote CPU. Preemption must be disabled.
515 */
516void
517softint_schedule_cpu(void *arg, struct cpu_info *ci)
518{
519 KASSERT(kpreempt_disabled());
520
521 if (curcpu() != ci) {
522 const softcpu_t *sc = ci->ci_data.cpu_softcpu;
523 const uintptr_t offset = (uintptr_t)arg;
524 const softhand_t *sh;
525
526 sh = (const softhand_t *)((const uint8_t *)sc + offset);
527 KASSERT((sh->sh_flags & SOFTINT_RCPU) != 0);
528 ipi_trigger(sh->sh_ipi_id, ci);
529 return;
530 }
531
532 /* Just a local CPU. */
533 softint_schedule(arg);
534}
535
536/*
537 * softint_execute:
538 *
539 * Invoke handlers for the specified soft interrupt.
540 * Must be entered at splhigh. Will drop the priority
541 * to the level specified, but returns back at splhigh.
542 */
543static inline void
544softint_execute(softint_t *si, lwp_t *l, int s)
545{
546 softhand_t *sh;
547 bool havelock;
548
549#ifdef __HAVE_FAST_SOFTINTS
550 KASSERT(si->si_lwp == curlwp);
551#else
552 /* May be running in user context. */
553#endif
554 KASSERT(si->si_cpu == curcpu());
555 KASSERT(si->si_lwp->l_wchan == NULL);
556 KASSERT(si->si_active);
557
558 havelock = false;
559
560 /*
561 * Note: due to priority inheritance we may have interrupted a
562 * higher priority LWP. Since the soft interrupt must be quick
563 * and is non-preemptable, we don't bother yielding.
564 */
565
566 while (!SIMPLEQ_EMPTY(&si->si_q)) {
567 /*
568 * Pick the longest waiting handler to run. We block
569 * interrupts but do not lock in order to do this, as
570 * we are protecting against the local CPU only.
571 */
572 sh = SIMPLEQ_FIRST(&si->si_q);
573 SIMPLEQ_REMOVE_HEAD(&si->si_q, sh_q);
574 KASSERT((sh->sh_flags & SOFTINT_PENDING) != 0);
575 KASSERT((sh->sh_flags & SOFTINT_ACTIVE) == 0);
576 sh->sh_flags ^= (SOFTINT_PENDING | SOFTINT_ACTIVE);
577 splx(s);
578
579 /* Run the handler. */
580 if (sh->sh_flags & SOFTINT_MPSAFE) {
581 if (havelock) {
582 KERNEL_UNLOCK_ONE(l);
583 havelock = false;
584 }
585 } else if (!havelock) {
586 KERNEL_LOCK(1, l);
587 havelock = true;
588 }
589 (*sh->sh_func)(sh->sh_arg);
590
591 /* Diagnostic: check that spin-locks have not leaked. */
592 KASSERTMSG(curcpu()->ci_mtx_count == 0,
593 "%s: ci_mtx_count (%d) != 0, sh_func %p\n",
594 __func__, curcpu()->ci_mtx_count, sh->sh_func);
595
596 (void)splhigh();
597 KASSERT((sh->sh_flags & SOFTINT_ACTIVE) != 0);
598 sh->sh_flags ^= SOFTINT_ACTIVE;
599 }
600
601 if (havelock) {
602 KERNEL_UNLOCK_ONE(l);
603 }
604
605 /*
606 * Unlocked, but only for statistics.
607 * Should be per-CPU to prevent cache ping-pong.
608 */
609 curcpu()->ci_data.cpu_nsoft++;
610
611 KASSERT(si->si_cpu == curcpu());
612 KASSERT(si->si_lwp->l_wchan == NULL);
613 KASSERT(si->si_active);
614 si->si_evcnt.ev_count++;
615 si->si_active = 0;
616}
617
618/*
619 * softint_block:
620 *
621 * Update statistics when the soft interrupt blocks.
622 */
623void
624softint_block(lwp_t *l)
625{
626 softint_t *si = l->l_private;
627
628 KASSERT((l->l_pflag & LP_INTR) != 0);
629 si->si_evcnt_block.ev_count++;
630}
631
632/*
633 * schednetisr:
634 *
635 * Trigger a legacy network interrupt. XXX Needs to go away.
636 */
637void
638schednetisr(int isr)
639{
640
641 softint_schedule(softint_netisrs[isr]);
642}
643
644#ifndef __HAVE_FAST_SOFTINTS
645
646#ifdef __HAVE_PREEMPTION
647#error __HAVE_PREEMPTION requires __HAVE_FAST_SOFTINTS
648#endif
649
650/*
651 * softint_init_md:
652 *
653 * Slow path: perform machine-dependent initialization.
654 */
655void
656softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep)
657{
658 softint_t *si;
659
660 *machdep = (1 << level);
661 si = l->l_private;
662
663 lwp_lock(l);
664 lwp_unlock_to(l, l->l_cpu->ci_schedstate.spc_mutex);
665 lwp_lock(l);
666 /* Cheat and make the KASSERT in softint_thread() happy. */
667 si->si_active = 1;
668 l->l_stat = LSRUN;
669 sched_enqueue(l, false);
670 lwp_unlock(l);
671}
672
673/*
674 * softint_trigger:
675 *
676 * Slow path: cause a soft interrupt handler to begin executing.
677 * Called at IPL_HIGH.
678 */
679void
680softint_trigger(uintptr_t machdep)
681{
682 struct cpu_info *ci;
683 lwp_t *l;
684
685 l = curlwp;
686 ci = l->l_cpu;
687 ci->ci_data.cpu_softints |= machdep;
688 if (l == ci->ci_data.cpu_idlelwp) {
689 cpu_need_resched(ci, 0);
690 } else {
691 /* MI equivalent of aston() */
692 cpu_signotify(l);
693 }
694}
695
696/*
697 * softint_thread:
698 *
699 * Slow path: MI software interrupt dispatch.
700 */
701void
702softint_thread(void *cookie)
703{
704 softint_t *si;
705 lwp_t *l;
706 int s;
707
708 l = curlwp;
709 si = l->l_private;
710
711 for (;;) {
712 /*
713 * Clear pending status and run it. We must drop the
714 * spl before mi_switch(), since IPL_HIGH may be higher
715 * than IPL_SCHED (and it is not safe to switch at a
716 * higher level).
717 */
718 s = splhigh();
719 l->l_cpu->ci_data.cpu_softints &= ~si->si_machdep;
720 softint_execute(si, l, s);
721 splx(s);
722
723 lwp_lock(l);
724 l->l_stat = LSIDL;
725 mi_switch(l);
726 }
727}
728
729/*
730 * softint_picklwp:
731 *
732 * Slow path: called from mi_switch() to pick the highest priority
733 * soft interrupt LWP that needs to run.
734 */
735lwp_t *
736softint_picklwp(void)
737{
738 struct cpu_info *ci;
739 u_int mask;
740 softint_t *si;
741 lwp_t *l;
742
743 ci = curcpu();
744 si = ((softcpu_t *)ci->ci_data.cpu_softcpu)->sc_int;
745 mask = ci->ci_data.cpu_softints;
746
747 if ((mask & (1 << SOFTINT_SERIAL)) != 0) {
748 l = si[SOFTINT_SERIAL].si_lwp;
749 } else if ((mask & (1 << SOFTINT_NET)) != 0) {
750 l = si[SOFTINT_NET].si_lwp;
751 } else if ((mask & (1 << SOFTINT_BIO)) != 0) {
752 l = si[SOFTINT_BIO].si_lwp;
753 } else if ((mask & (1 << SOFTINT_CLOCK)) != 0) {
754 l = si[SOFTINT_CLOCK].si_lwp;
755 } else {
756 panic("softint_picklwp");
757 }
758
759 return l;
760}
761
762/*
763 * softint_overlay:
764 *
765 * Slow path: called from lwp_userret() to run a soft interrupt
766 * within the context of a user thread.
767 */
768void
769softint_overlay(void)
770{
771 struct cpu_info *ci;
772 u_int softints, oflag;
773 softint_t *si;
774 pri_t obase;
775 lwp_t *l;
776 int s;
777
778 l = curlwp;
779 KASSERT((l->l_pflag & LP_INTR) == 0);
780
781 /*
782 * Arrange to elevate priority if the LWP blocks. Also, bind LWP
783 * to the CPU. Note: disable kernel preemption before doing that.
784 */
785 s = splhigh();
786 ci = l->l_cpu;
787 si = ((softcpu_t *)ci->ci_data.cpu_softcpu)->sc_int;
788
789 obase = l->l_kpribase;
790 l->l_kpribase = PRI_KERNEL_RT;
791 oflag = l->l_pflag;
792 l->l_pflag = oflag | LP_INTR | LP_BOUND;
793
794 while ((softints = ci->ci_data.cpu_softints) != 0) {
795 if ((softints & (1 << SOFTINT_SERIAL)) != 0) {
796 ci->ci_data.cpu_softints &= ~(1 << SOFTINT_SERIAL);
797 softint_execute(&si[SOFTINT_SERIAL], l, s);
798 continue;
799 }
800 if ((softints & (1 << SOFTINT_NET)) != 0) {
801 ci->ci_data.cpu_softints &= ~(1 << SOFTINT_NET);
802 softint_execute(&si[SOFTINT_NET], l, s);
803 continue;
804 }
805 if ((softints & (1 << SOFTINT_BIO)) != 0) {
806 ci->ci_data.cpu_softints &= ~(1 << SOFTINT_BIO);
807 softint_execute(&si[SOFTINT_BIO], l, s);
808 continue;
809 }
810 if ((softints & (1 << SOFTINT_CLOCK)) != 0) {
811 ci->ci_data.cpu_softints &= ~(1 << SOFTINT_CLOCK);
812 softint_execute(&si[SOFTINT_CLOCK], l, s);
813 continue;
814 }
815 }
816 l->l_pflag = oflag;
817 l->l_kpribase = obase;
818 splx(s);
819}
820
821#else /* !__HAVE_FAST_SOFTINTS */
822
823/*
824 * softint_thread:
825 *
826 * Fast path: the LWP is switched to without restoring any state,
827 * so we should not arrive here - there is a direct handoff between
828 * the interrupt stub and softint_dispatch().
829 */
830void
831softint_thread(void *cookie)
832{
833
834 panic("softint_thread");
835}
836
837/*
838 * softint_dispatch:
839 *
840 * Fast path: entry point from machine-dependent code.
841 */
842void
843softint_dispatch(lwp_t *pinned, int s)
844{
845 struct bintime now;
846 softint_t *si;
847 u_int timing;
848 lwp_t *l;
849
850 KASSERT((pinned->l_pflag & LP_RUNNING) != 0);
851 l = curlwp;
852 si = l->l_private;
853
854 /*
855 * Note the interrupted LWP, and mark the current LWP as running
856 * before proceeding. Although this must as a rule be done with
857 * the LWP locked, at this point no external agents will want to
858 * modify the interrupt LWP's state.
859 */
860 timing = (softint_timing ? LP_TIMEINTR : 0);
861 l->l_switchto = pinned;
862 l->l_stat = LSONPROC;
863 l->l_pflag |= (LP_RUNNING | timing);
864
865 /*
866 * Dispatch the interrupt. If softints are being timed, charge
867 * for it.
868 */
869 if (timing)
870 binuptime(&l->l_stime);
871 softint_execute(si, l, s);
872 if (timing) {
873 binuptime(&now);
874 updatertime(l, &now);
875 l->l_pflag &= ~LP_TIMEINTR;
876 }
877
878 /* Indicate a soft-interrupt switch. */
879 pserialize_switchpoint();
880
881 /*
882 * If we blocked while handling the interrupt, the pinned LWP is
883 * gone so switch to the idle LWP. It will select a new LWP to
884 * run.
885 *
886 * We must drop the priority level as switching at IPL_HIGH could
887 * deadlock the system. We have already set si->si_active = 0,
888 * which means another interrupt at this level can be triggered.
889 * That's not be a problem: we are lowering to level 's' which will
890 * prevent softint_dispatch() from being reentered at level 's',
891 * until the priority is finally dropped to IPL_NONE on entry to
892 * the LWP chosen by lwp_exit_switchaway().
893 */
894 l->l_stat = LSIDL;
895 if (l->l_switchto == NULL) {
896 splx(s);
897 pmap_deactivate(l);
898 lwp_exit_switchaway(l);
899 /* NOTREACHED */
900 }
901 l->l_switchto = NULL;
902 l->l_pflag &= ~LP_RUNNING;
903}
904
905#endif /* !__HAVE_FAST_SOFTINTS */
906