Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
4 : * Copyright (C) 2005-2006, Thomas Gleixner, Russell King
5 : *
6 : * This file contains the core interrupt handling code, for irq-chip based
7 : * architectures. Detailed information is available in
8 : * Documentation/core-api/genericirq.rst
9 : */
10 :
11 : #include <linux/irq.h>
12 : #include <linux/msi.h>
13 : #include <linux/module.h>
14 : #include <linux/interrupt.h>
15 : #include <linux/kernel_stat.h>
16 : #include <linux/irqdomain.h>
17 :
18 : #include <trace/events/irq.h>
19 :
20 : #include "internals.h"
21 :
22 0 : static irqreturn_t bad_chained_irq(int irq, void *dev_id)
23 : {
24 0 : WARN_ONCE(1, "Chained irq %d should not call an action\n", irq);
25 0 : return IRQ_NONE;
26 : }
27 :
28 : /*
29 : * Chained handlers should never call action on their IRQ. This default
30 : * action will emit warning if such thing happens.
31 : */
32 : struct irqaction chained_action = {
33 : .handler = bad_chained_irq,
34 : };
35 :
36 : /**
37 : * irq_set_chip - set the irq chip for an irq
38 : * @irq: irq number
39 : * @chip: pointer to irq chip description structure
40 : */
41 16 : int irq_set_chip(unsigned int irq, struct irq_chip *chip)
42 : {
43 16 : unsigned long flags;
44 16 : struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
45 :
46 16 : if (!desc)
47 : return -EINVAL;
48 :
49 16 : if (!chip)
50 0 : chip = &no_irq_chip;
51 :
52 16 : desc->irq_data.chip = chip;
53 16 : irq_put_desc_unlock(desc, flags);
54 : /*
55 : * For !CONFIG_SPARSE_IRQ make the irq show up in
56 : * allocated_irqs.
57 : */
58 16 : irq_mark_irq(irq);
59 16 : return 0;
60 : }
61 : EXPORT_SYMBOL(irq_set_chip);
62 :
63 : /**
64 : * irq_set_irq_type - set the irq trigger type for an irq
65 : * @irq: irq number
66 : * @type: IRQ_TYPE_{LEVEL,EDGE}_* value - see include/linux/irq.h
67 : */
68 0 : int irq_set_irq_type(unsigned int irq, unsigned int type)
69 : {
70 0 : unsigned long flags;
71 0 : struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
72 0 : int ret = 0;
73 :
74 0 : if (!desc)
75 : return -EINVAL;
76 :
77 0 : ret = __irq_set_trigger(desc, type);
78 0 : irq_put_desc_busunlock(desc, flags);
79 0 : return ret;
80 : }
81 : EXPORT_SYMBOL(irq_set_irq_type);
82 :
83 : /**
84 : * irq_set_handler_data - set irq handler data for an irq
85 : * @irq: Interrupt number
86 : * @data: Pointer to interrupt specific data
87 : *
88 : * Set the hardware irq controller data for an irq
89 : */
90 0 : int irq_set_handler_data(unsigned int irq, void *data)
91 : {
92 0 : unsigned long flags;
93 0 : struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
94 :
95 0 : if (!desc)
96 : return -EINVAL;
97 0 : desc->irq_common_data.handler_data = data;
98 0 : irq_put_desc_unlock(desc, flags);
99 0 : return 0;
100 : }
101 : EXPORT_SYMBOL(irq_set_handler_data);
102 :
103 : /**
104 : * irq_set_msi_desc_off - set MSI descriptor data for an irq at offset
105 : * @irq_base: Interrupt number base
106 : * @irq_offset: Interrupt number offset
107 : * @entry: Pointer to MSI descriptor data
108 : *
109 : * Set the MSI descriptor entry for an irq at offset
110 : */
111 0 : int irq_set_msi_desc_off(unsigned int irq_base, unsigned int irq_offset,
112 : struct msi_desc *entry)
113 : {
114 0 : unsigned long flags;
115 0 : struct irq_desc *desc = irq_get_desc_lock(irq_base + irq_offset, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
116 :
117 0 : if (!desc)
118 : return -EINVAL;
119 0 : desc->irq_common_data.msi_desc = entry;
120 0 : if (entry && !irq_offset)
121 0 : entry->irq = irq_base;
122 0 : irq_put_desc_unlock(desc, flags);
123 0 : return 0;
124 : }
125 :
126 : /**
127 : * irq_set_msi_desc - set MSI descriptor data for an irq
128 : * @irq: Interrupt number
129 : * @entry: Pointer to MSI descriptor data
130 : *
131 : * Set the MSI descriptor entry for an irq
132 : */
133 0 : int irq_set_msi_desc(unsigned int irq, struct msi_desc *entry)
134 : {
135 0 : return irq_set_msi_desc_off(irq, 0, entry);
136 : }
137 :
138 : /**
139 : * irq_set_chip_data - set irq chip data for an irq
140 : * @irq: Interrupt number
141 : * @data: Pointer to chip specific data
142 : *
143 : * Set the hardware irq chip data for an irq
144 : */
145 0 : int irq_set_chip_data(unsigned int irq, void *data)
146 : {
147 0 : unsigned long flags;
148 0 : struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
149 :
150 0 : if (!desc)
151 : return -EINVAL;
152 0 : desc->irq_data.chip_data = data;
153 0 : irq_put_desc_unlock(desc, flags);
154 0 : return 0;
155 : }
156 : EXPORT_SYMBOL(irq_set_chip_data);
157 :
158 137 : struct irq_data *irq_get_irq_data(unsigned int irq)
159 : {
160 137 : struct irq_desc *desc = irq_to_desc(irq);
161 :
162 137 : return desc ? &desc->irq_data : NULL;
163 : }
164 : EXPORT_SYMBOL_GPL(irq_get_irq_data);
165 :
166 4 : static void irq_state_clr_disabled(struct irq_desc *desc)
167 : {
168 4 : irqd_clear(&desc->irq_data, IRQD_IRQ_DISABLED);
169 : }
170 :
171 4 : static void irq_state_clr_masked(struct irq_desc *desc)
172 : {
173 4 : irqd_clear(&desc->irq_data, IRQD_IRQ_MASKED);
174 1 : }
175 :
176 0 : static void irq_state_clr_started(struct irq_desc *desc)
177 : {
178 0 : irqd_clear(&desc->irq_data, IRQD_IRQ_STARTED);
179 0 : }
180 :
181 4 : static void irq_state_set_started(struct irq_desc *desc)
182 : {
183 4 : irqd_set(&desc->irq_data, IRQD_IRQ_STARTED);
184 : }
185 :
186 : enum {
187 : IRQ_STARTUP_NORMAL,
188 : IRQ_STARTUP_MANAGED,
189 : IRQ_STARTUP_ABORT,
190 : };
191 :
192 : #ifdef CONFIG_SMP
193 : static int
194 4 : __irq_startup_managed(struct irq_desc *desc, struct cpumask *aff, bool force)
195 : {
196 4 : struct irq_data *d = irq_desc_get_irq_data(desc);
197 :
198 4 : if (!irqd_affinity_is_managed(d))
199 : return IRQ_STARTUP_NORMAL;
200 :
201 0 : irqd_clr_managed_shutdown(d);
202 :
203 0 : if (cpumask_any_and(aff, cpu_online_mask) >= nr_cpu_ids) {
204 : /*
205 : * Catch code which fiddles with enable_irq() on a managed
206 : * and potentially shutdown IRQ. Chained interrupt
207 : * installment or irq auto probing should not happen on
208 : * managed irqs either.
209 : */
210 0 : if (WARN_ON_ONCE(force))
211 : return IRQ_STARTUP_ABORT;
212 : /*
213 : * The interrupt was requested, but there is no online CPU
214 : * in it's affinity mask. Put it into managed shutdown
215 : * state and let the cpu hotplug mechanism start it up once
216 : * a CPU in the mask becomes available.
217 : */
218 : return IRQ_STARTUP_ABORT;
219 : }
220 : /*
221 : * Managed interrupts have reserved resources, so this should not
222 : * happen.
223 : */
224 0 : if (WARN_ON(irq_domain_activate_irq(d, false)))
225 0 : return IRQ_STARTUP_ABORT;
226 : return IRQ_STARTUP_MANAGED;
227 : }
228 : #else
229 : static __always_inline int
230 : __irq_startup_managed(struct irq_desc *desc, struct cpumask *aff, bool force)
231 : {
232 : return IRQ_STARTUP_NORMAL;
233 : }
234 : #endif
235 :
236 4 : static int __irq_startup(struct irq_desc *desc)
237 : {
238 4 : struct irq_data *d = irq_desc_get_irq_data(desc);
239 4 : int ret = 0;
240 :
241 : /* Warn if this interrupt is not activated but try nevertheless */
242 4 : WARN_ON_ONCE(!irqd_is_activated(d));
243 :
244 4 : if (d->chip->irq_startup) {
245 3 : ret = d->chip->irq_startup(d);
246 3 : irq_state_clr_disabled(desc);
247 3 : irq_state_clr_masked(desc);
248 : } else {
249 1 : irq_enable(desc);
250 : }
251 4 : irq_state_set_started(desc);
252 4 : return ret;
253 : }
254 :
255 4 : int irq_startup(struct irq_desc *desc, bool resend, bool force)
256 : {
257 4 : struct irq_data *d = irq_desc_get_irq_data(desc);
258 4 : struct cpumask *aff = irq_data_get_affinity_mask(d);
259 4 : int ret = 0;
260 :
261 4 : desc->depth = 0;
262 :
263 4 : if (irqd_is_started(d)) {
264 0 : irq_enable(desc);
265 : } else {
266 4 : switch (__irq_startup_managed(desc, aff, force)) {
267 4 : case IRQ_STARTUP_NORMAL:
268 4 : ret = __irq_startup(desc);
269 4 : irq_setup_affinity(desc);
270 4 : break;
271 0 : case IRQ_STARTUP_MANAGED:
272 0 : irq_do_set_affinity(d, aff, false);
273 0 : ret = __irq_startup(desc);
274 0 : break;
275 : case IRQ_STARTUP_ABORT:
276 0 : irqd_set_managed_shutdown(d);
277 0 : return 0;
278 : }
279 4 : }
280 4 : if (resend)
281 4 : check_irq_resend(desc, false);
282 :
283 : return ret;
284 : }
285 :
286 4 : int irq_activate(struct irq_desc *desc)
287 : {
288 4 : struct irq_data *d = irq_desc_get_irq_data(desc);
289 :
290 4 : if (!irqd_affinity_is_managed(d))
291 4 : return irq_domain_activate_irq(d, false);
292 : return 0;
293 : }
294 :
295 0 : int irq_activate_and_startup(struct irq_desc *desc, bool resend)
296 : {
297 0 : if (WARN_ON(irq_activate(desc)))
298 : return 0;
299 0 : return irq_startup(desc, resend, IRQ_START_FORCE);
300 : }
301 :
302 : static void __irq_disable(struct irq_desc *desc, bool mask);
303 :
304 0 : void irq_shutdown(struct irq_desc *desc)
305 : {
306 0 : if (irqd_is_started(&desc->irq_data)) {
307 0 : desc->depth = 1;
308 0 : if (desc->irq_data.chip->irq_shutdown) {
309 0 : desc->irq_data.chip->irq_shutdown(&desc->irq_data);
310 0 : irq_state_set_disabled(desc);
311 0 : irq_state_set_masked(desc);
312 : } else {
313 0 : __irq_disable(desc, true);
314 : }
315 0 : irq_state_clr_started(desc);
316 : }
317 0 : }
318 :
319 :
320 0 : void irq_shutdown_and_deactivate(struct irq_desc *desc)
321 : {
322 0 : irq_shutdown(desc);
323 : /*
324 : * This must be called even if the interrupt was never started up,
325 : * because the activation can happen before the interrupt is
326 : * available for request/startup. It has it's own state tracking so
327 : * it's safe to call it unconditionally.
328 : */
329 0 : irq_domain_deactivate_irq(&desc->irq_data);
330 0 : }
331 :
332 1 : void irq_enable(struct irq_desc *desc)
333 : {
334 1 : if (!irqd_irq_disabled(&desc->irq_data)) {
335 0 : unmask_irq(desc);
336 : } else {
337 1 : irq_state_clr_disabled(desc);
338 1 : if (desc->irq_data.chip->irq_enable) {
339 0 : desc->irq_data.chip->irq_enable(&desc->irq_data);
340 0 : irq_state_clr_masked(desc);
341 : } else {
342 1 : unmask_irq(desc);
343 : }
344 : }
345 1 : }
346 :
347 0 : static void __irq_disable(struct irq_desc *desc, bool mask)
348 : {
349 0 : if (irqd_irq_disabled(&desc->irq_data)) {
350 0 : if (mask)
351 0 : mask_irq(desc);
352 : } else {
353 0 : irq_state_set_disabled(desc);
354 0 : if (desc->irq_data.chip->irq_disable) {
355 0 : desc->irq_data.chip->irq_disable(&desc->irq_data);
356 0 : irq_state_set_masked(desc);
357 0 : } else if (mask) {
358 0 : mask_irq(desc);
359 : }
360 : }
361 0 : }
362 :
363 : /**
364 : * irq_disable - Mark interrupt disabled
365 : * @desc: irq descriptor which should be disabled
366 : *
367 : * If the chip does not implement the irq_disable callback, we
368 : * use a lazy disable approach. That means we mark the interrupt
369 : * disabled, but leave the hardware unmasked. That's an
370 : * optimization because we avoid the hardware access for the
371 : * common case where no interrupt happens after we marked it
372 : * disabled. If an interrupt happens, then the interrupt flow
373 : * handler masks the line at the hardware level and marks it
374 : * pending.
375 : *
376 : * If the interrupt chip does not implement the irq_disable callback,
377 : * a driver can disable the lazy approach for a particular irq line by
378 : * calling 'irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY)'. This can
379 : * be used for devices which cannot disable the interrupt at the
380 : * device level under certain circumstances and have to use
381 : * disable_irq[_nosync] instead.
382 : */
383 0 : void irq_disable(struct irq_desc *desc)
384 : {
385 0 : __irq_disable(desc, irq_settings_disable_unlazy(desc));
386 0 : }
387 :
388 0 : void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu)
389 : {
390 0 : if (desc->irq_data.chip->irq_enable)
391 0 : desc->irq_data.chip->irq_enable(&desc->irq_data);
392 : else
393 0 : desc->irq_data.chip->irq_unmask(&desc->irq_data);
394 0 : cpumask_set_cpu(cpu, desc->percpu_enabled);
395 0 : }
396 :
397 0 : void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu)
398 : {
399 0 : if (desc->irq_data.chip->irq_disable)
400 0 : desc->irq_data.chip->irq_disable(&desc->irq_data);
401 : else
402 0 : desc->irq_data.chip->irq_mask(&desc->irq_data);
403 0 : cpumask_clear_cpu(cpu, desc->percpu_enabled);
404 0 : }
405 :
406 0 : static inline void mask_ack_irq(struct irq_desc *desc)
407 : {
408 0 : if (desc->irq_data.chip->irq_mask_ack) {
409 0 : desc->irq_data.chip->irq_mask_ack(&desc->irq_data);
410 0 : irq_state_set_masked(desc);
411 : } else {
412 0 : mask_irq(desc);
413 0 : if (desc->irq_data.chip->irq_ack)
414 0 : desc->irq_data.chip->irq_ack(&desc->irq_data);
415 : }
416 0 : }
417 :
418 0 : void mask_irq(struct irq_desc *desc)
419 : {
420 0 : if (irqd_irq_masked(&desc->irq_data))
421 : return;
422 :
423 0 : if (desc->irq_data.chip->irq_mask) {
424 0 : desc->irq_data.chip->irq_mask(&desc->irq_data);
425 0 : irq_state_set_masked(desc);
426 : }
427 : }
428 :
429 1 : void unmask_irq(struct irq_desc *desc)
430 : {
431 1 : if (!irqd_irq_masked(&desc->irq_data))
432 : return;
433 :
434 1 : if (desc->irq_data.chip->irq_unmask) {
435 1 : desc->irq_data.chip->irq_unmask(&desc->irq_data);
436 1 : irq_state_clr_masked(desc);
437 : }
438 : }
439 :
440 0 : void unmask_threaded_irq(struct irq_desc *desc)
441 : {
442 0 : struct irq_chip *chip = desc->irq_data.chip;
443 :
444 0 : if (chip->flags & IRQCHIP_EOI_THREADED)
445 0 : chip->irq_eoi(&desc->irq_data);
446 :
447 0 : unmask_irq(desc);
448 0 : }
449 :
450 : /*
451 : * handle_nested_irq - Handle a nested irq from a irq thread
452 : * @irq: the interrupt number
453 : *
454 : * Handle interrupts which are nested into a threaded interrupt
455 : * handler. The handler function is called inside the calling
456 : * threads context.
457 : */
458 0 : void handle_nested_irq(unsigned int irq)
459 : {
460 0 : struct irq_desc *desc = irq_to_desc(irq);
461 0 : struct irqaction *action;
462 0 : irqreturn_t action_ret;
463 :
464 0 : might_sleep();
465 :
466 0 : raw_spin_lock_irq(&desc->lock);
467 :
468 0 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
469 :
470 0 : action = desc->action;
471 0 : if (unlikely(!action || irqd_irq_disabled(&desc->irq_data))) {
472 0 : desc->istate |= IRQS_PENDING;
473 0 : goto out_unlock;
474 : }
475 :
476 0 : kstat_incr_irqs_this_cpu(desc);
477 0 : irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);
478 0 : raw_spin_unlock_irq(&desc->lock);
479 :
480 0 : action_ret = IRQ_NONE;
481 0 : for_each_action_of_desc(desc, action)
482 0 : action_ret |= action->thread_fn(action->irq, action->dev_id);
483 :
484 0 : if (!noirqdebug)
485 0 : note_interrupt(desc, action_ret);
486 :
487 0 : raw_spin_lock_irq(&desc->lock);
488 0 : irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
489 :
490 0 : out_unlock:
491 0 : raw_spin_unlock_irq(&desc->lock);
492 0 : }
493 : EXPORT_SYMBOL_GPL(handle_nested_irq);
494 :
495 0 : static bool irq_check_poll(struct irq_desc *desc)
496 : {
497 0 : if (!(desc->istate & IRQS_POLL_INPROGRESS))
498 : return false;
499 0 : return irq_wait_for_poll(desc);
500 : }
501 :
502 3480 : static bool irq_may_run(struct irq_desc *desc)
503 : {
504 3480 : unsigned int mask = IRQD_IRQ_INPROGRESS | IRQD_WAKEUP_ARMED;
505 :
506 : /*
507 : * If the interrupt is not in progress and is not an armed
508 : * wakeup interrupt, proceed.
509 : */
510 3480 : if (!irqd_has_set(&desc->irq_data, mask))
511 : return true;
512 :
513 : /*
514 : * If the interrupt is an armed wakeup source, mark it pending
515 : * and suspended, disable it and notify the pm core about the
516 : * event.
517 : */
518 0 : if (irq_pm_check_wakeup(desc))
519 : return false;
520 :
521 : /*
522 : * Handle a potential concurrent poll on a different core.
523 : */
524 0 : return irq_check_poll(desc);
525 : }
526 :
527 : /**
528 : * handle_simple_irq - Simple and software-decoded IRQs.
529 : * @desc: the interrupt description structure for this irq
530 : *
531 : * Simple interrupts are either sent from a demultiplexing interrupt
532 : * handler or come from hardware, where no interrupt hardware control
533 : * is necessary.
534 : *
535 : * Note: The caller is expected to handle the ack, clear, mask and
536 : * unmask issues if necessary.
537 : */
538 0 : void handle_simple_irq(struct irq_desc *desc)
539 : {
540 0 : raw_spin_lock(&desc->lock);
541 :
542 0 : if (!irq_may_run(desc))
543 0 : goto out_unlock;
544 :
545 0 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
546 :
547 0 : if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
548 0 : desc->istate |= IRQS_PENDING;
549 0 : goto out_unlock;
550 : }
551 :
552 0 : kstat_incr_irqs_this_cpu(desc);
553 0 : handle_irq_event(desc);
554 :
555 0 : out_unlock:
556 0 : raw_spin_unlock(&desc->lock);
557 0 : }
558 : EXPORT_SYMBOL_GPL(handle_simple_irq);
559 :
560 : /**
561 : * handle_untracked_irq - Simple and software-decoded IRQs.
562 : * @desc: the interrupt description structure for this irq
563 : *
564 : * Untracked interrupts are sent from a demultiplexing interrupt
565 : * handler when the demultiplexer does not know which device it its
566 : * multiplexed irq domain generated the interrupt. IRQ's handled
567 : * through here are not subjected to stats tracking, randomness, or
568 : * spurious interrupt detection.
569 : *
570 : * Note: Like handle_simple_irq, the caller is expected to handle
571 : * the ack, clear, mask and unmask issues if necessary.
572 : */
573 0 : void handle_untracked_irq(struct irq_desc *desc)
574 : {
575 0 : unsigned int flags = 0;
576 :
577 0 : raw_spin_lock(&desc->lock);
578 :
579 0 : if (!irq_may_run(desc))
580 0 : goto out_unlock;
581 :
582 0 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
583 :
584 0 : if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
585 0 : desc->istate |= IRQS_PENDING;
586 0 : goto out_unlock;
587 : }
588 :
589 0 : desc->istate &= ~IRQS_PENDING;
590 0 : irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);
591 0 : raw_spin_unlock(&desc->lock);
592 :
593 0 : __handle_irq_event_percpu(desc, &flags);
594 :
595 0 : raw_spin_lock(&desc->lock);
596 0 : irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
597 :
598 0 : out_unlock:
599 0 : raw_spin_unlock(&desc->lock);
600 0 : }
601 : EXPORT_SYMBOL_GPL(handle_untracked_irq);
602 :
603 : /*
604 : * Called unconditionally from handle_level_irq() and only for oneshot
605 : * interrupts from handle_fasteoi_irq()
606 : */
607 0 : static void cond_unmask_irq(struct irq_desc *desc)
608 : {
609 : /*
610 : * We need to unmask in the following cases:
611 : * - Standard level irq (IRQF_ONESHOT is not set)
612 : * - Oneshot irq which did not wake the thread (caused by a
613 : * spurious interrupt or a primary handler handling it
614 : * completely).
615 : */
616 0 : if (!irqd_irq_disabled(&desc->irq_data) &&
617 0 : irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot)
618 0 : unmask_irq(desc);
619 0 : }
620 :
621 : /**
622 : * handle_level_irq - Level type irq handler
623 : * @desc: the interrupt description structure for this irq
624 : *
625 : * Level type interrupts are active as long as the hardware line has
626 : * the active level. This may require to mask the interrupt and unmask
627 : * it after the associated handler has acknowledged the device, so the
628 : * interrupt line is back to inactive.
629 : */
630 0 : void handle_level_irq(struct irq_desc *desc)
631 : {
632 0 : raw_spin_lock(&desc->lock);
633 0 : mask_ack_irq(desc);
634 :
635 0 : if (!irq_may_run(desc))
636 0 : goto out_unlock;
637 :
638 0 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
639 :
640 : /*
641 : * If its disabled or no action available
642 : * keep it masked and get out of here
643 : */
644 0 : if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
645 0 : desc->istate |= IRQS_PENDING;
646 0 : goto out_unlock;
647 : }
648 :
649 0 : kstat_incr_irqs_this_cpu(desc);
650 0 : handle_irq_event(desc);
651 :
652 0 : cond_unmask_irq(desc);
653 :
654 0 : out_unlock:
655 0 : raw_spin_unlock(&desc->lock);
656 0 : }
657 : EXPORT_SYMBOL_GPL(handle_level_irq);
658 :
659 0 : static void cond_unmask_eoi_irq(struct irq_desc *desc, struct irq_chip *chip)
660 : {
661 0 : if (!(desc->istate & IRQS_ONESHOT)) {
662 0 : chip->irq_eoi(&desc->irq_data);
663 0 : return;
664 : }
665 : /*
666 : * We need to unmask in the following cases:
667 : * - Oneshot irq which did not wake the thread (caused by a
668 : * spurious interrupt or a primary handler handling it
669 : * completely).
670 : */
671 0 : if (!irqd_irq_disabled(&desc->irq_data) &&
672 0 : irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot) {
673 0 : chip->irq_eoi(&desc->irq_data);
674 0 : unmask_irq(desc);
675 0 : } else if (!(chip->flags & IRQCHIP_EOI_THREADED)) {
676 0 : chip->irq_eoi(&desc->irq_data);
677 : }
678 : }
679 :
680 : /**
681 : * handle_fasteoi_irq - irq handler for transparent controllers
682 : * @desc: the interrupt description structure for this irq
683 : *
684 : * Only a single callback will be issued to the chip: an ->eoi()
685 : * call when the interrupt has been serviced. This enables support
686 : * for modern forms of interrupt handlers, which handle the flow
687 : * details in hardware, transparently.
688 : */
689 0 : void handle_fasteoi_irq(struct irq_desc *desc)
690 : {
691 0 : struct irq_chip *chip = desc->irq_data.chip;
692 :
693 0 : raw_spin_lock(&desc->lock);
694 :
695 0 : if (!irq_may_run(desc))
696 0 : goto out;
697 :
698 0 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
699 :
700 : /*
701 : * If its disabled or no action available
702 : * then mask it and get out of here:
703 : */
704 0 : if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
705 0 : desc->istate |= IRQS_PENDING;
706 0 : mask_irq(desc);
707 0 : goto out;
708 : }
709 :
710 0 : kstat_incr_irqs_this_cpu(desc);
711 0 : if (desc->istate & IRQS_ONESHOT)
712 0 : mask_irq(desc);
713 :
714 0 : handle_irq_event(desc);
715 :
716 0 : cond_unmask_eoi_irq(desc, chip);
717 :
718 0 : raw_spin_unlock(&desc->lock);
719 0 : return;
720 0 : out:
721 0 : if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
722 0 : chip->irq_eoi(&desc->irq_data);
723 0 : raw_spin_unlock(&desc->lock);
724 : }
725 : EXPORT_SYMBOL_GPL(handle_fasteoi_irq);
726 :
727 : /**
728 : * handle_fasteoi_nmi - irq handler for NMI interrupt lines
729 : * @desc: the interrupt description structure for this irq
730 : *
731 : * A simple NMI-safe handler, considering the restrictions
732 : * from request_nmi.
733 : *
734 : * Only a single callback will be issued to the chip: an ->eoi()
735 : * call when the interrupt has been serviced. This enables support
736 : * for modern forms of interrupt handlers, which handle the flow
737 : * details in hardware, transparently.
738 : */
739 0 : void handle_fasteoi_nmi(struct irq_desc *desc)
740 : {
741 0 : struct irq_chip *chip = irq_desc_get_chip(desc);
742 0 : struct irqaction *action = desc->action;
743 0 : unsigned int irq = irq_desc_get_irq(desc);
744 0 : irqreturn_t res;
745 :
746 0 : __kstat_incr_irqs_this_cpu(desc);
747 :
748 0 : trace_irq_handler_entry(irq, action);
749 : /*
750 : * NMIs cannot be shared, there is only one action.
751 : */
752 0 : res = action->handler(irq, action->dev_id);
753 0 : trace_irq_handler_exit(irq, action, res);
754 :
755 0 : if (chip->irq_eoi)
756 0 : chip->irq_eoi(&desc->irq_data);
757 0 : }
758 : EXPORT_SYMBOL_GPL(handle_fasteoi_nmi);
759 :
760 : /**
761 : * handle_edge_irq - edge type IRQ handler
762 : * @desc: the interrupt description structure for this irq
763 : *
764 : * Interrupt occures on the falling and/or rising edge of a hardware
765 : * signal. The occurrence is latched into the irq controller hardware
766 : * and must be acked in order to be reenabled. After the ack another
767 : * interrupt can happen on the same source even before the first one
768 : * is handled by the associated event handler. If this happens it
769 : * might be necessary to disable (mask) the interrupt depending on the
770 : * controller hardware. This requires to reenable the interrupt inside
771 : * of the loop which handles the interrupts which have arrived while
772 : * the handler was running. If all pending interrupts are handled, the
773 : * loop is left.
774 : */
775 3480 : void handle_edge_irq(struct irq_desc *desc)
776 : {
777 3480 : raw_spin_lock(&desc->lock);
778 :
779 3480 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
780 :
781 3480 : if (!irq_may_run(desc)) {
782 0 : desc->istate |= IRQS_PENDING;
783 0 : mask_ack_irq(desc);
784 0 : goto out_unlock;
785 : }
786 :
787 : /*
788 : * If its disabled or no action available then mask it and get
789 : * out of here.
790 : */
791 3480 : if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {
792 0 : desc->istate |= IRQS_PENDING;
793 0 : mask_ack_irq(desc);
794 0 : goto out_unlock;
795 : }
796 :
797 3480 : kstat_incr_irqs_this_cpu(desc);
798 :
799 : /* Start handling the irq */
800 3480 : desc->irq_data.chip->irq_ack(&desc->irq_data);
801 :
802 3480 : do {
803 3480 : if (unlikely(!desc->action)) {
804 0 : mask_irq(desc);
805 0 : goto out_unlock;
806 : }
807 :
808 : /*
809 : * When another irq arrived while we were handling
810 : * one, we could have masked the irq.
811 : * Renable it, if it was not disabled in meantime.
812 : */
813 3480 : if (unlikely(desc->istate & IRQS_PENDING)) {
814 0 : if (!irqd_irq_disabled(&desc->irq_data) &&
815 0 : irqd_irq_masked(&desc->irq_data))
816 0 : unmask_irq(desc);
817 : }
818 :
819 3480 : handle_irq_event(desc);
820 :
821 0 : } while ((desc->istate & IRQS_PENDING) &&
822 3480 : !irqd_irq_disabled(&desc->irq_data));
823 :
824 3480 : out_unlock:
825 3480 : raw_spin_unlock(&desc->lock);
826 3480 : }
827 : EXPORT_SYMBOL(handle_edge_irq);
828 :
829 : #ifdef CONFIG_IRQ_EDGE_EOI_HANDLER
830 : /**
831 : * handle_edge_eoi_irq - edge eoi type IRQ handler
832 : * @desc: the interrupt description structure for this irq
833 : *
834 : * Similar as the above handle_edge_irq, but using eoi and w/o the
835 : * mask/unmask logic.
836 : */
837 : void handle_edge_eoi_irq(struct irq_desc *desc)
838 : {
839 : struct irq_chip *chip = irq_desc_get_chip(desc);
840 :
841 : raw_spin_lock(&desc->lock);
842 :
843 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
844 :
845 : if (!irq_may_run(desc)) {
846 : desc->istate |= IRQS_PENDING;
847 : goto out_eoi;
848 : }
849 :
850 : /*
851 : * If its disabled or no action available then mask it and get
852 : * out of here.
853 : */
854 : if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {
855 : desc->istate |= IRQS_PENDING;
856 : goto out_eoi;
857 : }
858 :
859 : kstat_incr_irqs_this_cpu(desc);
860 :
861 : do {
862 : if (unlikely(!desc->action))
863 : goto out_eoi;
864 :
865 : handle_irq_event(desc);
866 :
867 : } while ((desc->istate & IRQS_PENDING) &&
868 : !irqd_irq_disabled(&desc->irq_data));
869 :
870 : out_eoi:
871 : chip->irq_eoi(&desc->irq_data);
872 : raw_spin_unlock(&desc->lock);
873 : }
874 : #endif
875 :
876 : /**
877 : * handle_percpu_irq - Per CPU local irq handler
878 : * @desc: the interrupt description structure for this irq
879 : *
880 : * Per CPU interrupts on SMP machines without locking requirements
881 : */
882 0 : void handle_percpu_irq(struct irq_desc *desc)
883 : {
884 0 : struct irq_chip *chip = irq_desc_get_chip(desc);
885 :
886 : /*
887 : * PER CPU interrupts are not serialized. Do not touch
888 : * desc->tot_count.
889 : */
890 0 : __kstat_incr_irqs_this_cpu(desc);
891 :
892 0 : if (chip->irq_ack)
893 0 : chip->irq_ack(&desc->irq_data);
894 :
895 0 : handle_irq_event_percpu(desc);
896 :
897 0 : if (chip->irq_eoi)
898 0 : chip->irq_eoi(&desc->irq_data);
899 0 : }
900 :
901 : /**
902 : * handle_percpu_devid_irq - Per CPU local irq handler with per cpu dev ids
903 : * @desc: the interrupt description structure for this irq
904 : *
905 : * Per CPU interrupts on SMP machines without locking requirements. Same as
906 : * handle_percpu_irq() above but with the following extras:
907 : *
908 : * action->percpu_dev_id is a pointer to percpu variables which
909 : * contain the real device id for the cpu on which this handler is
910 : * called
911 : */
912 0 : void handle_percpu_devid_irq(struct irq_desc *desc)
913 : {
914 0 : struct irq_chip *chip = irq_desc_get_chip(desc);
915 0 : struct irqaction *action = desc->action;
916 0 : unsigned int irq = irq_desc_get_irq(desc);
917 0 : irqreturn_t res;
918 :
919 : /*
920 : * PER CPU interrupts are not serialized. Do not touch
921 : * desc->tot_count.
922 : */
923 0 : __kstat_incr_irqs_this_cpu(desc);
924 :
925 0 : if (chip->irq_ack)
926 0 : chip->irq_ack(&desc->irq_data);
927 :
928 0 : if (likely(action)) {
929 0 : trace_irq_handler_entry(irq, action);
930 0 : res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
931 0 : trace_irq_handler_exit(irq, action, res);
932 : } else {
933 0 : unsigned int cpu = smp_processor_id();
934 0 : bool enabled = cpumask_test_cpu(cpu, desc->percpu_enabled);
935 :
936 0 : if (enabled)
937 0 : irq_percpu_disable(desc, cpu);
938 :
939 0 : pr_err_once("Spurious%s percpu IRQ%u on CPU%u\n",
940 : enabled ? " and unmasked" : "", irq, cpu);
941 : }
942 :
943 0 : if (chip->irq_eoi)
944 0 : chip->irq_eoi(&desc->irq_data);
945 0 : }
946 :
947 : /**
948 : * handle_percpu_devid_fasteoi_nmi - Per CPU local NMI handler with per cpu
949 : * dev ids
950 : * @desc: the interrupt description structure for this irq
951 : *
952 : * Similar to handle_fasteoi_nmi, but handling the dev_id cookie
953 : * as a percpu pointer.
954 : */
955 0 : void handle_percpu_devid_fasteoi_nmi(struct irq_desc *desc)
956 : {
957 0 : struct irq_chip *chip = irq_desc_get_chip(desc);
958 0 : struct irqaction *action = desc->action;
959 0 : unsigned int irq = irq_desc_get_irq(desc);
960 0 : irqreturn_t res;
961 :
962 0 : __kstat_incr_irqs_this_cpu(desc);
963 :
964 0 : trace_irq_handler_entry(irq, action);
965 0 : res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
966 0 : trace_irq_handler_exit(irq, action, res);
967 :
968 0 : if (chip->irq_eoi)
969 0 : chip->irq_eoi(&desc->irq_data);
970 0 : }
971 :
972 : static void
973 31 : __irq_do_set_handler(struct irq_desc *desc, irq_flow_handler_t handle,
974 : int is_chained, const char *name)
975 : {
976 31 : if (!handle) {
977 : handle = handle_bad_irq;
978 : } else {
979 31 : struct irq_data *irq_data = &desc->irq_data;
980 : #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
981 : /*
982 : * With hierarchical domains we might run into a
983 : * situation where the outermost chip is not yet set
984 : * up, but the inner chips are there. Instead of
985 : * bailing we install the handler, but obviously we
986 : * cannot enable/startup the interrupt at this point.
987 : */
988 31 : while (irq_data) {
989 31 : if (irq_data->chip != &no_irq_chip)
990 : break;
991 : /*
992 : * Bail out if the outer chip is not set up
993 : * and the interrupt supposed to be started
994 : * right away.
995 : */
996 0 : if (WARN_ON(is_chained))
997 : return;
998 : /* Try the parent */
999 0 : irq_data = irq_data->parent_data;
1000 : }
1001 : #endif
1002 62 : if (WARN_ON(!irq_data || irq_data->chip == &no_irq_chip))
1003 : return;
1004 : }
1005 :
1006 : /* Uninstall? */
1007 31 : if (handle == handle_bad_irq) {
1008 0 : if (desc->irq_data.chip != &no_irq_chip)
1009 0 : mask_ack_irq(desc);
1010 0 : irq_state_set_disabled(desc);
1011 0 : if (is_chained)
1012 0 : desc->action = NULL;
1013 0 : desc->depth = 1;
1014 : }
1015 31 : desc->handle_irq = handle;
1016 31 : desc->name = name;
1017 :
1018 31 : if (handle != handle_bad_irq && is_chained) {
1019 0 : unsigned int type = irqd_get_trigger_type(&desc->irq_data);
1020 :
1021 : /*
1022 : * We're about to start this interrupt immediately,
1023 : * hence the need to set the trigger configuration.
1024 : * But the .set_type callback may have overridden the
1025 : * flow handler, ignoring that we're dealing with a
1026 : * chained interrupt. Reset it immediately because we
1027 : * do know better.
1028 : */
1029 0 : if (type != IRQ_TYPE_NONE) {
1030 0 : __irq_set_trigger(desc, type);
1031 0 : desc->handle_irq = handle;
1032 : }
1033 :
1034 0 : irq_settings_set_noprobe(desc);
1035 0 : irq_settings_set_norequest(desc);
1036 0 : irq_settings_set_nothread(desc);
1037 0 : desc->action = &chained_action;
1038 0 : irq_activate_and_startup(desc, IRQ_RESEND);
1039 : }
1040 : }
1041 :
1042 : void
1043 31 : __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
1044 : const char *name)
1045 : {
1046 31 : unsigned long flags;
1047 31 : struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, 0);
1048 :
1049 31 : if (!desc)
1050 0 : return;
1051 :
1052 31 : __irq_do_set_handler(desc, handle, is_chained, name);
1053 31 : irq_put_desc_busunlock(desc, flags);
1054 : }
1055 : EXPORT_SYMBOL_GPL(__irq_set_handler);
1056 :
1057 : void
1058 0 : irq_set_chained_handler_and_data(unsigned int irq, irq_flow_handler_t handle,
1059 : void *data)
1060 : {
1061 0 : unsigned long flags;
1062 0 : struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, 0);
1063 :
1064 0 : if (!desc)
1065 0 : return;
1066 :
1067 0 : desc->irq_common_data.handler_data = data;
1068 0 : __irq_do_set_handler(desc, handle, 1, NULL);
1069 :
1070 0 : irq_put_desc_busunlock(desc, flags);
1071 : }
1072 : EXPORT_SYMBOL_GPL(irq_set_chained_handler_and_data);
1073 :
1074 : void
1075 16 : irq_set_chip_and_handler_name(unsigned int irq, struct irq_chip *chip,
1076 : irq_flow_handler_t handle, const char *name)
1077 : {
1078 16 : irq_set_chip(irq, chip);
1079 16 : __irq_set_handler(irq, handle, 0, name);
1080 16 : }
1081 : EXPORT_SYMBOL_GPL(irq_set_chip_and_handler_name);
1082 :
1083 30 : void irq_modify_status(unsigned int irq, unsigned long clr, unsigned long set)
1084 : {
1085 30 : unsigned long flags, trigger, tmp;
1086 30 : struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
1087 :
1088 30 : if (!desc)
1089 0 : return;
1090 :
1091 : /*
1092 : * Warn when a driver sets the no autoenable flag on an already
1093 : * active interrupt.
1094 : */
1095 60 : WARN_ON_ONCE(!desc->depth && (set & _IRQ_NOAUTOEN));
1096 :
1097 30 : irq_settings_clr_and_set(desc, clr, set);
1098 :
1099 30 : trigger = irqd_get_trigger_type(&desc->irq_data);
1100 :
1101 30 : irqd_clear(&desc->irq_data, IRQD_NO_BALANCING | IRQD_PER_CPU |
1102 : IRQD_TRIGGER_MASK | IRQD_LEVEL | IRQD_MOVE_PCNTXT);
1103 30 : if (irq_settings_has_no_balance_set(desc))
1104 0 : irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
1105 30 : if (irq_settings_is_per_cpu(desc))
1106 0 : irqd_set(&desc->irq_data, IRQD_PER_CPU);
1107 30 : if (irq_settings_can_move_pcntxt(desc))
1108 0 : irqd_set(&desc->irq_data, IRQD_MOVE_PCNTXT);
1109 30 : if (irq_settings_is_level(desc))
1110 0 : irqd_set(&desc->irq_data, IRQD_LEVEL);
1111 :
1112 30 : tmp = irq_settings_get_trigger_mask(desc);
1113 30 : if (tmp != IRQ_TYPE_NONE)
1114 0 : trigger = tmp;
1115 :
1116 30 : irqd_set(&desc->irq_data, trigger);
1117 :
1118 30 : irq_put_desc_unlock(desc, flags);
1119 : }
1120 : EXPORT_SYMBOL_GPL(irq_modify_status);
1121 :
1122 : /**
1123 : * irq_cpu_online - Invoke all irq_cpu_online functions.
1124 : *
1125 : * Iterate through all irqs and invoke the chip.irq_cpu_online()
1126 : * for each.
1127 : */
1128 0 : void irq_cpu_online(void)
1129 : {
1130 0 : struct irq_desc *desc;
1131 0 : struct irq_chip *chip;
1132 0 : unsigned long flags;
1133 0 : unsigned int irq;
1134 :
1135 0 : for_each_active_irq(irq) {
1136 0 : desc = irq_to_desc(irq);
1137 0 : if (!desc)
1138 0 : continue;
1139 :
1140 0 : raw_spin_lock_irqsave(&desc->lock, flags);
1141 :
1142 0 : chip = irq_data_get_irq_chip(&desc->irq_data);
1143 0 : if (chip && chip->irq_cpu_online &&
1144 0 : (!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
1145 0 : !irqd_irq_disabled(&desc->irq_data)))
1146 0 : chip->irq_cpu_online(&desc->irq_data);
1147 :
1148 0 : raw_spin_unlock_irqrestore(&desc->lock, flags);
1149 : }
1150 0 : }
1151 :
1152 : /**
1153 : * irq_cpu_offline - Invoke all irq_cpu_offline functions.
1154 : *
1155 : * Iterate through all irqs and invoke the chip.irq_cpu_offline()
1156 : * for each.
1157 : */
1158 0 : void irq_cpu_offline(void)
1159 : {
1160 0 : struct irq_desc *desc;
1161 0 : struct irq_chip *chip;
1162 0 : unsigned long flags;
1163 0 : unsigned int irq;
1164 :
1165 0 : for_each_active_irq(irq) {
1166 0 : desc = irq_to_desc(irq);
1167 0 : if (!desc)
1168 0 : continue;
1169 :
1170 0 : raw_spin_lock_irqsave(&desc->lock, flags);
1171 :
1172 0 : chip = irq_data_get_irq_chip(&desc->irq_data);
1173 0 : if (chip && chip->irq_cpu_offline &&
1174 0 : (!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
1175 0 : !irqd_irq_disabled(&desc->irq_data)))
1176 0 : chip->irq_cpu_offline(&desc->irq_data);
1177 :
1178 0 : raw_spin_unlock_irqrestore(&desc->lock, flags);
1179 : }
1180 0 : }
1181 :
1182 : #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
1183 :
1184 : #ifdef CONFIG_IRQ_FASTEOI_HIERARCHY_HANDLERS
1185 : /**
1186 : * handle_fasteoi_ack_irq - irq handler for edge hierarchy
1187 : * stacked on transparent controllers
1188 : *
1189 : * @desc: the interrupt description structure for this irq
1190 : *
1191 : * Like handle_fasteoi_irq(), but for use with hierarchy where
1192 : * the irq_chip also needs to have its ->irq_ack() function
1193 : * called.
1194 : */
1195 : void handle_fasteoi_ack_irq(struct irq_desc *desc)
1196 : {
1197 : struct irq_chip *chip = desc->irq_data.chip;
1198 :
1199 : raw_spin_lock(&desc->lock);
1200 :
1201 : if (!irq_may_run(desc))
1202 : goto out;
1203 :
1204 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
1205 :
1206 : /*
1207 : * If its disabled or no action available
1208 : * then mask it and get out of here:
1209 : */
1210 : if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
1211 : desc->istate |= IRQS_PENDING;
1212 : mask_irq(desc);
1213 : goto out;
1214 : }
1215 :
1216 : kstat_incr_irqs_this_cpu(desc);
1217 : if (desc->istate & IRQS_ONESHOT)
1218 : mask_irq(desc);
1219 :
1220 : /* Start handling the irq */
1221 : desc->irq_data.chip->irq_ack(&desc->irq_data);
1222 :
1223 : handle_irq_event(desc);
1224 :
1225 : cond_unmask_eoi_irq(desc, chip);
1226 :
1227 : raw_spin_unlock(&desc->lock);
1228 : return;
1229 : out:
1230 : if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
1231 : chip->irq_eoi(&desc->irq_data);
1232 : raw_spin_unlock(&desc->lock);
1233 : }
1234 : EXPORT_SYMBOL_GPL(handle_fasteoi_ack_irq);
1235 :
1236 : /**
1237 : * handle_fasteoi_mask_irq - irq handler for level hierarchy
1238 : * stacked on transparent controllers
1239 : *
1240 : * @desc: the interrupt description structure for this irq
1241 : *
1242 : * Like handle_fasteoi_irq(), but for use with hierarchy where
1243 : * the irq_chip also needs to have its ->irq_mask_ack() function
1244 : * called.
1245 : */
1246 : void handle_fasteoi_mask_irq(struct irq_desc *desc)
1247 : {
1248 : struct irq_chip *chip = desc->irq_data.chip;
1249 :
1250 : raw_spin_lock(&desc->lock);
1251 : mask_ack_irq(desc);
1252 :
1253 : if (!irq_may_run(desc))
1254 : goto out;
1255 :
1256 : desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
1257 :
1258 : /*
1259 : * If its disabled or no action available
1260 : * then mask it and get out of here:
1261 : */
1262 : if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
1263 : desc->istate |= IRQS_PENDING;
1264 : mask_irq(desc);
1265 : goto out;
1266 : }
1267 :
1268 : kstat_incr_irqs_this_cpu(desc);
1269 : if (desc->istate & IRQS_ONESHOT)
1270 : mask_irq(desc);
1271 :
1272 : handle_irq_event(desc);
1273 :
1274 : cond_unmask_eoi_irq(desc, chip);
1275 :
1276 : raw_spin_unlock(&desc->lock);
1277 : return;
1278 : out:
1279 : if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
1280 : chip->irq_eoi(&desc->irq_data);
1281 : raw_spin_unlock(&desc->lock);
1282 : }
1283 : EXPORT_SYMBOL_GPL(handle_fasteoi_mask_irq);
1284 :
1285 : #endif /* CONFIG_IRQ_FASTEOI_HIERARCHY_HANDLERS */
1286 :
1287 : /**
1288 : * irq_chip_set_parent_state - set the state of a parent interrupt.
1289 : *
1290 : * @data: Pointer to interrupt specific data
1291 : * @which: State to be restored (one of IRQCHIP_STATE_*)
1292 : * @val: Value corresponding to @which
1293 : *
1294 : * Conditional success, if the underlying irqchip does not implement it.
1295 : */
1296 0 : int irq_chip_set_parent_state(struct irq_data *data,
1297 : enum irqchip_irq_state which,
1298 : bool val)
1299 : {
1300 0 : data = data->parent_data;
1301 :
1302 0 : if (!data || !data->chip->irq_set_irqchip_state)
1303 : return 0;
1304 :
1305 0 : return data->chip->irq_set_irqchip_state(data, which, val);
1306 : }
1307 : EXPORT_SYMBOL_GPL(irq_chip_set_parent_state);
1308 :
1309 : /**
1310 : * irq_chip_get_parent_state - get the state of a parent interrupt.
1311 : *
1312 : * @data: Pointer to interrupt specific data
1313 : * @which: one of IRQCHIP_STATE_* the caller wants to know
1314 : * @state: a pointer to a boolean where the state is to be stored
1315 : *
1316 : * Conditional success, if the underlying irqchip does not implement it.
1317 : */
1318 0 : int irq_chip_get_parent_state(struct irq_data *data,
1319 : enum irqchip_irq_state which,
1320 : bool *state)
1321 : {
1322 0 : data = data->parent_data;
1323 :
1324 0 : if (!data || !data->chip->irq_get_irqchip_state)
1325 : return 0;
1326 :
1327 0 : return data->chip->irq_get_irqchip_state(data, which, state);
1328 : }
1329 : EXPORT_SYMBOL_GPL(irq_chip_get_parent_state);
1330 :
1331 : /**
1332 : * irq_chip_enable_parent - Enable the parent interrupt (defaults to unmask if
1333 : * NULL)
1334 : * @data: Pointer to interrupt specific data
1335 : */
1336 0 : void irq_chip_enable_parent(struct irq_data *data)
1337 : {
1338 0 : data = data->parent_data;
1339 0 : if (data->chip->irq_enable)
1340 0 : data->chip->irq_enable(data);
1341 : else
1342 0 : data->chip->irq_unmask(data);
1343 0 : }
1344 : EXPORT_SYMBOL_GPL(irq_chip_enable_parent);
1345 :
1346 : /**
1347 : * irq_chip_disable_parent - Disable the parent interrupt (defaults to mask if
1348 : * NULL)
1349 : * @data: Pointer to interrupt specific data
1350 : */
1351 0 : void irq_chip_disable_parent(struct irq_data *data)
1352 : {
1353 0 : data = data->parent_data;
1354 0 : if (data->chip->irq_disable)
1355 0 : data->chip->irq_disable(data);
1356 : else
1357 0 : data->chip->irq_mask(data);
1358 0 : }
1359 : EXPORT_SYMBOL_GPL(irq_chip_disable_parent);
1360 :
1361 : /**
1362 : * irq_chip_ack_parent - Acknowledge the parent interrupt
1363 : * @data: Pointer to interrupt specific data
1364 : */
1365 3480 : void irq_chip_ack_parent(struct irq_data *data)
1366 : {
1367 3480 : data = data->parent_data;
1368 3480 : data->chip->irq_ack(data);
1369 3480 : }
1370 : EXPORT_SYMBOL_GPL(irq_chip_ack_parent);
1371 :
1372 : /**
1373 : * irq_chip_mask_parent - Mask the parent interrupt
1374 : * @data: Pointer to interrupt specific data
1375 : */
1376 0 : void irq_chip_mask_parent(struct irq_data *data)
1377 : {
1378 0 : data = data->parent_data;
1379 0 : data->chip->irq_mask(data);
1380 0 : }
1381 : EXPORT_SYMBOL_GPL(irq_chip_mask_parent);
1382 :
1383 : /**
1384 : * irq_chip_mask_ack_parent - Mask and acknowledge the parent interrupt
1385 : * @data: Pointer to interrupt specific data
1386 : */
1387 0 : void irq_chip_mask_ack_parent(struct irq_data *data)
1388 : {
1389 0 : data = data->parent_data;
1390 0 : data->chip->irq_mask_ack(data);
1391 0 : }
1392 : EXPORT_SYMBOL_GPL(irq_chip_mask_ack_parent);
1393 :
1394 : /**
1395 : * irq_chip_unmask_parent - Unmask the parent interrupt
1396 : * @data: Pointer to interrupt specific data
1397 : */
1398 0 : void irq_chip_unmask_parent(struct irq_data *data)
1399 : {
1400 0 : data = data->parent_data;
1401 0 : data->chip->irq_unmask(data);
1402 0 : }
1403 : EXPORT_SYMBOL_GPL(irq_chip_unmask_parent);
1404 :
1405 : /**
1406 : * irq_chip_eoi_parent - Invoke EOI on the parent interrupt
1407 : * @data: Pointer to interrupt specific data
1408 : */
1409 0 : void irq_chip_eoi_parent(struct irq_data *data)
1410 : {
1411 0 : data = data->parent_data;
1412 0 : data->chip->irq_eoi(data);
1413 0 : }
1414 : EXPORT_SYMBOL_GPL(irq_chip_eoi_parent);
1415 :
1416 : /**
1417 : * irq_chip_set_affinity_parent - Set affinity on the parent interrupt
1418 : * @data: Pointer to interrupt specific data
1419 : * @dest: The affinity mask to set
1420 : * @force: Flag to enforce setting (disable online checks)
1421 : *
1422 : * Conditinal, as the underlying parent chip might not implement it.
1423 : */
1424 0 : int irq_chip_set_affinity_parent(struct irq_data *data,
1425 : const struct cpumask *dest, bool force)
1426 : {
1427 0 : data = data->parent_data;
1428 0 : if (data->chip->irq_set_affinity)
1429 0 : return data->chip->irq_set_affinity(data, dest, force);
1430 :
1431 : return -ENOSYS;
1432 : }
1433 : EXPORT_SYMBOL_GPL(irq_chip_set_affinity_parent);
1434 :
1435 : /**
1436 : * irq_chip_set_type_parent - Set IRQ type on the parent interrupt
1437 : * @data: Pointer to interrupt specific data
1438 : * @type: IRQ_TYPE_{LEVEL,EDGE}_* value - see include/linux/irq.h
1439 : *
1440 : * Conditional, as the underlying parent chip might not implement it.
1441 : */
1442 0 : int irq_chip_set_type_parent(struct irq_data *data, unsigned int type)
1443 : {
1444 0 : data = data->parent_data;
1445 :
1446 0 : if (data->chip->irq_set_type)
1447 0 : return data->chip->irq_set_type(data, type);
1448 :
1449 : return -ENOSYS;
1450 : }
1451 : EXPORT_SYMBOL_GPL(irq_chip_set_type_parent);
1452 :
1453 : /**
1454 : * irq_chip_retrigger_hierarchy - Retrigger an interrupt in hardware
1455 : * @data: Pointer to interrupt specific data
1456 : *
1457 : * Iterate through the domain hierarchy of the interrupt and check
1458 : * whether a hw retrigger function exists. If yes, invoke it.
1459 : */
1460 0 : int irq_chip_retrigger_hierarchy(struct irq_data *data)
1461 : {
1462 0 : for (data = data->parent_data; data; data = data->parent_data)
1463 0 : if (data->chip && data->chip->irq_retrigger)
1464 0 : return data->chip->irq_retrigger(data);
1465 :
1466 : return 0;
1467 : }
1468 : EXPORT_SYMBOL_GPL(irq_chip_retrigger_hierarchy);
1469 :
1470 : /**
1471 : * irq_chip_set_vcpu_affinity_parent - Set vcpu affinity on the parent interrupt
1472 : * @data: Pointer to interrupt specific data
1473 : * @vcpu_info: The vcpu affinity information
1474 : */
1475 0 : int irq_chip_set_vcpu_affinity_parent(struct irq_data *data, void *vcpu_info)
1476 : {
1477 0 : data = data->parent_data;
1478 0 : if (data->chip->irq_set_vcpu_affinity)
1479 0 : return data->chip->irq_set_vcpu_affinity(data, vcpu_info);
1480 :
1481 : return -ENOSYS;
1482 : }
1483 : EXPORT_SYMBOL_GPL(irq_chip_set_vcpu_affinity_parent);
1484 : /**
1485 : * irq_chip_set_wake_parent - Set/reset wake-up on the parent interrupt
1486 : * @data: Pointer to interrupt specific data
1487 : * @on: Whether to set or reset the wake-up capability of this irq
1488 : *
1489 : * Conditional, as the underlying parent chip might not implement it.
1490 : */
1491 0 : int irq_chip_set_wake_parent(struct irq_data *data, unsigned int on)
1492 : {
1493 0 : data = data->parent_data;
1494 :
1495 0 : if (data->chip->flags & IRQCHIP_SKIP_SET_WAKE)
1496 : return 0;
1497 :
1498 0 : if (data->chip->irq_set_wake)
1499 0 : return data->chip->irq_set_wake(data, on);
1500 :
1501 : return -ENOSYS;
1502 : }
1503 : EXPORT_SYMBOL_GPL(irq_chip_set_wake_parent);
1504 :
1505 : /**
1506 : * irq_chip_request_resources_parent - Request resources on the parent interrupt
1507 : * @data: Pointer to interrupt specific data
1508 : */
1509 0 : int irq_chip_request_resources_parent(struct irq_data *data)
1510 : {
1511 0 : data = data->parent_data;
1512 :
1513 0 : if (data->chip->irq_request_resources)
1514 0 : return data->chip->irq_request_resources(data);
1515 :
1516 : return -ENOSYS;
1517 : }
1518 : EXPORT_SYMBOL_GPL(irq_chip_request_resources_parent);
1519 :
1520 : /**
1521 : * irq_chip_release_resources_parent - Release resources on the parent interrupt
1522 : * @data: Pointer to interrupt specific data
1523 : */
1524 0 : void irq_chip_release_resources_parent(struct irq_data *data)
1525 : {
1526 0 : data = data->parent_data;
1527 0 : if (data->chip->irq_release_resources)
1528 0 : data->chip->irq_release_resources(data);
1529 0 : }
1530 : EXPORT_SYMBOL_GPL(irq_chip_release_resources_parent);
1531 : #endif
1532 :
1533 : /**
1534 : * irq_chip_compose_msi_msg - Componse msi message for a irq chip
1535 : * @data: Pointer to interrupt specific data
1536 : * @msg: Pointer to the MSI message
1537 : *
1538 : * For hierarchical domains we find the first chip in the hierarchy
1539 : * which implements the irq_compose_msi_msg callback. For non
1540 : * hierarchical we use the top level chip.
1541 : */
1542 6 : int irq_chip_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
1543 : {
1544 6 : struct irq_data *pos;
1545 :
1546 18 : for (pos = NULL; !pos && data; data = irqd_get_parent_data(data)) {
1547 12 : if (data->chip && data->chip->irq_compose_msi_msg)
1548 6 : pos = data;
1549 : }
1550 :
1551 6 : if (!pos)
1552 : return -ENOSYS;
1553 :
1554 6 : pos->chip->irq_compose_msi_msg(pos, msg);
1555 6 : return 0;
1556 : }
1557 :
1558 : /**
1559 : * irq_chip_pm_get - Enable power for an IRQ chip
1560 : * @data: Pointer to interrupt specific data
1561 : *
1562 : * Enable the power to the IRQ chip referenced by the interrupt data
1563 : * structure.
1564 : */
1565 4 : int irq_chip_pm_get(struct irq_data *data)
1566 : {
1567 4 : int retval;
1568 :
1569 4 : if (IS_ENABLED(CONFIG_PM) && data->chip->parent_device) {
1570 : retval = pm_runtime_get_sync(data->chip->parent_device);
1571 : if (retval < 0) {
1572 : pm_runtime_put_noidle(data->chip->parent_device);
1573 : return retval;
1574 : }
1575 : }
1576 :
1577 4 : return 0;
1578 : }
1579 :
1580 : /**
1581 : * irq_chip_pm_put - Disable power for an IRQ chip
1582 : * @data: Pointer to interrupt specific data
1583 : *
1584 : * Disable the power to the IRQ chip referenced by the interrupt data
1585 : * structure, belongs. Note that power will only be disabled, once this
1586 : * function has been called for all IRQs that have called irq_chip_pm_get().
1587 : */
1588 0 : int irq_chip_pm_put(struct irq_data *data)
1589 : {
1590 0 : int retval = 0;
1591 :
1592 0 : if (IS_ENABLED(CONFIG_PM) && data->chip->parent_device)
1593 : retval = pm_runtime_put(data->chip->parent_device);
1594 :
1595 0 : return (retval < 0) ? retval : 0;
1596 : }
|
