| 1 | |
| 2 | |
| 3 | * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
|
| 4 | * Copyright (C) 2005-2006 Thomas Gleixner
|
| 5 | *
|
| 6 | * This file contains driver APIs to the irq subsystem.
|
| 7 |
|
| 8 |
|
| 9 | #define pr_fmt(fmt) "genirq: " fmt
|
| 10 |
|
| 11 | #include <linux/irq.h>
|
| 12 | #include <linux/kthread.h>
|
| 13 | #include <linux/module.h>
|
| 14 | #include <linux/random.h>
|
| 15 | #include <linux/interrupt.h>
|
| 16 | #include <linux/irqdomain.h>
|
| 17 | #include <linux/slab.h>
|
| 18 | #include <linux/sched.h>
|
| 19 | #include <linux/sched/rt.h>
|
| 20 | #include <linux/sched/task.h>
|
| 21 | #include <linux/sched/isolation.h>
|
| 22 | #include <uapi/linux/sched/types.h>
|
| 23 | #include <linux/task_work.h>
|
| 24 |
|
| 25 | #include "internals.h"
|
| 26 |
|
| 27 | #if defined(CONFIG_IRQ_FORCED_THREADING) && !defined(CONFIG_PREEMPT_RT)
|
| 28 | DEFINE_STATIC_KEY_FALSE(force_irqthreads_key);
|
| 29 |
|
| 30 | static int __init setup_forced_irqthreads(char *arg)
|
| 31 | {
|
| 32 | static_branch_enable(&force_irqthreads_key);
|
| 33 | return 0;
|
| 34 | }
|
| 35 | early_param("threadirqs", setup_forced_irqthreads);
|
| 36 | #endif
|
| 37 |
|
| 38 | #ifdef CONFIG_SMP
|
| 39 | static inline void synchronize_irqwork(struct irq_desc *desc)
|
| 40 | {
|
| 41 |
|
| 42 | irq_work_sync(&desc->redirect.work);
|
| 43 | }
|
| 44 | #else
|
| 45 | static inline void synchronize_irqwork(struct irq_desc *desc) { }
|
| 46 | #endif
|
| 47 |
|
| 48 | static int __irq_get_irqchip_state(struct irq_data *d, enum irqchip_irq_state which, bool *state);
|
| 49 |
|
| 50 | static void __synchronize_hardirq(struct irq_desc *desc, bool sync_chip)
|
| 51 | {
|
| 52 | struct irq_data *irqd = irq_desc_get_irq_data(desc);
|
| 53 | bool inprogress;
|
| 54 |
|
| 55 | do {
|
| 56 | |
| 57 | * Wait until we're out of the critical section. This might
|
| 58 | * give the wrong answer due to the lack of memory barriers.
|
| 59 |
|
| 60 | while (irqd_irq_inprogress(&desc->irq_data))
|
| 61 | cpu_relax();
|
| 62 |
|
| 63 |
|
| 64 | guard(raw_spinlock_irqsave)(&desc->lock);
|
| 65 | inprogress = irqd_irq_inprogress(&desc->irq_data);
|
| 66 |
|
| 67 | |
| 68 | * If requested and supported, check at the chip whether it
|
| 69 | * is in flight at the hardware level, i.e. already pending
|
| 70 | * in a CPU and waiting for service and acknowledge.
|
| 71 |
|
| 72 | if (!inprogress && sync_chip) {
|
| 73 | |
| 74 | * Ignore the return code. inprogress is only updated
|
| 75 | * when the chip supports it.
|
| 76 |
|
| 77 | __irq_get_irqchip_state(irqd, IRQCHIP_STATE_ACTIVE,
|
| 78 | &inprogress);
|
| 79 | }
|
| 80 |
|
| 81 | } while (inprogress);
|
| 82 | }
|
| 83 |
|
| 84 | |
| 85 | * synchronize_hardirq - wait for pending hard IRQ handlers (on other CPUs)
|
| 86 | * @irq: interrupt number to wait for
|
| 87 | *
|
| 88 | * This function waits for any pending hard IRQ handlers for this interrupt
|
| 89 | * to complete before returning. If you use this function while holding a
|
| 90 | * resource the IRQ handler may need you will deadlock. It does not take
|
| 91 | * associated threaded handlers into account.
|
| 92 | *
|
| 93 | * Do not use this for shutdown scenarios where you must be sure that all
|
| 94 | * parts (hardirq and threaded handler) have completed.
|
| 95 | *
|
| 96 | * Returns: false if a threaded handler is active.
|
| 97 | *
|
| 98 | * This function may be called - with care - from IRQ context.
|
| 99 | *
|
| 100 | * It does not check whether there is an interrupt in flight at the
|
| 101 | * hardware level, but not serviced yet, as this might deadlock when called
|
| 102 | * with interrupts disabled and the target CPU of the interrupt is the
|
| 103 | * current CPU.
|
| 104 |
|
| 105 | bool synchronize_hardirq(unsigned int irq)
|
| 106 | {
|
| 107 | struct irq_desc *desc = irq_to_desc(irq);
|
| 108 |
|
| 109 | if (desc) {
|
| 110 | __synchronize_hardirq(desc, false);
|
| 111 | return !atomic_read(&desc->threads_active);
|
| 112 | }
|
| 113 |
|
| 114 | return true;
|
| 115 | }
|
| 116 | EXPORT_SYMBOL(synchronize_hardirq);
|
| 117 |
|
| 118 | static void __synchronize_irq(struct irq_desc *desc)
|
| 119 | {
|
| 120 | synchronize_irqwork(desc);
|
| 121 | __synchronize_hardirq(desc, true);
|
| 122 |
|
| 123 | |
| 124 | * We made sure that no hardirq handler is running. Now verify that no
|
| 125 | * threaded handlers are active.
|
| 126 |
|
| 127 | wait_event(desc->wait_for_threads, !atomic_read(&desc->threads_active));
|
| 128 | }
|
| 129 |
|
| 130 | |
| 131 | * synchronize_irq - wait for pending IRQ handlers (on other CPUs)
|
| 132 | * @irq: interrupt number to wait for
|
| 133 | *
|
| 134 | * This function waits for any pending IRQ handlers for this interrupt to
|
| 135 | * complete before returning. If you use this function while holding a
|
| 136 | * resource the IRQ handler may need you will deadlock.
|
| 137 | *
|
| 138 | * Can only be called from preemptible code as it might sleep when
|
| 139 | * an interrupt thread is associated to @irq.
|
| 140 | *
|
| 141 | * It optionally makes sure (when the irq chip supports that method)
|
| 142 | * that the interrupt is not pending in any CPU and waiting for
|
| 143 | * service.
|
| 144 |
|
| 145 | void synchronize_irq(unsigned int irq)
|
| 146 | {
|
| 147 | struct irq_desc *desc = irq_to_desc(irq);
|
| 148 |
|
| 149 | if (desc)
|
| 150 | __synchronize_irq(desc);
|
| 151 | }
|
| 152 | EXPORT_SYMBOL(synchronize_irq);
|
| 153 |
|
| 154 | #ifdef CONFIG_SMP
|
| 155 | cpumask_var_t irq_default_affinity;
|
| 156 |
|
| 157 | static bool __irq_can_set_affinity(struct irq_desc *desc)
|
| 158 | {
|
| 159 | if (!desc || !irqd_can_balance(&desc->irq_data) ||
|
| 160 | !desc->irq_data.chip || !desc->irq_data.chip->irq_set_affinity)
|
| 161 | return false;
|
| 162 | return true;
|
| 163 | }
|
| 164 |
|
| 165 | |
| 166 | * irq_can_set_affinity - Check if the affinity of a given irq can be set
|
| 167 | * @irq: Interrupt to check
|
| 168 | *
|
| 169 |
|
| 170 | int irq_can_set_affinity(unsigned int irq)
|
| 171 | {
|
| 172 | return __irq_can_set_affinity(irq_to_desc(irq));
|
| 173 | }
|
| 174 | EXPORT_SYMBOL_GPL(irq_can_set_affinity);
|
| 175 |
|
| 176 | |
| 177 | * irq_can_set_affinity_usr - Check if affinity of a irq can be set from user space
|
| 178 | * @irq: Interrupt to check
|
| 179 | *
|
| 180 | * Like irq_can_set_affinity() above, but additionally checks for the
|
| 181 | * AFFINITY_MANAGED flag.
|
| 182 |
|
| 183 | bool irq_can_set_affinity_usr(unsigned int irq)
|
| 184 | {
|
| 185 | struct irq_desc *desc = irq_to_desc(irq);
|
| 186 |
|
| 187 | return __irq_can_set_affinity(desc) &&
|
| 188 | !irqd_affinity_is_managed(&desc->irq_data);
|
| 189 | }
|
| 190 |
|
| 191 | |
| 192 | * irq_set_thread_affinity - Notify irq threads to adjust affinity
|
| 193 | * @desc: irq descriptor which has affinity changed
|
| 194 | *
|
| 195 | * Just set IRQTF_AFFINITY and delegate the affinity setting to the
|
| 196 | * interrupt thread itself. We can not call set_cpus_allowed_ptr() here as
|
| 197 | * we hold desc->lock and this code can be called from hard interrupt
|
| 198 | * context.
|
| 199 |
|
| 200 | static void irq_set_thread_affinity(struct irq_desc *desc)
|
| 201 | {
|
| 202 | struct irqaction *action;
|
| 203 |
|
| 204 | for_each_action_of_desc(desc, action) {
|
| 205 | if (action->thread) {
|
| 206 | set_bit(IRQTF_AFFINITY, &action->thread_flags);
|
| 207 | wake_up_process(action->thread);
|
| 208 | }
|
| 209 | if (action->secondary && action->secondary->thread) {
|
| 210 | set_bit(IRQTF_AFFINITY, &action->secondary->thread_flags);
|
| 211 | wake_up_process(action->secondary->thread);
|
| 212 | }
|
| 213 | }
|
| 214 | }
|
| 215 |
|
| 216 | #ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
|
| 217 | static void irq_validate_effective_affinity(struct irq_data *data)
|
| 218 | {
|
| 219 | const struct cpumask *m = irq_data_get_effective_affinity_mask(data);
|
| 220 | struct irq_chip *chip = irq_data_get_irq_chip(data);
|
| 221 |
|
| 222 | if (!cpumask_empty(m))
|
| 223 | return;
|
| 224 | pr_warn_once("irq_chip %s did not update eff. affinity mask of irq %u\n",
|
| 225 | chip->name, data->irq);
|
| 226 | }
|
| 227 | #else
|
| 228 | static inline void irq_validate_effective_affinity(struct irq_data *data) { }
|
| 229 | #endif
|
| 230 |
|
| 231 | static DEFINE_PER_CPU(struct cpumask, __tmp_mask);
|
| 232 |
|
| 233 | int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force)
|
| 234 | {
|
| 235 | struct cpumask *tmp_mask = this_cpu_ptr(&__tmp_mask);
|
| 236 | struct irq_desc *desc = irq_data_to_desc(data);
|
| 237 | struct irq_chip *chip = irq_data_get_irq_chip(data);
|
| 238 | const struct cpumask *prog_mask;
|
| 239 | int ret;
|
| 240 |
|
| 241 | if (!chip || !chip->irq_set_affinity)
|
| 242 | return -EINVAL;
|
| 243 |
|
| 244 | |
| 245 | * If this is a managed interrupt and housekeeping is enabled on
|
| 246 | * it check whether the requested affinity mask intersects with
|
| 247 | * a housekeeping CPU. If so, then remove the isolated CPUs from
|
| 248 | * the mask and just keep the housekeeping CPU(s). This prevents
|
| 249 | * the affinity setter from routing the interrupt to an isolated
|
| 250 | * CPU to avoid that I/O submitted from a housekeeping CPU causes
|
| 251 | * interrupts on an isolated one.
|
| 252 | *
|
| 253 | * If the masks do not intersect or include online CPU(s) then
|
| 254 | * keep the requested mask. The isolated target CPUs are only
|
| 255 | * receiving interrupts when the I/O operation was submitted
|
| 256 | * directly from them.
|
| 257 | *
|
| 258 | * If all housekeeping CPUs in the affinity mask are offline, the
|
| 259 | * interrupt will be migrated by the CPU hotplug code once a
|
| 260 | * housekeeping CPU which belongs to the affinity mask comes
|
| 261 | * online.
|
| 262 |
|
| 263 | if (irqd_affinity_is_managed(data) &&
|
| 264 | housekeeping_enabled(HK_TYPE_MANAGED_IRQ)) {
|
| 265 | const struct cpumask *hk_mask;
|
| 266 |
|
| 267 | hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);
|
| 268 |
|
| 269 | cpumask_and(tmp_mask, mask, hk_mask);
|
| 270 | if (!cpumask_intersects(tmp_mask, cpu_online_mask))
|
| 271 | prog_mask = mask;
|
| 272 | else
|
| 273 | prog_mask = tmp_mask;
|
| 274 | } else {
|
| 275 | prog_mask = mask;
|
| 276 | }
|
| 277 |
|
| 278 | |
| 279 | * Make sure we only provide online CPUs to the irqchip,
|
| 280 | * unless we are being asked to force the affinity (in which
|
| 281 | * case we do as we are told).
|
| 282 |
|
| 283 | cpumask_and(tmp_mask, prog_mask, cpu_online_mask);
|
| 284 | if (!force && !cpumask_empty(tmp_mask))
|
| 285 | ret = chip->irq_set_affinity(data, tmp_mask, force);
|
| 286 | else if (force)
|
| 287 | ret = chip->irq_set_affinity(data, mask, force);
|
| 288 | else
|
| 289 | ret = -EINVAL;
|
| 290 |
|
| 291 | switch (ret) {
|
| 292 | case IRQ_SET_MASK_OK:
|
| 293 | case IRQ_SET_MASK_OK_DONE:
|
| 294 | cpumask_copy(desc->irq_common_data.affinity, mask);
|
| 295 | fallthrough;
|
| 296 | case IRQ_SET_MASK_OK_NOCOPY:
|
| 297 | irq_validate_effective_affinity(data);
|
| 298 | irq_set_thread_affinity(desc);
|
| 299 | ret = 0;
|
| 300 | }
|
| 301 |
|
| 302 | return ret;
|
| 303 | }
|
| 304 |
|
| 305 | #ifdef CONFIG_GENERIC_PENDING_IRQ
|
| 306 | static inline int irq_set_affinity_pending(struct irq_data *data,
|
| 307 | const struct cpumask *dest)
|
| 308 | {
|
| 309 | struct irq_desc *desc = irq_data_to_desc(data);
|
| 310 |
|
| 311 | irqd_set_move_pending(data);
|
| 312 | irq_copy_pending(desc, dest);
|
| 313 | return 0;
|
| 314 | }
|
| 315 | #else
|
| 316 | static inline int irq_set_affinity_pending(struct irq_data *data,
|
| 317 | const struct cpumask *dest)
|
| 318 | {
|
| 319 | return -EBUSY;
|
| 320 | }
|
| 321 | #endif
|
| 322 |
|
| 323 | static int irq_try_set_affinity(struct irq_data *data,
|
| 324 | const struct cpumask *dest, bool force)
|
| 325 | {
|
| 326 | int ret = irq_do_set_affinity(data, dest, force);
|
| 327 |
|
| 328 | |
| 329 | * In case that the underlying vector management is busy and the
|
| 330 | * architecture supports the generic pending mechanism then utilize
|
| 331 | * this to avoid returning an error to user space.
|
| 332 |
|
| 333 | if (ret == -EBUSY && !force)
|
| 334 | ret = irq_set_affinity_pending(data, dest);
|
| 335 | return ret;
|
| 336 | }
|
| 337 |
|
| 338 | static bool irq_set_affinity_deactivated(struct irq_data *data,
|
| 339 | const struct cpumask *mask)
|
| 340 | {
|
| 341 | struct irq_desc *desc = irq_data_to_desc(data);
|
| 342 |
|
| 343 | |
| 344 | * Handle irq chips which can handle affinity only in activated
|
| 345 | * state correctly
|
| 346 | *
|
| 347 | * If the interrupt is not yet activated, just store the affinity
|
| 348 | * mask and do not call the chip driver at all. On activation the
|
| 349 | * driver has to make sure anyway that the interrupt is in a
|
| 350 | * usable state so startup works.
|
| 351 |
|
| 352 | if (!IS_ENABLED(CONFIG_IRQ_DOMAIN_HIERARCHY) ||
|
| 353 | irqd_is_activated(data) || !irqd_affinity_on_activate(data))
|
| 354 | return false;
|
| 355 |
|
| 356 | cpumask_copy(desc->irq_common_data.affinity, mask);
|
| 357 | irq_data_update_effective_affinity(data, mask);
|
| 358 | irqd_set(data, IRQD_AFFINITY_SET);
|
| 359 | return true;
|
| 360 | }
|
| 361 |
|
| 362 | |
| 363 | * irq_affinity_schedule_notify_work - Schedule work to notify about affinity change
|
| 364 | * @desc: Interrupt descriptor whose affinity changed
|
| 365 |
|
| 366 | void irq_affinity_schedule_notify_work(struct irq_desc *desc)
|
| 367 | {
|
| 368 | lockdep_assert_held(&desc->lock);
|
| 369 |
|
| 370 | kref_get(&desc->affinity_notify->kref);
|
| 371 | if (!schedule_work(&desc->affinity_notify->work)) {
|
| 372 |
|
| 373 | kref_put(&desc->affinity_notify->kref, desc->affinity_notify->release);
|
| 374 | }
|
| 375 | }
|
| 376 |
|
| 377 | int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask,
|
| 378 | bool force)
|
| 379 | {
|
| 380 | struct irq_chip *chip = irq_data_get_irq_chip(data);
|
| 381 | struct irq_desc *desc = irq_data_to_desc(data);
|
| 382 | int ret = 0;
|
| 383 |
|
| 384 | if (!chip || !chip->irq_set_affinity)
|
| 385 | return -EINVAL;
|
| 386 |
|
| 387 | if (irq_set_affinity_deactivated(data, mask))
|
| 388 | return 0;
|
| 389 |
|
| 390 | if (irq_can_move_pcntxt(data) && !irqd_is_setaffinity_pending(data)) {
|
| 391 | ret = irq_try_set_affinity(data, mask, force);
|
| 392 | } else {
|
| 393 | irqd_set_move_pending(data);
|
| 394 | irq_copy_pending(desc, mask);
|
| 395 | }
|
| 396 |
|
| 397 | if (desc->affinity_notify)
|
| 398 | irq_affinity_schedule_notify_work(desc);
|
| 399 |
|
| 400 | irqd_set(data, IRQD_AFFINITY_SET);
|
| 401 |
|
| 402 | return ret;
|
| 403 | }
|
| 404 |
|
| 405 | |
| 406 | * irq_update_affinity_desc - Update affinity management for an interrupt
|
| 407 | * @irq: The interrupt number to update
|
| 408 | * @affinity: Pointer to the affinity descriptor
|
| 409 | *
|
| 410 | * This interface can be used to configure the affinity management of
|
| 411 | * interrupts which have been allocated already.
|
| 412 | *
|
| 413 | * There are certain limitations on when it may be used - attempts to use it
|
| 414 | * for when the kernel is configured for generic IRQ reservation mode (in
|
| 415 | * config GENERIC_IRQ_RESERVATION_MODE) will fail, as it may conflict with
|
| 416 | * managed/non-managed interrupt accounting. In addition, attempts to use it on
|
| 417 | * an interrupt which is already started or which has already been configured
|
| 418 | * as managed will also fail, as these mean invalid init state or double init.
|
| 419 |
|
| 420 | int irq_update_affinity_desc(unsigned int irq, struct irq_affinity_desc *affinity)
|
| 421 | {
|
| 422 | |
| 423 | * Supporting this with the reservation scheme used by x86 needs
|
| 424 | * some more thought. Fail it for now.
|
| 425 |
|
| 426 | if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE))
|
| 427 | return -EOPNOTSUPP;
|
| 428 |
|
| 429 | scoped_irqdesc_get_and_buslock(irq, 0) {
|
| 430 | struct irq_desc *desc = scoped_irqdesc;
|
| 431 | bool activated;
|
| 432 |
|
| 433 |
|
| 434 | if (irqd_is_started(&desc->irq_data))
|
| 435 | return -EBUSY;
|
| 436 |
|
| 437 |
|
| 438 | if (irqd_affinity_is_managed(&desc->irq_data))
|
| 439 | return -EBUSY;
|
| 440 | |
| 441 | * Deactivate the interrupt. That's required to undo
|
| 442 | * anything an earlier activation has established.
|
| 443 |
|
| 444 | activated = irqd_is_activated(&desc->irq_data);
|
| 445 | if (activated)
|
| 446 | irq_domain_deactivate_irq(&desc->irq_data);
|
| 447 |
|
| 448 | if (affinity->is_managed) {
|
| 449 | irqd_set(&desc->irq_data, IRQD_AFFINITY_MANAGED);
|
| 450 | irqd_set(&desc->irq_data, IRQD_MANAGED_SHUTDOWN);
|
| 451 | }
|
| 452 |
|
| 453 | cpumask_copy(desc->irq_common_data.affinity, &affinity->mask);
|
| 454 |
|
| 455 |
|
| 456 | if (activated)
|
| 457 | irq_domain_activate_irq(&desc->irq_data, false);
|
| 458 | return 0;
|
| 459 | }
|
| 460 | return -EINVAL;
|
| 461 | }
|
| 462 |
|
| 463 | static int __irq_set_affinity(unsigned int irq, const struct cpumask *mask,
|
| 464 | bool force)
|
| 465 | {
|
| 466 | struct irq_desc *desc = irq_to_desc(irq);
|
| 467 |
|
| 468 | if (!desc)
|
| 469 | return -EINVAL;
|
| 470 |
|
| 471 | guard(raw_spinlock_irqsave)(&desc->lock);
|
| 472 | return irq_set_affinity_locked(irq_desc_get_irq_data(desc), mask, force);
|
| 473 | }
|
| 474 |
|
| 475 | |
| 476 | * irq_set_affinity - Set the irq affinity of a given irq
|
| 477 | * @irq: Interrupt to set affinity
|
| 478 | * @cpumask: cpumask
|
| 479 | *
|
| 480 | * Fails if cpumask does not contain an online CPU
|
| 481 |
|
| 482 | int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask)
|
| 483 | {
|
| 484 | return __irq_set_affinity(irq, cpumask, false);
|
| 485 | }
|
| 486 | EXPORT_SYMBOL_GPL(irq_set_affinity);
|
| 487 |
|
| 488 | |
| 489 | * irq_force_affinity - Force the irq affinity of a given irq
|
| 490 | * @irq: Interrupt to set affinity
|
| 491 | * @cpumask: cpumask
|
| 492 | *
|
| 493 | * Same as irq_set_affinity, but without checking the mask against
|
| 494 | * online cpus.
|
| 495 | *
|
| 496 | * Solely for low level cpu hotplug code, where we need to make per
|
| 497 | * cpu interrupts affine before the cpu becomes online.
|
| 498 |
|
| 499 | int irq_force_affinity(unsigned int irq, const struct cpumask *cpumask)
|
| 500 | {
|
| 501 | return __irq_set_affinity(irq, cpumask, true);
|
| 502 | }
|
| 503 | EXPORT_SYMBOL_GPL(irq_force_affinity);
|
| 504 |
|
| 505 | int __irq_apply_affinity_hint(unsigned int irq, const struct cpumask *m, bool setaffinity)
|
| 506 | {
|
| 507 | int ret = -EINVAL;
|
| 508 |
|
| 509 | scoped_irqdesc_get_and_lock(irq, IRQ_GET_DESC_CHECK_GLOBAL) {
|
| 510 | scoped_irqdesc->affinity_hint = m;
|
| 511 | ret = 0;
|
| 512 | }
|
| 513 |
|
| 514 | if (!ret && m && setaffinity)
|
| 515 | __irq_set_affinity(irq, m, false);
|
| 516 | return ret;
|
| 517 | }
|
| 518 | EXPORT_SYMBOL_GPL(__irq_apply_affinity_hint);
|
| 519 |
|
| 520 | static void irq_affinity_notify(struct work_struct *work)
|
| 521 | {
|
| 522 | struct irq_affinity_notify *notify = container_of(work, struct irq_affinity_notify, work);
|
| 523 | struct irq_desc *desc = irq_to_desc(notify->irq);
|
| 524 | cpumask_var_t cpumask;
|
| 525 |
|
| 526 | if (!desc || !alloc_cpumask_var(&cpumask, GFP_KERNEL))
|
| 527 | goto out;
|
| 528 |
|
| 529 | scoped_guard(raw_spinlock_irqsave, &desc->lock) {
|
| 530 | if (irq_move_pending(&desc->irq_data))
|
| 531 | irq_get_pending(cpumask, desc);
|
| 532 | else
|
| 533 | cpumask_copy(cpumask, desc->irq_common_data.affinity);
|
| 534 | }
|
| 535 |
|
| 536 | notify->notify(notify, cpumask);
|
| 537 |
|
| 538 | free_cpumask_var(cpumask);
|
| 539 | out:
|
| 540 | kref_put(¬ify->kref, notify->release);
|
| 541 | }
|
| 542 |
|
| 543 | |
| 544 | * irq_set_affinity_notifier - control notification of IRQ affinity changes
|
| 545 | * @irq: Interrupt for which to enable/disable notification
|
| 546 | * @notify: Context for notification, or %NULL to disable
|
| 547 | * notification. Function pointers must be initialised;
|
| 548 | * the other fields will be initialised by this function.
|
| 549 | *
|
| 550 | * Must be called in process context. Notification may only be enabled
|
| 551 | * after the IRQ is allocated and must be disabled before the IRQ is freed
|
| 552 | * using free_irq().
|
| 553 |
|
| 554 | int irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
|
| 555 | {
|
| 556 | struct irq_desc *desc = irq_to_desc(irq);
|
| 557 | struct irq_affinity_notify *old_notify;
|
| 558 |
|
| 559 |
|
| 560 | might_sleep();
|
| 561 |
|
| 562 | if (!desc || irq_is_nmi(desc))
|
| 563 | return -EINVAL;
|
| 564 |
|
| 565 |
|
| 566 | if (notify) {
|
| 567 | notify->irq = irq;
|
| 568 | kref_init(¬ify->kref);
|
| 569 | INIT_WORK(¬ify->work, irq_affinity_notify);
|
| 570 | }
|
| 571 |
|
| 572 | scoped_guard(raw_spinlock_irq, &desc->lock) {
|
| 573 | old_notify = desc->affinity_notify;
|
| 574 | desc->affinity_notify = notify;
|
| 575 | }
|
| 576 |
|
| 577 | if (old_notify) {
|
| 578 | if (cancel_work_sync(&old_notify->work)) {
|
| 579 |
|
| 580 | kref_put(&old_notify->kref, old_notify->release);
|
| 581 | }
|
| 582 | kref_put(&old_notify->kref, old_notify->release);
|
| 583 | }
|
| 584 |
|
| 585 | return 0;
|
| 586 | }
|
| 587 | EXPORT_SYMBOL_GPL(irq_set_affinity_notifier);
|
| 588 |
|
| 589 | #ifndef CONFIG_AUTO_IRQ_AFFINITY
|
| 590 | |
| 591 | * Generic version of the affinity autoselector.
|
| 592 |
|
| 593 | int irq_setup_affinity(struct irq_desc *desc)
|
| 594 | {
|
| 595 | struct cpumask *set = irq_default_affinity;
|
| 596 | int node = irq_desc_get_node(desc);
|
| 597 |
|
| 598 | static DEFINE_RAW_SPINLOCK(mask_lock);
|
| 599 | static struct cpumask mask;
|
| 600 |
|
| 601 |
|
| 602 | if (!__irq_can_set_affinity(desc))
|
| 603 | return 0;
|
| 604 |
|
| 605 | guard(raw_spinlock)(&mask_lock);
|
| 606 | |
| 607 | * Preserve the managed affinity setting and a userspace affinity
|
| 608 | * setup, but make sure that one of the targets is online.
|
| 609 |
|
| 610 | if (irqd_affinity_is_managed(&desc->irq_data) ||
|
| 611 | irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) {
|
| 612 | if (cpumask_intersects(desc->irq_common_data.affinity,
|
| 613 | cpu_online_mask))
|
| 614 | set = desc->irq_common_data.affinity;
|
| 615 | else
|
| 616 | irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET);
|
| 617 | }
|
| 618 |
|
| 619 | cpumask_and(&mask, cpu_online_mask, set);
|
| 620 | if (cpumask_empty(&mask))
|
| 621 | cpumask_copy(&mask, cpu_online_mask);
|
| 622 |
|
| 623 | if (node != NUMA_NO_NODE) {
|
| 624 | const struct cpumask *nodemask = cpumask_of_node(node);
|
| 625 |
|
| 626 |
|
| 627 | if (cpumask_intersects(&mask, nodemask))
|
| 628 | cpumask_and(&mask, &mask, nodemask);
|
| 629 | }
|
| 630 | return irq_do_set_affinity(&desc->irq_data, &mask, false);
|
| 631 | }
|
| 632 | #else
|
| 633 |
|
| 634 | int irq_setup_affinity(struct irq_desc *desc)
|
| 635 | {
|
| 636 | return irq_select_affinity(irq_desc_get_irq(desc));
|
| 637 | }
|
| 638 | #endif
|
| 639 | #endif
|
| 640 |
|
| 641 |
|
| 642 | |
| 643 | * irq_set_vcpu_affinity - Set vcpu affinity for the interrupt
|
| 644 | * @irq: interrupt number to set affinity
|
| 645 | * @vcpu_info: vCPU specific data or pointer to a percpu array of vCPU
|
| 646 | * specific data for percpu_devid interrupts
|
| 647 | *
|
| 648 | * This function uses the vCPU specific data to set the vCPU affinity for
|
| 649 | * an irq. The vCPU specific data is passed from outside, such as KVM. One
|
| 650 | * example code path is as below: KVM -> IOMMU -> irq_set_vcpu_affinity().
|
| 651 |
|
| 652 | int irq_set_vcpu_affinity(unsigned int irq, void *vcpu_info)
|
| 653 | {
|
| 654 | scoped_irqdesc_get_and_lock(irq, 0) {
|
| 655 | struct irq_desc *desc = scoped_irqdesc;
|
| 656 | struct irq_data *data;
|
| 657 | struct irq_chip *chip;
|
| 658 |
|
| 659 | data = irq_desc_get_irq_data(desc);
|
| 660 | do {
|
| 661 | chip = irq_data_get_irq_chip(data);
|
| 662 | if (chip && chip->irq_set_vcpu_affinity)
|
| 663 | break;
|
| 664 |
|
| 665 | data = irqd_get_parent_data(data);
|
| 666 | } while (data);
|
| 667 |
|
| 668 | if (!data)
|
| 669 | return -ENOSYS;
|
| 670 | return chip->irq_set_vcpu_affinity(data, vcpu_info);
|
| 671 | }
|
| 672 | return -EINVAL;
|
| 673 | }
|
| 674 | EXPORT_SYMBOL_GPL(irq_set_vcpu_affinity);
|
| 675 |
|
| 676 | void __disable_irq(struct irq_desc *desc)
|
| 677 | {
|
| 678 | if (!desc->depth++)
|
| 679 | irq_disable(desc);
|
| 680 | }
|
| 681 |
|
| 682 | static int __disable_irq_nosync(unsigned int irq)
|
| 683 | {
|
| 684 | scoped_irqdesc_get_and_buslock(irq, IRQ_GET_DESC_CHECK_GLOBAL) {
|
| 685 | __disable_irq(scoped_irqdesc);
|
| 686 | return 0;
|
| 687 | }
|
| 688 | return -EINVAL;
|
| 689 | }
|
| 690 |
|
| 691 | |
| 692 | * disable_irq_nosync - disable an irq without waiting
|
| 693 | * @irq: Interrupt to disable
|
| 694 | *
|
| 695 | * Disable the selected interrupt line. Disables and Enables are
|
| 696 | * nested.
|
| 697 | * Unlike disable_irq(), this function does not ensure existing
|
| 698 | * instances of the IRQ handler have completed before returning.
|
| 699 | *
|
| 700 | * This function may be called from IRQ context.
|
| 701 |
|
| 702 | void disable_irq_nosync(unsigned int irq)
|
| 703 | {
|
| 704 | __disable_irq_nosync(irq);
|
| 705 | }
|
| 706 | EXPORT_SYMBOL(disable_irq_nosync);
|
| 707 |
|
| 708 | |
| 709 | * disable_irq - disable an irq and wait for completion
|
| 710 | * @irq: Interrupt to disable
|
| 711 | *
|
| 712 | * Disable the selected interrupt line. Enables and Disables are nested.
|
| 713 | *
|
| 714 | * This function waits for any pending IRQ handlers for this interrupt to
|
| 715 | * complete before returning. If you use this function while holding a
|
| 716 | * resource the IRQ handler may need you will deadlock.
|
| 717 | *
|
| 718 | * Can only be called from preemptible code as it might sleep when an
|
| 719 | * interrupt thread is associated to @irq.
|
| 720 | *
|
| 721 |
|
| 722 | void disable_irq(unsigned int irq)
|
| 723 | {
|
| 724 | might_sleep();
|
| 725 | if (!__disable_irq_nosync(irq))
|
| 726 | synchronize_irq(irq);
|
| 727 | }
|
| 728 | EXPORT_SYMBOL(disable_irq);
|
| 729 |
|
| 730 | |
| 731 | * disable_hardirq - disables an irq and waits for hardirq completion
|
| 732 | * @irq: Interrupt to disable
|
| 733 | *
|
| 734 | * Disable the selected interrupt line. Enables and Disables are nested.
|
| 735 | *
|
| 736 | * This function waits for any pending hard IRQ handlers for this interrupt
|
| 737 | * to complete before returning. If you use this function while holding a
|
| 738 | * resource the hard IRQ handler may need you will deadlock.
|
| 739 | *
|
| 740 | * When used to optimistically disable an interrupt from atomic context the
|
| 741 | * return value must be checked.
|
| 742 | *
|
| 743 | * Returns: false if a threaded handler is active.
|
| 744 | *
|
| 745 | * This function may be called - with care - from IRQ context.
|
| 746 |
|
| 747 | bool disable_hardirq(unsigned int irq)
|
| 748 | {
|
| 749 | if (!__disable_irq_nosync(irq))
|
| 750 | return synchronize_hardirq(irq);
|
| 751 | return false;
|
| 752 | }
|
| 753 | EXPORT_SYMBOL_GPL(disable_hardirq);
|
| 754 |
|
| 755 | |
| 756 | * disable_nmi_nosync - disable an nmi without waiting
|
| 757 | * @irq: Interrupt to disable
|
| 758 | *
|
| 759 | * Disable the selected interrupt line. Disables and enables are nested.
|
| 760 | *
|
| 761 | * The interrupt to disable must have been requested through request_nmi.
|
| 762 | * Unlike disable_nmi(), this function does not ensure existing
|
| 763 | * instances of the IRQ handler have completed before returning.
|
| 764 |
|
| 765 | void disable_nmi_nosync(unsigned int irq)
|
| 766 | {
|
| 767 | disable_irq_nosync(irq);
|
| 768 | }
|
| 769 |
|
| 770 | void __enable_irq(struct irq_desc *desc)
|
| 771 | {
|
| 772 | switch (desc->depth) {
|
| 773 | case 0:
|
| 774 | err_out:
|
| 775 | WARN(1, KERN_WARNING "Unbalanced enable for IRQ %d\n",
|
| 776 | irq_desc_get_irq(desc));
|
| 777 | break;
|
| 778 | case 1: {
|
| 779 | if (desc->istate & IRQS_SUSPENDED)
|
| 780 | goto err_out;
|
| 781 |
|
| 782 | irq_settings_set_noprobe(desc);
|
| 783 | |
| 784 | * Call irq_startup() not irq_enable() here because the
|
| 785 | * interrupt might be marked NOAUTOEN so irq_startup()
|
| 786 | * needs to be invoked when it gets enabled the first time.
|
| 787 | * This is also required when __enable_irq() is invoked for
|
| 788 | * a managed and shutdown interrupt from the S3 resume
|
| 789 | * path.
|
| 790 | *
|
| 791 | * If it was already started up, then irq_startup() will
|
| 792 | * invoke irq_enable() under the hood.
|
| 793 |
|
| 794 | irq_startup(desc, IRQ_RESEND, IRQ_START_FORCE);
|
| 795 | break;
|
| 796 | }
|
| 797 | default:
|
| 798 | desc->depth--;
|
| 799 | }
|
| 800 | }
|
| 801 |
|
| 802 | |
| 803 | * enable_irq - enable handling of an irq
|
| 804 | * @irq: Interrupt to enable
|
| 805 | *
|
| 806 | * Undoes the effect of one call to disable_irq(). If this matches the
|
| 807 | * last disable, processing of interrupts on this IRQ line is re-enabled.
|
| 808 | *
|
| 809 | * This function may be called from IRQ context only when
|
| 810 | * desc->irq_data.chip->bus_lock and desc->chip->bus_sync_unlock are NULL !
|
| 811 |
|
| 812 | void enable_irq(unsigned int irq)
|
| 813 | {
|
| 814 | scoped_irqdesc_get_and_buslock(irq, IRQ_GET_DESC_CHECK_GLOBAL) {
|
| 815 | struct irq_desc *desc = scoped_irqdesc;
|
| 816 |
|
| 817 | if (WARN(!desc->irq_data.chip, "enable_irq before setup/request_irq: irq %u\n", irq))
|
| 818 | return;
|
| 819 | __enable_irq(desc);
|
| 820 | }
|
| 821 | }
|
| 822 | EXPORT_SYMBOL(enable_irq);
|
| 823 |
|
| 824 | |
| 825 | * enable_nmi - enable handling of an nmi
|
| 826 | * @irq: Interrupt to enable
|
| 827 | *
|
| 828 | * The interrupt to enable must have been requested through request_nmi.
|
| 829 | * Undoes the effect of one call to disable_nmi(). If this matches the last
|
| 830 | * disable, processing of interrupts on this IRQ line is re-enabled.
|
| 831 |
|
| 832 | void enable_nmi(unsigned int irq)
|
| 833 | {
|
| 834 | enable_irq(irq);
|
| 835 | }
|
| 836 |
|
| 837 | static int set_irq_wake_real(unsigned int irq, unsigned int on)
|
| 838 | {
|
| 839 | struct irq_desc *desc = irq_to_desc(irq);
|
| 840 | int ret = -ENXIO;
|
| 841 |
|
| 842 | if (irq_desc_get_chip(desc)->flags & IRQCHIP_SKIP_SET_WAKE)
|
| 843 | return 0;
|
| 844 |
|
| 845 | if (desc->irq_data.chip->irq_set_wake)
|
| 846 | ret = desc->irq_data.chip->irq_set_wake(&desc->irq_data, on);
|
| 847 |
|
| 848 | return ret;
|
| 849 | }
|
| 850 |
|
| 851 | |
| 852 | * irq_set_irq_wake - control irq power management wakeup
|
| 853 | * @irq: interrupt to control
|
| 854 | * @on: enable/disable power management wakeup
|
| 855 | *
|
| 856 | * Enable/disable power management wakeup mode, which is disabled by
|
| 857 | * default. Enables and disables must match, just as they match for
|
| 858 | * non-wakeup mode support.
|
| 859 | *
|
| 860 | * Wakeup mode lets this IRQ wake the system from sleep states like
|
| 861 | * "suspend to RAM".
|
| 862 | *
|
| 863 | * Note: irq enable/disable state is completely orthogonal to the
|
| 864 | * enable/disable state of irq wake. An irq can be disabled with
|
| 865 | * disable_irq() and still wake the system as long as the irq has wake
|
| 866 | * enabled. If this does not hold, then the underlying irq chip and the
|
| 867 | * related driver need to be investigated.
|
| 868 |
|
| 869 | int irq_set_irq_wake(unsigned int irq, unsigned int on)
|
| 870 | {
|
| 871 | scoped_irqdesc_get_and_buslock(irq, IRQ_GET_DESC_CHECK_GLOBAL) {
|
| 872 | struct irq_desc *desc = scoped_irqdesc;
|
| 873 | int ret = 0;
|
| 874 |
|
| 875 |
|
| 876 | if (irq_is_nmi(desc))
|
| 877 | return -EINVAL;
|
| 878 |
|
| 879 | |
| 880 | * wakeup-capable irqs can be shared between drivers that
|
| 881 | * don't need to have the same sleep mode behaviors.
|
| 882 |
|
| 883 | if (on) {
|
| 884 | if (desc->wake_depth++ == 0) {
|
| 885 | ret = set_irq_wake_real(irq, on);
|
| 886 | if (ret)
|
| 887 | desc->wake_depth = 0;
|
| 888 | else
|
| 889 | irqd_set(&desc->irq_data, IRQD_WAKEUP_STATE);
|
| 890 | }
|
| 891 | } else {
|
| 892 | if (desc->wake_depth == 0) {
|
| 893 | WARN(1, "Unbalanced IRQ %d wake disable\n", irq);
|
| 894 | } else if (--desc->wake_depth == 0) {
|
| 895 | ret = set_irq_wake_real(irq, on);
|
| 896 | if (ret)
|
| 897 | desc->wake_depth = 1;
|
| 898 | else
|
| 899 | irqd_clear(&desc->irq_data, IRQD_WAKEUP_STATE);
|
| 900 | }
|
| 901 | }
|
| 902 | return ret;
|
| 903 | }
|
| 904 | return -EINVAL;
|
| 905 | }
|
| 906 | EXPORT_SYMBOL(irq_set_irq_wake);
|
| 907 |
|
| 908 | |
| 909 | * Internal function that tells the architecture code whether a
|
| 910 | * particular irq has been exclusively allocated or is available
|
| 911 | * for driver use.
|
| 912 |
|
| 913 | bool can_request_irq(unsigned int irq, unsigned long irqflags)
|
| 914 | {
|
| 915 | scoped_irqdesc_get_and_lock(irq, IRQ_GET_DESC_CHECK_GLOBAL) {
|
| 916 | struct irq_desc *desc = scoped_irqdesc;
|
| 917 |
|
| 918 | if (irq_settings_can_request(desc)) {
|
| 919 | if (!desc->action || irqflags & desc->action->flags & IRQF_SHARED)
|
| 920 | return true;
|
| 921 | }
|
| 922 | }
|
| 923 | return false;
|
| 924 | }
|
| 925 |
|
| 926 | int __irq_set_trigger(struct irq_desc *desc, unsigned long flags)
|
| 927 | {
|
| 928 | struct irq_chip *chip = desc->irq_data.chip;
|
| 929 | int ret, unmask = 0;
|
| 930 |
|
| 931 | if (!chip || !chip->irq_set_type) {
|
| 932 | |
| 933 | * IRQF_TRIGGER_* but the PIC does not support multiple
|
| 934 | * flow-types?
|
| 935 |
|
| 936 | pr_debug("No set_type function for IRQ %d (%s)\n",
|
| 937 | irq_desc_get_irq(desc),
|
| 938 | chip ? (chip->name ? : "unknown") : "unknown");
|
| 939 | return 0;
|
| 940 | }
|
| 941 |
|
| 942 | if (chip->flags & IRQCHIP_SET_TYPE_MASKED) {
|
| 943 | if (!irqd_irq_masked(&desc->irq_data))
|
| 944 | mask_irq(desc);
|
| 945 | if (!irqd_irq_disabled(&desc->irq_data))
|
| 946 | unmask = 1;
|
| 947 | }
|
| 948 |
|
| 949 |
|
| 950 | flags &= IRQ_TYPE_SENSE_MASK;
|
| 951 | ret = chip->irq_set_type(&desc->irq_data, flags);
|
| 952 |
|
| 953 | switch (ret) {
|
| 954 | case IRQ_SET_MASK_OK:
|
| 955 | case IRQ_SET_MASK_OK_DONE:
|
| 956 | irqd_clear(&desc->irq_data, IRQD_TRIGGER_MASK);
|
| 957 | irqd_set(&desc->irq_data, flags);
|
| 958 | fallthrough;
|
| 959 |
|
| 960 | case IRQ_SET_MASK_OK_NOCOPY:
|
| 961 | flags = irqd_get_trigger_type(&desc->irq_data);
|
| 962 | irq_settings_set_trigger_mask(desc, flags);
|
| 963 | irqd_clear(&desc->irq_data, IRQD_LEVEL);
|
| 964 | irq_settings_clr_level(desc);
|
| 965 | if (flags & IRQ_TYPE_LEVEL_MASK) {
|
| 966 | irq_settings_set_level(desc);
|
| 967 | irqd_set(&desc->irq_data, IRQD_LEVEL);
|
| 968 | }
|
| 969 |
|
| 970 | ret = 0;
|
| 971 | break;
|
| 972 | default:
|
| 973 | pr_err("Setting trigger mode %lu for irq %u failed (%pS)\n",
|
| 974 | flags, irq_desc_get_irq(desc), chip->irq_set_type);
|
| 975 | }
|
| 976 | if (unmask)
|
| 977 | unmask_irq(desc);
|
| 978 | return ret;
|
| 979 | }
|
| 980 |
|
| 981 | #ifdef CONFIG_HARDIRQS_SW_RESEND
|
| 982 | int irq_set_parent(int irq, int parent_irq)
|
| 983 | {
|
| 984 | scoped_irqdesc_get_and_lock(irq, 0) {
|
| 985 | scoped_irqdesc->parent_irq = parent_irq;
|
| 986 | return 0;
|
| 987 | }
|
| 988 | return -EINVAL;
|
| 989 | }
|
| 990 | EXPORT_SYMBOL_GPL(irq_set_parent);
|
| 991 | #endif
|
| 992 |
|
| 993 | |
| 994 | * Default primary interrupt handler for threaded interrupts. Is
|
| 995 | * assigned as primary handler when request_threaded_irq is called
|
| 996 | * with handler == NULL. Useful for oneshot interrupts.
|
| 997 |
|
| 998 | static irqreturn_t irq_default_primary_handler(int irq, void *dev_id)
|
| 999 | {
|
| 1000 | return IRQ_WAKE_THREAD;
|
| 1001 | }
|
| 1002 |
|
| 1003 | |
| 1004 | * Primary handler for nested threaded interrupts. Should never be
|
| 1005 | * called.
|
| 1006 |
|
| 1007 | static irqreturn_t irq_nested_primary_handler(int irq, void *dev_id)
|
| 1008 | {
|
| 1009 | WARN(1, "Primary handler called for nested irq %d\n", irq);
|
| 1010 | return IRQ_NONE;
|
| 1011 | }
|
| 1012 |
|
| 1013 | static irqreturn_t irq_forced_secondary_handler(int irq, void *dev_id)
|
| 1014 | {
|
| 1015 | WARN(1, "Secondary action handler called for irq %d\n", irq);
|
| 1016 | return IRQ_NONE;
|
| 1017 | }
|
| 1018 |
|
| 1019 | #ifdef CONFIG_SMP
|
| 1020 | |
| 1021 | * Check whether we need to change the affinity of the interrupt thread.
|
| 1022 |
|
| 1023 | static void irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action)
|
| 1024 | {
|
| 1025 | cpumask_var_t mask;
|
| 1026 |
|
| 1027 | if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags))
|
| 1028 | return;
|
| 1029 |
|
| 1030 | __set_current_state(TASK_RUNNING);
|
| 1031 |
|
| 1032 | |
| 1033 | * In case we are out of memory we set IRQTF_AFFINITY again and
|
| 1034 | * try again next time
|
| 1035 |
|
| 1036 | if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
|
| 1037 | set_bit(IRQTF_AFFINITY, &action->thread_flags);
|
| 1038 | return;
|
| 1039 | }
|
| 1040 |
|
| 1041 | scoped_guard(raw_spinlock_irq, &desc->lock) {
|
| 1042 | const struct cpumask *m;
|
| 1043 |
|
| 1044 | m = irq_data_get_effective_affinity_mask(&desc->irq_data);
|
| 1045 | cpumask_copy(mask, m);
|
| 1046 | }
|
| 1047 |
|
| 1048 | set_cpus_allowed_ptr(current, mask);
|
| 1049 | free_cpumask_var(mask);
|
| 1050 | }
|
| 1051 | #else
|
| 1052 | static inline void irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action) { }
|
| 1053 | #endif
|
| 1054 |
|
| 1055 | static int irq_wait_for_interrupt(struct irq_desc *desc,
|
| 1056 | struct irqaction *action)
|
| 1057 | {
|
| 1058 | for (;;) {
|
| 1059 | set_current_state(TASK_INTERRUPTIBLE);
|
| 1060 | irq_thread_check_affinity(desc, action);
|
| 1061 |
|
| 1062 | if (kthread_should_stop()) {
|
| 1063 |
|
| 1064 | if (test_and_clear_bit(IRQTF_RUNTHREAD,
|
| 1065 | &action->thread_flags)) {
|
| 1066 | __set_current_state(TASK_RUNNING);
|
| 1067 | return 0;
|
| 1068 | }
|
| 1069 | __set_current_state(TASK_RUNNING);
|
| 1070 | return -1;
|
| 1071 | }
|
| 1072 |
|
| 1073 | if (test_and_clear_bit(IRQTF_RUNTHREAD,
|
| 1074 | &action->thread_flags)) {
|
| 1075 | __set_current_state(TASK_RUNNING);
|
| 1076 | return 0;
|
| 1077 | }
|
| 1078 | schedule();
|
| 1079 | }
|
| 1080 | }
|
| 1081 |
|
| 1082 | |
| 1083 | * Oneshot interrupts keep the irq line masked until the threaded
|
| 1084 | * handler finished. unmask if the interrupt has not been disabled and
|
| 1085 | * is marked MASKED.
|
| 1086 |
|
| 1087 | static void irq_finalize_oneshot(struct irq_desc *desc,
|
| 1088 | struct irqaction *action)
|
| 1089 | {
|
| 1090 | if (!(desc->istate & IRQS_ONESHOT) ||
|
| 1091 | action->handler == irq_forced_secondary_handler)
|
| 1092 | return;
|
| 1093 | again:
|
| 1094 | chip_bus_lock(desc);
|
| 1095 | raw_spin_lock_irq(&desc->lock);
|
| 1096 |
|
| 1097 | |
| 1098 | * Implausible though it may be we need to protect us against
|
| 1099 | * the following scenario:
|
| 1100 | *
|
| 1101 | * The thread is faster done than the hard interrupt handler
|
| 1102 | * on the other CPU. If we unmask the irq line then the
|
| 1103 | * interrupt can come in again and masks the line, leaves due
|
| 1104 | * to IRQS_INPROGRESS and the irq line is masked forever.
|
| 1105 | *
|
| 1106 | * This also serializes the state of shared oneshot handlers
|
| 1107 | * versus "desc->threads_oneshot |= action->thread_mask;" in
|
| 1108 | * irq_wake_thread(). See the comment there which explains the
|
| 1109 | * serialization.
|
| 1110 |
|
| 1111 | if (unlikely(irqd_irq_inprogress(&desc->irq_data))) {
|
| 1112 | raw_spin_unlock_irq(&desc->lock);
|
| 1113 | chip_bus_sync_unlock(desc);
|
| 1114 | cpu_relax();
|
| 1115 | goto again;
|
| 1116 | }
|
| 1117 |
|
| 1118 | |
| 1119 | * Now check again, whether the thread should run. Otherwise
|
| 1120 | * we would clear the threads_oneshot bit of this thread which
|
| 1121 | * was just set.
|
| 1122 |
|
| 1123 | if (test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
|
| 1124 | goto out_unlock;
|
| 1125 |
|
| 1126 | desc->threads_oneshot &= ~action->thread_mask;
|
| 1127 |
|
| 1128 | if (!desc->threads_oneshot && !irqd_irq_disabled(&desc->irq_data) &&
|
| 1129 | irqd_irq_masked(&desc->irq_data))
|
| 1130 | unmask_threaded_irq(desc);
|
| 1131 |
|
| 1132 | out_unlock:
|
| 1133 | raw_spin_unlock_irq(&desc->lock);
|
| 1134 | chip_bus_sync_unlock(desc);
|
| 1135 | }
|
| 1136 |
|
| 1137 | |
| 1138 | * Interrupts explicitly requested as threaded interrupts want to be
|
| 1139 | * preemptible - many of them need to sleep and wait for slow busses to
|
| 1140 | * complete.
|
| 1141 |
|
| 1142 | static irqreturn_t irq_thread_fn(struct irq_desc *desc, struct irqaction *action)
|
| 1143 | {
|
| 1144 | irqreturn_t ret = action->thread_fn(action->irq, action->dev_id);
|
| 1145 |
|
| 1146 | if (ret == IRQ_HANDLED)
|
| 1147 | atomic_inc(&desc->threads_handled);
|
| 1148 |
|
| 1149 | irq_finalize_oneshot(desc, action);
|
| 1150 | return ret;
|
| 1151 | }
|
| 1152 |
|
| 1153 | |
| 1154 | * Interrupts which are not explicitly requested as threaded
|
| 1155 | * interrupts rely on the implicit bh/preempt disable of the hard irq
|
| 1156 | * context. So we need to disable bh here to avoid deadlocks and other
|
| 1157 | * side effects.
|
| 1158 |
|
| 1159 | static irqreturn_t irq_forced_thread_fn(struct irq_desc *desc, struct irqaction *action)
|
| 1160 | {
|
| 1161 | irqreturn_t ret;
|
| 1162 |
|
| 1163 | local_bh_disable();
|
| 1164 | if (!IS_ENABLED(CONFIG_PREEMPT_RT))
|
| 1165 | local_irq_disable();
|
| 1166 | ret = irq_thread_fn(desc, action);
|
| 1167 | if (!IS_ENABLED(CONFIG_PREEMPT_RT))
|
| 1168 | local_irq_enable();
|
| 1169 | local_bh_enable();
|
| 1170 | return ret;
|
| 1171 | }
|
| 1172 |
|
| 1173 | void wake_threads_waitq(struct irq_desc *desc)
|
| 1174 | {
|
| 1175 | if (atomic_dec_and_test(&desc->threads_active))
|
| 1176 | wake_up(&desc->wait_for_threads);
|
| 1177 | }
|
| 1178 |
|
| 1179 | static void irq_thread_dtor(struct callback_head *unused)
|
| 1180 | {
|
| 1181 | struct task_struct *tsk = current;
|
| 1182 | struct irq_desc *desc;
|
| 1183 | struct irqaction *action;
|
| 1184 |
|
| 1185 | if (WARN_ON_ONCE(!(current->flags & PF_EXITING)))
|
| 1186 | return;
|
| 1187 |
|
| 1188 | action = kthread_data(tsk);
|
| 1189 |
|
| 1190 | pr_err("exiting task \"%s\" (%d) is an active IRQ thread (irq %d)\n",
|
| 1191 | tsk->comm, tsk->pid, action->irq);
|
| 1192 |
|
| 1193 |
|
| 1194 | desc = irq_to_desc(action->irq);
|
| 1195 | |
| 1196 | * If IRQTF_RUNTHREAD is set, we need to decrement
|
| 1197 | * desc->threads_active and wake possible waiters.
|
| 1198 |
|
| 1199 | if (test_and_clear_bit(IRQTF_RUNTHREAD, &action->thread_flags))
|
| 1200 | wake_threads_waitq(desc);
|
| 1201 |
|
| 1202 |
|
| 1203 | irq_finalize_oneshot(desc, action);
|
| 1204 | }
|
| 1205 |
|
| 1206 | static void irq_wake_secondary(struct irq_desc *desc, struct irqaction *action)
|
| 1207 | {
|
| 1208 | struct irqaction *secondary = action->secondary;
|
| 1209 |
|
| 1210 | if (WARN_ON_ONCE(!secondary))
|
| 1211 | return;
|
| 1212 |
|
| 1213 | guard(raw_spinlock_irq)(&desc->lock);
|
| 1214 | __irq_wake_thread(desc, secondary);
|
| 1215 | }
|
| 1216 |
|
| 1217 | |
| 1218 | * Internal function to notify that a interrupt thread is ready.
|
| 1219 |
|
| 1220 | static void irq_thread_set_ready(struct irq_desc *desc,
|
| 1221 | struct irqaction *action)
|
| 1222 | {
|
| 1223 | set_bit(IRQTF_READY, &action->thread_flags);
|
| 1224 | wake_up(&desc->wait_for_threads);
|
| 1225 | }
|
| 1226 |
|
| 1227 | |
| 1228 | * Internal function to wake up a interrupt thread and wait until it is
|
| 1229 | * ready.
|
| 1230 |
|
| 1231 | static void wake_up_and_wait_for_irq_thread_ready(struct irq_desc *desc,
|
| 1232 | struct irqaction *action)
|
| 1233 | {
|
| 1234 | if (!action || !action->thread)
|
| 1235 | return;
|
| 1236 |
|
| 1237 | wake_up_process(action->thread);
|
| 1238 | wait_event(desc->wait_for_threads,
|
| 1239 | test_bit(IRQTF_READY, &action->thread_flags));
|
| 1240 | }
|
| 1241 |
|
| 1242 | |
| 1243 | * Interrupt handler thread
|
| 1244 |
|
| 1245 | static int irq_thread(void *data)
|
| 1246 | {
|
| 1247 | struct callback_head on_exit_work;
|
| 1248 | struct irqaction *action = data;
|
| 1249 | struct irq_desc *desc = irq_to_desc(action->irq);
|
| 1250 | irqreturn_t (*handler_fn)(struct irq_desc *desc,
|
| 1251 | struct irqaction *action);
|
| 1252 |
|
| 1253 | irq_thread_set_ready(desc, action);
|
| 1254 |
|
| 1255 | if (action->handler == irq_forced_secondary_handler)
|
| 1256 | sched_set_fifo_secondary(current);
|
| 1257 | else
|
| 1258 | sched_set_fifo(current);
|
| 1259 |
|
| 1260 | if (force_irqthreads() && test_bit(IRQTF_FORCED_THREAD,
|
| 1261 | &action->thread_flags))
|
| 1262 | handler_fn = irq_forced_thread_fn;
|
| 1263 | else
|
| 1264 | handler_fn = irq_thread_fn;
|
| 1265 |
|
| 1266 | init_task_work(&on_exit_work, irq_thread_dtor);
|
| 1267 | task_work_add(current, &on_exit_work, TWA_NONE);
|
| 1268 |
|
| 1269 | while (!irq_wait_for_interrupt(desc, action)) {
|
| 1270 | irqreturn_t action_ret;
|
| 1271 |
|
| 1272 | action_ret = handler_fn(desc, action);
|
| 1273 | if (action_ret == IRQ_WAKE_THREAD)
|
| 1274 | irq_wake_secondary(desc, action);
|
| 1275 |
|
| 1276 | wake_threads_waitq(desc);
|
| 1277 | }
|
| 1278 |
|
| 1279 | |
| 1280 | * This is the regular exit path. __free_irq() is stopping the
|
| 1281 | * thread via kthread_stop() after calling
|
| 1282 | * synchronize_hardirq(). So neither IRQTF_RUNTHREAD nor the
|
| 1283 | * oneshot mask bit can be set.
|
| 1284 |
|
| 1285 | task_work_cancel_func(current, irq_thread_dtor);
|
| 1286 | return 0;
|
| 1287 | }
|
| 1288 |
|
| 1289 | |
| 1290 | * irq_wake_thread - wake the irq thread for the action identified by dev_id
|
| 1291 | * @irq: Interrupt line
|
| 1292 | * @dev_id: Device identity for which the thread should be woken
|
| 1293 |
|
| 1294 | void irq_wake_thread(unsigned int irq, void *dev_id)
|
| 1295 | {
|
| 1296 | struct irq_desc *desc = irq_to_desc(irq);
|
| 1297 | struct irqaction *action;
|
| 1298 |
|
| 1299 | if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
|
| 1300 | return;
|
| 1301 |
|
| 1302 | guard(raw_spinlock_irqsave)(&desc->lock);
|
| 1303 | for_each_action_of_desc(desc, action) {
|
| 1304 | if (action->dev_id == dev_id) {
|
| 1305 | if (action->thread)
|
| 1306 | __irq_wake_thread(desc, action);
|
| 1307 | break;
|
| 1308 | }
|
| 1309 | }
|
| 1310 | }
|
| 1311 | EXPORT_SYMBOL_GPL(irq_wake_thread);
|
| 1312 |
|
| 1313 | static int irq_setup_forced_threading(struct irqaction *new)
|
| 1314 | {
|
| 1315 | if (!force_irqthreads())
|
| 1316 | return 0;
|
| 1317 | if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))
|
| 1318 | return 0;
|
| 1319 |
|
| 1320 | |
| 1321 | * No further action required for interrupts which are requested as
|
| 1322 | * threaded interrupts already
|
| 1323 |
|
| 1324 | if (new->handler == irq_default_primary_handler)
|
| 1325 | return 0;
|
| 1326 |
|
| 1327 | new->flags |= IRQF_ONESHOT;
|
| 1328 |
|
| 1329 | |
| 1330 | * Handle the case where we have a real primary handler and a
|
| 1331 | * thread handler. We force thread them as well by creating a
|
| 1332 | * secondary action.
|
| 1333 |
|
| 1334 | if (new->handler && new->thread_fn) {
|
| 1335 |
|
| 1336 | new->secondary = kzalloc_obj(struct irqaction);
|
| 1337 | if (!new->secondary)
|
| 1338 | return -ENOMEM;
|
| 1339 | new->secondary->handler = irq_forced_secondary_handler;
|
| 1340 | new->secondary->thread_fn = new->thread_fn;
|
| 1341 | new->secondary->dev_id = new->dev_id;
|
| 1342 | new->secondary->irq = new->irq;
|
| 1343 | new->secondary->name = new->name;
|
| 1344 | }
|
| 1345 |
|
| 1346 | set_bit(IRQTF_FORCED_THREAD, &new->thread_flags);
|
| 1347 | new->thread_fn = new->handler;
|
| 1348 | new->handler = irq_default_primary_handler;
|
| 1349 | return 0;
|
| 1350 | }
|
| 1351 |
|
| 1352 | static int irq_request_resources(struct irq_desc *desc)
|
| 1353 | {
|
| 1354 | struct irq_data *d = &desc->irq_data;
|
| 1355 | struct irq_chip *c = d->chip;
|
| 1356 |
|
| 1357 | return c->irq_request_resources ? c->irq_request_resources(d) : 0;
|
| 1358 | }
|
| 1359 |
|
| 1360 | static void irq_release_resources(struct irq_desc *desc)
|
| 1361 | {
|
| 1362 | struct irq_data *d = &desc->irq_data;
|
| 1363 | struct irq_chip *c = d->chip;
|
| 1364 |
|
| 1365 | if (c->irq_release_resources)
|
| 1366 | c->irq_release_resources(d);
|
| 1367 | }
|
| 1368 |
|
| 1369 | static bool irq_supports_nmi(struct irq_desc *desc)
|
| 1370 | {
|
| 1371 | struct irq_data *d = irq_desc_get_irq_data(desc);
|
| 1372 |
|
| 1373 | #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
|
| 1374 |
|
| 1375 | if (d->parent_data)
|
| 1376 | return false;
|
| 1377 | #endif
|
| 1378 |
|
| 1379 | if (d->chip->irq_bus_lock || d->chip->irq_bus_sync_unlock)
|
| 1380 | return false;
|
| 1381 |
|
| 1382 | return d->chip->flags & IRQCHIP_SUPPORTS_NMI;
|
| 1383 | }
|
| 1384 |
|
| 1385 | static int irq_nmi_setup(struct irq_desc *desc)
|
| 1386 | {
|
| 1387 | struct irq_data *d = irq_desc_get_irq_data(desc);
|
| 1388 | struct irq_chip *c = d->chip;
|
| 1389 |
|
| 1390 | return c->irq_nmi_setup ? c->irq_nmi_setup(d) : -EINVAL;
|
| 1391 | }
|
| 1392 |
|
| 1393 | static void irq_nmi_teardown(struct irq_desc *desc)
|
| 1394 | {
|
| 1395 | struct irq_data *d = irq_desc_get_irq_data(desc);
|
| 1396 | struct irq_chip *c = d->chip;
|
| 1397 |
|
| 1398 | if (c->irq_nmi_teardown)
|
| 1399 | c->irq_nmi_teardown(d);
|
| 1400 | }
|
| 1401 |
|
| 1402 | static int
|
| 1403 | setup_irq_thread(struct irqaction *new, unsigned int irq, bool secondary)
|
| 1404 | {
|
| 1405 | struct task_struct *t;
|
| 1406 |
|
| 1407 | if (!secondary) {
|
| 1408 | t = kthread_create(irq_thread, new, "irq/%d-%s", irq,
|
| 1409 | new->name);
|
| 1410 | } else {
|
| 1411 | t = kthread_create(irq_thread, new, "irq/%d-s-%s", irq,
|
| 1412 | new->name);
|
| 1413 | }
|
| 1414 |
|
| 1415 | if (IS_ERR(t))
|
| 1416 | return PTR_ERR(t);
|
| 1417 |
|
| 1418 | |
| 1419 | * We keep the reference to the task struct even if
|
| 1420 | * the thread dies to avoid that the interrupt code
|
| 1421 | * references an already freed task_struct.
|
| 1422 |
|
| 1423 | new->thread = get_task_struct(t);
|
| 1424 |
|
| 1425 | |
| 1426 | * The affinity can not be established yet, but it will be once the
|
| 1427 | * interrupt is enabled. Delay and defer the actual setting to the
|
| 1428 | * thread itself once it is ready to run. In the meantime, prevent
|
| 1429 | * it from ever being re-affined directly by cpuset or
|
| 1430 | * housekeeping. The proper way to do it is to re-affine the whole
|
| 1431 | * vector.
|
| 1432 |
|
| 1433 | kthread_bind_mask(t, cpu_possible_mask);
|
| 1434 |
|
| 1435 | |
| 1436 | * Ensure the thread adjusts the affinity once it reaches the
|
| 1437 | * thread function.
|
| 1438 |
|
| 1439 | set_bit(IRQTF_AFFINITY, &new->thread_flags);
|
| 1440 |
|
| 1441 | return 0;
|
| 1442 | }
|
| 1443 |
|
| 1444 | static bool valid_percpu_irqaction(struct irqaction *old, struct irqaction *new)
|
| 1445 | {
|
| 1446 | do {
|
| 1447 | if (cpumask_intersects(old->affinity, new->affinity) ||
|
| 1448 | old->percpu_dev_id == new->percpu_dev_id)
|
| 1449 | return false;
|
| 1450 |
|
| 1451 | old = old->next;
|
| 1452 | } while (old);
|
| 1453 |
|
| 1454 | return true;
|
| 1455 | }
|
| 1456 |
|
| 1457 | |
| 1458 | * Internal function to register an irqaction - typically used to
|
| 1459 | * allocate special interrupts that are part of the architecture.
|
| 1460 | *
|
| 1461 | * Locking rules:
|
| 1462 | *
|
| 1463 | * desc->request_mutex Provides serialization against a concurrent free_irq()
|
| 1464 | * chip_bus_lock Provides serialization for slow bus operations
|
| 1465 | * desc->lock Provides serialization against hard interrupts
|
| 1466 | *
|
| 1467 | * chip_bus_lock and desc->lock are sufficient for all other management and
|
| 1468 | * interrupt related functions. desc->request_mutex solely serializes
|
| 1469 | * request/free_irq().
|
| 1470 |
|
| 1471 | static int
|
| 1472 | __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
|
| 1473 | {
|
| 1474 | struct irqaction *old, **old_ptr;
|
| 1475 | unsigned long flags, thread_mask = 0;
|
| 1476 | int ret, nested, shared = 0;
|
| 1477 | bool per_cpu_devid;
|
| 1478 |
|
| 1479 | if (!desc)
|
| 1480 | return -EINVAL;
|
| 1481 |
|
| 1482 | if (desc->irq_data.chip == &no_irq_chip)
|
| 1483 | return -ENOSYS;
|
| 1484 | if (!try_module_get(desc->owner))
|
| 1485 | return -ENODEV;
|
| 1486 |
|
| 1487 | per_cpu_devid = irq_settings_is_per_cpu_devid(desc);
|
| 1488 |
|
| 1489 | new->irq = irq;
|
| 1490 |
|
| 1491 | |
| 1492 | * If the trigger type is not specified by the caller,
|
| 1493 | * then use the default for this interrupt.
|
| 1494 |
|
| 1495 | if (!(new->flags & IRQF_TRIGGER_MASK))
|
| 1496 | new->flags |= irqd_get_trigger_type(&desc->irq_data);
|
| 1497 |
|
| 1498 | |
| 1499 | * IRQF_ONESHOT means the interrupt source in the IRQ chip will be
|
| 1500 | * masked until the threaded handled is done. If there is no thread
|
| 1501 | * handler then it makes no sense to have IRQF_ONESHOT.
|
| 1502 |
|
| 1503 | WARN_ON_ONCE(new->flags & IRQF_ONESHOT && !new->thread_fn);
|
| 1504 |
|
| 1505 | |
| 1506 | * Check whether the interrupt nests into another interrupt
|
| 1507 | * thread.
|
| 1508 |
|
| 1509 | nested = irq_settings_is_nested_thread(desc);
|
| 1510 | if (nested) {
|
| 1511 | if (!new->thread_fn) {
|
| 1512 | ret = -EINVAL;
|
| 1513 | goto out_mput;
|
| 1514 | }
|
| 1515 | |
| 1516 | * Replace the primary handler which was provided from
|
| 1517 | * the driver for non nested interrupt handling by the
|
| 1518 | * dummy function which warns when called.
|
| 1519 |
|
| 1520 | new->handler = irq_nested_primary_handler;
|
| 1521 | } else {
|
| 1522 | if (irq_settings_can_thread(desc)) {
|
| 1523 | ret = irq_setup_forced_threading(new);
|
| 1524 | if (ret)
|
| 1525 | goto out_mput;
|
| 1526 | }
|
| 1527 | }
|
| 1528 |
|
| 1529 | |
| 1530 | * Create a handler thread when a thread function is supplied
|
| 1531 | * and the interrupt does not nest into another interrupt
|
| 1532 | * thread.
|
| 1533 |
|
| 1534 | if (new->thread_fn && !nested) {
|
| 1535 | ret = setup_irq_thread(new, irq, false);
|
| 1536 | if (ret)
|
| 1537 | goto out_mput;
|
| 1538 | if (new->secondary) {
|
| 1539 | ret = setup_irq_thread(new->secondary, irq, true);
|
| 1540 | if (ret)
|
| 1541 | goto out_thread;
|
| 1542 | }
|
| 1543 | }
|
| 1544 |
|
| 1545 | |
| 1546 | * Drivers are often written to work w/o knowledge about the
|
| 1547 | * underlying irq chip implementation, so a request for a
|
| 1548 | * threaded irq without a primary hard irq context handler
|
| 1549 | * requires the ONESHOT flag to be set. Some irq chips like
|
| 1550 | * MSI based interrupts are per se one shot safe. Check the
|
| 1551 | * chip flags, so we can avoid the unmask dance at the end of
|
| 1552 | * the threaded handler for those.
|
| 1553 |
|
| 1554 | if (desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)
|
| 1555 | new->flags &= ~IRQF_ONESHOT;
|
| 1556 |
|
| 1557 | |
| 1558 | * Protects against a concurrent __free_irq() call which might wait
|
| 1559 | * for synchronize_hardirq() to complete without holding the optional
|
| 1560 | * chip bus lock and desc->lock. Also protects against handing out
|
| 1561 | * a recycled oneshot thread_mask bit while it's still in use by
|
| 1562 | * its previous owner.
|
| 1563 |
|
| 1564 | mutex_lock(&desc->request_mutex);
|
| 1565 |
|
| 1566 | |
| 1567 | * Acquire bus lock as the irq_request_resources() callback below
|
| 1568 | * might rely on the serialization or the magic power management
|
| 1569 | * functions which are abusing the irq_bus_lock() callback,
|
| 1570 |
|
| 1571 | chip_bus_lock(desc);
|
| 1572 |
|
| 1573 |
|
| 1574 | if (!desc->action) {
|
| 1575 | ret = irq_request_resources(desc);
|
| 1576 | if (ret) {
|
| 1577 | pr_err("Failed to request resources for %s (irq %d) on irqchip %s\n",
|
| 1578 | new->name, irq, desc->irq_data.chip->name);
|
| 1579 | goto out_bus_unlock;
|
| 1580 | }
|
| 1581 | }
|
| 1582 |
|
| 1583 | |
| 1584 | * The following block of code has to be executed atomically
|
| 1585 | * protected against a concurrent interrupt and any of the other
|
| 1586 | * management calls which are not serialized via
|
| 1587 | * desc->request_mutex or the optional bus lock.
|
| 1588 |
|
| 1589 | raw_spin_lock_irqsave(&desc->lock, flags);
|
| 1590 | old_ptr = &desc->action;
|
| 1591 | old = *old_ptr;
|
| 1592 | if (old) {
|
| 1593 | |
| 1594 | * Can't share interrupts unless both agree to and are
|
| 1595 | * the same type (level, edge, polarity). So both flag
|
| 1596 | * fields must have IRQF_SHARED set and the bits which
|
| 1597 | * set the trigger type must match. Also all must
|
| 1598 | * agree on ONESHOT.
|
| 1599 | * Interrupt lines used for NMIs cannot be shared.
|
| 1600 |
|
| 1601 | unsigned int oldtype;
|
| 1602 |
|
| 1603 | if (irq_is_nmi(desc) && !per_cpu_devid) {
|
| 1604 | pr_err("Invalid attempt to share NMI for %s (irq %d) on irqchip %s.\n",
|
| 1605 | new->name, irq, desc->irq_data.chip->name);
|
| 1606 | ret = -EINVAL;
|
| 1607 | goto out_unlock;
|
| 1608 | }
|
| 1609 |
|
| 1610 | if (per_cpu_devid && !valid_percpu_irqaction(old, new)) {
|
| 1611 | pr_err("Overlapping affinities for %s (irq %d) on irqchip %s.\n",
|
| 1612 | new->name, irq, desc->irq_data.chip->name);
|
| 1613 | ret = -EINVAL;
|
| 1614 | goto out_unlock;
|
| 1615 | }
|
| 1616 |
|
| 1617 | |
| 1618 | * If nobody did set the configuration before, inherit
|
| 1619 | * the one provided by the requester.
|
| 1620 |
|
| 1621 | if (irqd_trigger_type_was_set(&desc->irq_data)) {
|
| 1622 | oldtype = irqd_get_trigger_type(&desc->irq_data);
|
| 1623 | } else {
|
| 1624 | oldtype = new->flags & IRQF_TRIGGER_MASK;
|
| 1625 | irqd_set_trigger_type(&desc->irq_data, oldtype);
|
| 1626 | }
|
| 1627 |
|
| 1628 | if (!((old->flags & new->flags) & IRQF_SHARED) ||
|
| 1629 | (oldtype != (new->flags & IRQF_TRIGGER_MASK)))
|
| 1630 | goto mismatch;
|
| 1631 |
|
| 1632 | if ((old->flags & IRQF_ONESHOT) &&
|
| 1633 | (new->flags & IRQF_COND_ONESHOT))
|
| 1634 | new->flags |= IRQF_ONESHOT;
|
| 1635 | else if ((old->flags ^ new->flags) & IRQF_ONESHOT)
|
| 1636 | goto mismatch;
|
| 1637 |
|
| 1638 |
|
| 1639 | if ((old->flags & IRQF_PERCPU) !=
|
| 1640 | (new->flags & IRQF_PERCPU))
|
| 1641 | goto mismatch;
|
| 1642 |
|
| 1643 |
|
| 1644 | do {
|
| 1645 | |
| 1646 | * Or all existing action->thread_mask bits,
|
| 1647 | * so we can find the next zero bit for this
|
| 1648 | * new action.
|
| 1649 |
|
| 1650 | thread_mask |= old->thread_mask;
|
| 1651 | old_ptr = &old->next;
|
| 1652 | old = *old_ptr;
|
| 1653 | } while (old);
|
| 1654 | shared = 1;
|
| 1655 | }
|
| 1656 |
|
| 1657 | |
| 1658 | * Setup the thread mask for this irqaction for ONESHOT. For
|
| 1659 | * !ONESHOT irqs the thread mask is 0 so we can avoid a
|
| 1660 | * conditional in irq_wake_thread().
|
| 1661 |
|
| 1662 | if (new->flags & IRQF_ONESHOT) {
|
| 1663 | |
| 1664 | * Unlikely to have 32 resp 64 irqs sharing one line,
|
| 1665 | * but who knows.
|
| 1666 |
|
| 1667 | if (thread_mask == ~0UL) {
|
| 1668 | ret = -EBUSY;
|
| 1669 | goto out_unlock;
|
| 1670 | }
|
| 1671 | |
| 1672 | * The thread_mask for the action is or'ed to
|
| 1673 | * desc->thread_active to indicate that the
|
| 1674 | * IRQF_ONESHOT thread handler has been woken, but not
|
| 1675 | * yet finished. The bit is cleared when a thread
|
| 1676 | * completes. When all threads of a shared interrupt
|
| 1677 | * line have completed desc->threads_active becomes
|
| 1678 | * zero and the interrupt line is unmasked. See
|
| 1679 | * handle.c:irq_wake_thread() for further information.
|
| 1680 | *
|
| 1681 | * If no thread is woken by primary (hard irq context)
|
| 1682 | * interrupt handlers, then desc->threads_active is
|
| 1683 | * also checked for zero to unmask the irq line in the
|
| 1684 | * affected hard irq flow handlers
|
| 1685 | * (handle_[fasteoi|level]_irq).
|
| 1686 | *
|
| 1687 | * The new action gets the first zero bit of
|
| 1688 | * thread_mask assigned. See the loop above which or's
|
| 1689 | * all existing action->thread_mask bits.
|
| 1690 |
|
| 1691 | new->thread_mask = 1UL << ffz(thread_mask);
|
| 1692 |
|
| 1693 | } else if (new->handler == irq_default_primary_handler &&
|
| 1694 | !(desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)) {
|
| 1695 | |
| 1696 | * The interrupt was requested with handler = NULL, so
|
| 1697 | * we use the default primary handler for it. But it
|
| 1698 | * does not have the oneshot flag set. In combination
|
| 1699 | * with level interrupts this is deadly, because the
|
| 1700 | * default primary handler just wakes the thread, then
|
| 1701 | * the irq lines is reenabled, but the device still
|
| 1702 | * has the level irq asserted. Rinse and repeat....
|
| 1703 | *
|
| 1704 | * While this works for edge type interrupts, we play
|
| 1705 | * it safe and reject unconditionally because we can't
|
| 1706 | * say for sure which type this interrupt really
|
| 1707 | * has. The type flags are unreliable as the
|
| 1708 | * underlying chip implementation can override them.
|
| 1709 |
|
| 1710 | pr_err("Threaded irq requested with handler=NULL and !ONESHOT for %s (irq %d)\n",
|
| 1711 | new->name, irq);
|
| 1712 | ret = -EINVAL;
|
| 1713 | goto out_unlock;
|
| 1714 | }
|
| 1715 |
|
| 1716 | if (!shared) {
|
| 1717 |
|
| 1718 | if (new->flags & IRQF_TRIGGER_MASK) {
|
| 1719 | ret = __irq_set_trigger(desc,
|
| 1720 | new->flags & IRQF_TRIGGER_MASK);
|
| 1721 |
|
| 1722 | if (ret)
|
| 1723 | goto out_unlock;
|
| 1724 | }
|
| 1725 |
|
| 1726 | |
| 1727 | * Activate the interrupt. That activation must happen
|
| 1728 | * independently of IRQ_NOAUTOEN. request_irq() can fail
|
| 1729 | * and the callers are supposed to handle
|
| 1730 | * that. enable_irq() of an interrupt requested with
|
| 1731 | * IRQ_NOAUTOEN is not supposed to fail. The activation
|
| 1732 | * keeps it in shutdown mode, it merily associates
|
| 1733 | * resources if necessary and if that's not possible it
|
| 1734 | * fails. Interrupts which are in managed shutdown mode
|
| 1735 | * will simply ignore that activation request.
|
| 1736 |
|
| 1737 | ret = irq_activate(desc);
|
| 1738 | if (ret)
|
| 1739 | goto out_unlock;
|
| 1740 |
|
| 1741 | desc->istate &= ~(IRQS_AUTODETECT | IRQS_SPURIOUS_DISABLED | \
|
| 1742 | IRQS_ONESHOT | IRQS_WAITING);
|
| 1743 | irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
|
| 1744 |
|
| 1745 | if (new->flags & IRQF_PERCPU) {
|
| 1746 | irqd_set(&desc->irq_data, IRQD_PER_CPU);
|
| 1747 | irq_settings_set_per_cpu(desc);
|
| 1748 | if (new->flags & IRQF_NO_DEBUG)
|
| 1749 | irq_settings_set_no_debug(desc);
|
| 1750 | }
|
| 1751 |
|
| 1752 | if (noirqdebug)
|
| 1753 | irq_settings_set_no_debug(desc);
|
| 1754 |
|
| 1755 | if (new->flags & IRQF_ONESHOT)
|
| 1756 | desc->istate |= IRQS_ONESHOT;
|
| 1757 |
|
| 1758 |
|
| 1759 | if (new->flags & IRQF_NOBALANCING) {
|
| 1760 | irq_settings_set_no_balancing(desc);
|
| 1761 | irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
|
| 1762 | }
|
| 1763 |
|
| 1764 | if (!(new->flags & IRQF_NO_AUTOEN) &&
|
| 1765 | irq_settings_can_autoenable(desc)) {
|
| 1766 | irq_startup(desc, IRQ_RESEND, IRQ_START_COND);
|
| 1767 | } else if (!per_cpu_devid) {
|
| 1768 | |
| 1769 | * Shared interrupts do not go well with disabling
|
| 1770 | * auto enable. The sharing interrupt might request
|
| 1771 | * it while it's still disabled and then wait for
|
| 1772 | * interrupts forever.
|
| 1773 |
|
| 1774 | WARN_ON_ONCE(new->flags & IRQF_SHARED);
|
| 1775 |
|
| 1776 | desc->depth = 1;
|
| 1777 | }
|
| 1778 |
|
| 1779 | } else if (new->flags & IRQF_TRIGGER_MASK) {
|
| 1780 | unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK;
|
| 1781 | unsigned int omsk = irqd_get_trigger_type(&desc->irq_data);
|
| 1782 |
|
| 1783 | if (nmsk != omsk)
|
| 1784 |
|
| 1785 | pr_warn("irq %d uses trigger mode %u; requested %u\n",
|
| 1786 | irq, omsk, nmsk);
|
| 1787 | }
|
| 1788 |
|
| 1789 | *old_ptr = new;
|
| 1790 |
|
| 1791 | irq_pm_install_action(desc, new);
|
| 1792 |
|
| 1793 |
|
| 1794 | desc->irq_count = 0;
|
| 1795 | desc->irqs_unhandled = 0;
|
| 1796 |
|
| 1797 | |
| 1798 | * Check whether we disabled the irq via the spurious handler
|
| 1799 | * before. Reenable it and give it another chance.
|
| 1800 |
|
| 1801 | if (shared && (desc->istate & IRQS_SPURIOUS_DISABLED)) {
|
| 1802 | desc->istate &= ~IRQS_SPURIOUS_DISABLED;
|
| 1803 | __enable_irq(desc);
|
| 1804 | }
|
| 1805 |
|
| 1806 | irq_proc_update_valid(desc);
|
| 1807 | raw_spin_unlock_irqrestore(&desc->lock, flags);
|
| 1808 | chip_bus_sync_unlock(desc);
|
| 1809 | mutex_unlock(&desc->request_mutex);
|
| 1810 |
|
| 1811 | wake_up_and_wait_for_irq_thread_ready(desc, new);
|
| 1812 | wake_up_and_wait_for_irq_thread_ready(desc, new->secondary);
|
| 1813 |
|
| 1814 | register_irq_proc(irq, desc);
|
| 1815 | new->dir = NULL;
|
| 1816 | register_handler_proc(irq, new);
|
| 1817 | return 0;
|
| 1818 |
|
| 1819 | mismatch:
|
| 1820 | if (!(new->flags & IRQF_PROBE_SHARED)) {
|
| 1821 | pr_err("Flags mismatch irq %d. %08x (%s) vs. %08x (%s)\n",
|
| 1822 | irq, new->flags, new->name, old->flags, old->name);
|
| 1823 | #ifdef CONFIG_DEBUG_SHIRQ
|
| 1824 | dump_stack();
|
| 1825 | #endif
|
| 1826 | }
|
| 1827 | ret = -EBUSY;
|
| 1828 |
|
| 1829 | out_unlock:
|
| 1830 | raw_spin_unlock_irqrestore(&desc->lock, flags);
|
| 1831 |
|
| 1832 | if (!desc->action)
|
| 1833 | irq_release_resources(desc);
|
| 1834 | out_bus_unlock:
|
| 1835 | chip_bus_sync_unlock(desc);
|
| 1836 | mutex_unlock(&desc->request_mutex);
|
| 1837 |
|
| 1838 | out_thread:
|
| 1839 | if (new->thread) {
|
| 1840 | struct task_struct *t = new->thread;
|
| 1841 |
|
| 1842 | new->thread = NULL;
|
| 1843 | kthread_stop_put(t);
|
| 1844 | }
|
| 1845 | if (new->secondary && new->secondary->thread) {
|
| 1846 | struct task_struct *t = new->secondary->thread;
|
| 1847 |
|
| 1848 | new->secondary->thread = NULL;
|
| 1849 | kthread_stop_put(t);
|
| 1850 | }
|
| 1851 | out_mput:
|
| 1852 | module_put(desc->owner);
|
| 1853 | return ret;
|
| 1854 | }
|
| 1855 |
|
| 1856 | |
| 1857 | * Internal function to unregister an irqaction - used to free
|
| 1858 | * regular and special interrupts that are part of the architecture.
|
| 1859 |
|
| 1860 | static struct irqaction *__free_irq(struct irq_desc *desc, void *dev_id)
|
| 1861 | {
|
| 1862 | unsigned irq = desc->irq_data.irq;
|
| 1863 | struct irqaction *action, **action_ptr;
|
| 1864 | unsigned long flags;
|
| 1865 |
|
| 1866 | WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
|
| 1867 |
|
| 1868 | mutex_lock(&desc->request_mutex);
|
| 1869 | chip_bus_lock(desc);
|
| 1870 | raw_spin_lock_irqsave(&desc->lock, flags);
|
| 1871 |
|
| 1872 | |
| 1873 | * There can be multiple actions per IRQ descriptor, find the right
|
| 1874 | * one based on the dev_id:
|
| 1875 |
|
| 1876 | action_ptr = &desc->action;
|
| 1877 | for (;;) {
|
| 1878 | action = *action_ptr;
|
| 1879 |
|
| 1880 | if (!action) {
|
| 1881 | WARN(1, "Trying to free already-free IRQ %d\n", irq);
|
| 1882 | raw_spin_unlock_irqrestore(&desc->lock, flags);
|
| 1883 | chip_bus_sync_unlock(desc);
|
| 1884 | mutex_unlock(&desc->request_mutex);
|
| 1885 | return NULL;
|
| 1886 | }
|
| 1887 |
|
| 1888 | if (action->dev_id == dev_id)
|
| 1889 | break;
|
| 1890 | action_ptr = &action->next;
|
| 1891 | }
|
| 1892 |
|
| 1893 |
|
| 1894 | *action_ptr = action->next;
|
| 1895 |
|
| 1896 | irq_pm_remove_action(desc, action);
|
| 1897 |
|
| 1898 |
|
| 1899 | if (!desc->action) {
|
| 1900 | irq_settings_clr_disable_unlazy(desc);
|
| 1901 |
|
| 1902 | irq_shutdown(desc);
|
| 1903 | }
|
| 1904 |
|
| 1905 | #ifdef CONFIG_SMP
|
| 1906 |
|
| 1907 | if (WARN_ON_ONCE(desc->affinity_hint))
|
| 1908 | desc->affinity_hint = NULL;
|
| 1909 | #endif
|
| 1910 |
|
| 1911 | irq_proc_update_valid(desc);
|
| 1912 | raw_spin_unlock_irqrestore(&desc->lock, flags);
|
| 1913 | |
| 1914 | * Drop bus_lock here so the changes which were done in the chip
|
| 1915 | * callbacks above are synced out to the irq chips which hang
|
| 1916 | * behind a slow bus (I2C, SPI) before calling synchronize_hardirq().
|
| 1917 | *
|
| 1918 | * Aside of that the bus_lock can also be taken from the threaded
|
| 1919 | * handler in irq_finalize_oneshot() which results in a deadlock
|
| 1920 | * because kthread_stop() would wait forever for the thread to
|
| 1921 | * complete, which is blocked on the bus lock.
|
| 1922 | *
|
| 1923 | * The still held desc->request_mutex() protects against a
|
| 1924 | * concurrent request_irq() of this irq so the release of resources
|
| 1925 | * and timing data is properly serialized.
|
| 1926 |
|
| 1927 | chip_bus_sync_unlock(desc);
|
| 1928 |
|
| 1929 | unregister_handler_proc(irq, action);
|
| 1930 |
|
| 1931 | |
| 1932 | * Make sure it's not being used on another CPU and if the chip
|
| 1933 | * supports it also make sure that there is no (not yet serviced)
|
| 1934 | * interrupt in flight at the hardware level.
|
| 1935 |
|
| 1936 | __synchronize_irq(desc);
|
| 1937 |
|
| 1938 | #ifdef CONFIG_DEBUG_SHIRQ
|
| 1939 | |
| 1940 | * It's a shared IRQ -- the driver ought to be prepared for an IRQ
|
| 1941 | * event to happen even now it's being freed, so let's make sure that
|
| 1942 | * is so by doing an extra call to the handler ....
|
| 1943 | *
|
| 1944 | * ( We do this after actually deregistering it, to make sure that a
|
| 1945 | * 'real' IRQ doesn't run in parallel with our fake. )
|
| 1946 |
|
| 1947 | if (action->flags & IRQF_SHARED) {
|
| 1948 | local_irq_save(flags);
|
| 1949 | action->handler(irq, dev_id);
|
| 1950 | local_irq_restore(flags);
|
| 1951 | }
|
| 1952 | #endif
|
| 1953 |
|
| 1954 | |
| 1955 | * The action has already been removed above, but the thread writes
|
| 1956 | * its oneshot mask bit when it completes. Though request_mutex is
|
| 1957 | * held across this which prevents __setup_irq() from handing out
|
| 1958 | * the same bit to a newly requested action.
|
| 1959 |
|
| 1960 | if (action->thread) {
|
| 1961 | kthread_stop_put(action->thread);
|
| 1962 | if (action->secondary && action->secondary->thread)
|
| 1963 | kthread_stop_put(action->secondary->thread);
|
| 1964 | }
|
| 1965 |
|
| 1966 |
|
| 1967 | if (!desc->action) {
|
| 1968 | |
| 1969 | * Reacquire bus lock as irq_release_resources() might
|
| 1970 | * require it to deallocate resources over the slow bus.
|
| 1971 |
|
| 1972 | chip_bus_lock(desc);
|
| 1973 | |
| 1974 | * There is no interrupt on the fly anymore. Deactivate it
|
| 1975 | * completely.
|
| 1976 |
|
| 1977 | scoped_guard(raw_spinlock_irqsave, &desc->lock)
|
| 1978 | irq_domain_deactivate_irq(&desc->irq_data);
|
| 1979 |
|
| 1980 | irq_release_resources(desc);
|
| 1981 | chip_bus_sync_unlock(desc);
|
| 1982 | }
|
| 1983 |
|
| 1984 | mutex_unlock(&desc->request_mutex);
|
| 1985 |
|
| 1986 | irq_chip_pm_put(&desc->irq_data);
|
| 1987 | module_put(desc->owner);
|
| 1988 | kfree(action->secondary);
|
| 1989 | return action;
|
| 1990 | }
|
| 1991 |
|
| 1992 | |
| 1993 | * free_irq - free an interrupt allocated with request_irq
|
| 1994 | * @irq: Interrupt line to free
|
| 1995 | * @dev_id: Device identity to free
|
| 1996 | *
|
| 1997 | * Remove an interrupt handler. The handler is removed and if the interrupt
|
| 1998 | * line is no longer in use by any driver it is disabled. On a shared IRQ
|
| 1999 | * the caller must ensure the interrupt is disabled on the card it drives
|
| 2000 | * before calling this function. The function does not return until any
|
| 2001 | * executing interrupts for this IRQ have completed.
|
| 2002 | *
|
| 2003 | * This function must not be called from interrupt context.
|
| 2004 | *
|
| 2005 | * Returns the devname argument passed to request_irq.
|
| 2006 |
|
| 2007 | const void *free_irq(unsigned int irq, void *dev_id)
|
| 2008 | {
|
| 2009 | struct irq_desc *desc = irq_to_desc(irq);
|
| 2010 | struct irqaction *action;
|
| 2011 | const char *devname;
|
| 2012 |
|
| 2013 | if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
|
| 2014 | return NULL;
|
| 2015 |
|
| 2016 | #ifdef CONFIG_SMP
|
| 2017 | if (WARN_ON(desc->affinity_notify))
|
| 2018 | desc->affinity_notify = NULL;
|
| 2019 | #endif
|
| 2020 |
|
| 2021 | action = __free_irq(desc, dev_id);
|
| 2022 |
|
| 2023 | if (!action)
|
| 2024 | return NULL;
|
| 2025 |
|
| 2026 | devname = action->name;
|
| 2027 | kfree(action);
|
| 2028 | return devname;
|
| 2029 | }
|
| 2030 | EXPORT_SYMBOL(free_irq);
|
| 2031 |
|
| 2032 | static const void *__cleanup_nmi(unsigned int irq, struct irq_desc *desc)
|
| 2033 | {
|
| 2034 | struct irqaction *action = NULL;
|
| 2035 | const char *devname = NULL;
|
| 2036 |
|
| 2037 | scoped_guard(raw_spinlock_irqsave, &desc->lock) {
|
| 2038 | irq_nmi_teardown(desc);
|
| 2039 |
|
| 2040 | desc->istate &= ~IRQS_NMI;
|
| 2041 |
|
| 2042 | if (!WARN_ON(desc->action == NULL)) {
|
| 2043 | action = desc->action;
|
| 2044 | irq_pm_remove_action(desc, action);
|
| 2045 | devname = action->name;
|
| 2046 | }
|
| 2047 | desc->action = NULL;
|
| 2048 |
|
| 2049 | irq_settings_clr_disable_unlazy(desc);
|
| 2050 | irq_shutdown_and_deactivate(desc);
|
| 2051 | }
|
| 2052 |
|
| 2053 | irq_proc_update_valid(desc);
|
| 2054 |
|
| 2055 | if (action)
|
| 2056 | unregister_handler_proc(irq, action);
|
| 2057 | kfree(action);
|
| 2058 |
|
| 2059 | irq_release_resources(desc);
|
| 2060 |
|
| 2061 | irq_chip_pm_put(&desc->irq_data);
|
| 2062 | module_put(desc->owner);
|
| 2063 |
|
| 2064 | return devname;
|
| 2065 | }
|
| 2066 |
|
| 2067 | const void *free_nmi(unsigned int irq, void *dev_id)
|
| 2068 | {
|
| 2069 | struct irq_desc *desc = irq_to_desc(irq);
|
| 2070 |
|
| 2071 | if (!desc || WARN_ON(!irq_is_nmi(desc)))
|
| 2072 | return NULL;
|
| 2073 |
|
| 2074 | if (WARN_ON(irq_settings_is_per_cpu_devid(desc)))
|
| 2075 | return NULL;
|
| 2076 |
|
| 2077 |
|
| 2078 | if (WARN_ON(desc->depth == 0))
|
| 2079 | disable_nmi_nosync(irq);
|
| 2080 |
|
| 2081 | return __cleanup_nmi(irq, desc);
|
| 2082 | }
|
| 2083 |
|
| 2084 | |
| 2085 | * request_threaded_irq - allocate an interrupt line
|
| 2086 | * @irq: Interrupt line to allocate
|
| 2087 | * @handler: Function to be called when the IRQ occurs.
|
| 2088 | * Primary handler for threaded interrupts.
|
| 2089 | * If handler is NULL and thread_fn != NULL
|
| 2090 | * the default primary handler is installed.
|
| 2091 | * @thread_fn: Function called from the irq handler thread
|
| 2092 | * If NULL, no irq thread is created
|
| 2093 | * @irqflags: Interrupt type flags
|
| 2094 | * @devname: An ascii name for the claiming device
|
| 2095 | * @dev_id: A cookie passed back to the handler function
|
| 2096 | *
|
| 2097 | * This call allocates interrupt resources and enables the interrupt line
|
| 2098 | * and IRQ handling. From the point this call is made your handler function
|
| 2099 | * may be invoked. Since your handler function must clear any interrupt the
|
| 2100 | * board raises, you must take care both to initialise your hardware and to
|
| 2101 | * set up the interrupt handler in the right order.
|
| 2102 | *
|
| 2103 | * If you want to set up a threaded irq handler for your device then you
|
| 2104 | * need to supply @handler and @thread_fn. @handler is still called in hard
|
| 2105 | * interrupt context and has to check whether the interrupt originates from
|
| 2106 | * the device. If yes it needs to disable the interrupt on the device and
|
| 2107 | * return IRQ_WAKE_THREAD which will wake up the handler thread and run
|
| 2108 | * @thread_fn. This split handler design is necessary to support shared
|
| 2109 | * interrupts.
|
| 2110 | *
|
| 2111 | * @dev_id must be globally unique. Normally the address of the device data
|
| 2112 | * structure is used as the cookie. Since the handler receives this value
|
| 2113 | * it makes sense to use it.
|
| 2114 | *
|
| 2115 | * If your interrupt is shared you must pass a non NULL dev_id as this is
|
| 2116 | * required when freeing the interrupt.
|
| 2117 | *
|
| 2118 | * Flags:
|
| 2119 | *
|
| 2120 | * IRQF_SHARED Interrupt is shared
|
| 2121 | * IRQF_TRIGGER_* Specify active edge(s) or level
|
| 2122 | * IRQF_ONESHOT Run thread_fn with interrupt line masked
|
| 2123 |
|
| 2124 | int request_threaded_irq(unsigned int irq, irq_handler_t handler,
|
| 2125 | irq_handler_t thread_fn, unsigned long irqflags,
|
| 2126 | const char *devname, void *dev_id)
|
| 2127 | {
|
| 2128 | struct irqaction *action;
|
| 2129 | struct irq_desc *desc;
|
| 2130 | int retval;
|
| 2131 |
|
| 2132 | if (irq == IRQ_NOTCONNECTED)
|
| 2133 | return -ENOTCONN;
|
| 2134 |
|
| 2135 | |
| 2136 | * Sanity-check: shared interrupts must pass in a real dev-ID,
|
| 2137 | * otherwise we'll have trouble later trying to figure out
|
| 2138 | * which interrupt is which (messes up the interrupt freeing
|
| 2139 | * logic etc).
|
| 2140 | *
|
| 2141 | * Also shared interrupts do not go well with disabling auto enable.
|
| 2142 | * The sharing interrupt might request it while it's still disabled
|
| 2143 | * and then wait for interrupts forever.
|
| 2144 | *
|
| 2145 | * Also IRQF_COND_SUSPEND only makes sense for shared interrupts and
|
| 2146 | * it cannot be set along with IRQF_NO_SUSPEND.
|
| 2147 |
|
| 2148 | if (((irqflags & IRQF_SHARED) && !dev_id) ||
|
| 2149 | ((irqflags & IRQF_SHARED) && (irqflags & IRQF_NO_AUTOEN)) ||
|
| 2150 | (!(irqflags & IRQF_SHARED) && (irqflags & IRQF_COND_SUSPEND)) ||
|
| 2151 | ((irqflags & IRQF_NO_SUSPEND) && (irqflags & IRQF_COND_SUSPEND)))
|
| 2152 | return -EINVAL;
|
| 2153 |
|
| 2154 | desc = irq_to_desc(irq);
|
| 2155 | if (!desc)
|
| 2156 | return -EINVAL;
|
| 2157 |
|
| 2158 | if (!irq_settings_can_request(desc) ||
|
| 2159 | WARN_ON(irq_settings_is_per_cpu_devid(desc)))
|
| 2160 | return -EINVAL;
|
| 2161 |
|
| 2162 | if (!handler) {
|
| 2163 | if (!thread_fn)
|
| 2164 | return -EINVAL;
|
| 2165 | handler = irq_default_primary_handler;
|
| 2166 | }
|
| 2167 |
|
| 2168 | action = kzalloc_obj(struct irqaction);
|
| 2169 | if (!action)
|
| 2170 | return -ENOMEM;
|
| 2171 |
|
| 2172 | action->handler = handler;
|
| 2173 | action->thread_fn = thread_fn;
|
| 2174 | action->flags = irqflags;
|
| 2175 | action->name = devname;
|
| 2176 | action->dev_id = dev_id;
|
| 2177 |
|
| 2178 | retval = irq_chip_pm_get(&desc->irq_data);
|
| 2179 | if (retval < 0) {
|
| 2180 | kfree(action);
|
| 2181 | return retval;
|
| 2182 | }
|
| 2183 |
|
| 2184 | retval = __setup_irq(irq, desc, action);
|
| 2185 |
|
| 2186 | if (retval) {
|
| 2187 | irq_chip_pm_put(&desc->irq_data);
|
| 2188 | kfree(action->secondary);
|
| 2189 | kfree(action);
|
| 2190 | }
|
| 2191 |
|
| 2192 | #ifdef CONFIG_DEBUG_SHIRQ_FIXME
|
| 2193 | if (!retval && (irqflags & IRQF_SHARED)) {
|
| 2194 | |
| 2195 | * It's a shared IRQ -- the driver ought to be prepared for it
|
| 2196 | * to happen immediately, so let's make sure....
|
| 2197 | * We disable the irq to make sure that a 'real' IRQ doesn't
|
| 2198 | * run in parallel with our fake.
|
| 2199 |
|
| 2200 | unsigned long flags;
|
| 2201 |
|
| 2202 | disable_irq(irq);
|
| 2203 | local_irq_save(flags);
|
| 2204 |
|
| 2205 | handler(irq, dev_id);
|
| 2206 |
|
| 2207 | local_irq_restore(flags);
|
| 2208 | enable_irq(irq);
|
| 2209 | }
|
| 2210 | #endif
|
| 2211 | return retval;
|
| 2212 | }
|
| 2213 | EXPORT_SYMBOL(request_threaded_irq);
|
| 2214 |
|
| 2215 | |
| 2216 | * request_any_context_irq - allocate an interrupt line
|
| 2217 | * @irq: Interrupt line to allocate
|
| 2218 | * @handler: Function to be called when the IRQ occurs.
|
| 2219 | * Threaded handler for threaded interrupts.
|
| 2220 | * @flags: Interrupt type flags
|
| 2221 | * @name: An ascii name for the claiming device
|
| 2222 | * @dev_id: A cookie passed back to the handler function
|
| 2223 | *
|
| 2224 | * This call allocates interrupt resources and enables the interrupt line
|
| 2225 | * and IRQ handling. It selects either a hardirq or threaded handling
|
| 2226 | * method depending on the context.
|
| 2227 | *
|
| 2228 | * Returns: On failure, it returns a negative value. On success, it returns either
|
| 2229 | * IRQC_IS_HARDIRQ or IRQC_IS_NESTED.
|
| 2230 |
|
| 2231 | int request_any_context_irq(unsigned int irq, irq_handler_t handler,
|
| 2232 | unsigned long flags, const char *name, void *dev_id)
|
| 2233 | {
|
| 2234 | struct irq_desc *desc;
|
| 2235 | int ret;
|
| 2236 |
|
| 2237 | if (irq == IRQ_NOTCONNECTED)
|
| 2238 | return -ENOTCONN;
|
| 2239 |
|
| 2240 | desc = irq_to_desc(irq);
|
| 2241 | if (!desc)
|
| 2242 | return -EINVAL;
|
| 2243 |
|
| 2244 | if (irq_settings_is_nested_thread(desc)) {
|
| 2245 | ret = request_threaded_irq(irq, NULL, handler,
|
| 2246 | flags, name, dev_id);
|
| 2247 | return !ret ? IRQC_IS_NESTED : ret;
|
| 2248 | }
|
| 2249 |
|
| 2250 | ret = request_irq(irq, handler, flags, name, dev_id);
|
| 2251 | return !ret ? IRQC_IS_HARDIRQ : ret;
|
| 2252 | }
|
| 2253 | EXPORT_SYMBOL_GPL(request_any_context_irq);
|
| 2254 |
|
| 2255 | |
| 2256 | * request_nmi - allocate an interrupt line for NMI delivery
|
| 2257 | * @irq: Interrupt line to allocate
|
| 2258 | * @handler: Function to be called when the IRQ occurs.
|
| 2259 | * Threaded handler for threaded interrupts.
|
| 2260 | * @irqflags: Interrupt type flags
|
| 2261 | * @name: An ascii name for the claiming device
|
| 2262 | * @dev_id: A cookie passed back to the handler function
|
| 2263 | *
|
| 2264 | * This call allocates interrupt resources and enables the interrupt line
|
| 2265 | * and IRQ handling. It sets up the IRQ line to be handled as an NMI.
|
| 2266 | *
|
| 2267 | * An interrupt line delivering NMIs cannot be shared and IRQ handling
|
| 2268 | * cannot be threaded.
|
| 2269 | *
|
| 2270 | * Interrupt lines requested for NMI delivering must produce per cpu
|
| 2271 | * interrupts and have auto enabling setting disabled.
|
| 2272 | *
|
| 2273 | * @dev_id must be globally unique. Normally the address of the device data
|
| 2274 | * structure is used as the cookie. Since the handler receives this value
|
| 2275 | * it makes sense to use it.
|
| 2276 | *
|
| 2277 | * If the interrupt line cannot be used to deliver NMIs, function will fail
|
| 2278 | * and return a negative value.
|
| 2279 |
|
| 2280 | int request_nmi(unsigned int irq, irq_handler_t handler,
|
| 2281 | unsigned long irqflags, const char *name, void *dev_id)
|
| 2282 | {
|
| 2283 | struct irqaction *action;
|
| 2284 | struct irq_desc *desc;
|
| 2285 | int retval;
|
| 2286 |
|
| 2287 | if (irq == IRQ_NOTCONNECTED)
|
| 2288 | return -ENOTCONN;
|
| 2289 |
|
| 2290 |
|
| 2291 | if (irqflags & (IRQF_SHARED | IRQF_COND_SUSPEND | IRQF_IRQPOLL))
|
| 2292 | return -EINVAL;
|
| 2293 |
|
| 2294 | if (!(irqflags & IRQF_PERCPU))
|
| 2295 | return -EINVAL;
|
| 2296 |
|
| 2297 | if (!handler)
|
| 2298 | return -EINVAL;
|
| 2299 |
|
| 2300 | desc = irq_to_desc(irq);
|
| 2301 |
|
| 2302 | if (!desc || (irq_settings_can_autoenable(desc) &&
|
| 2303 | !(irqflags & IRQF_NO_AUTOEN)) ||
|
| 2304 | !irq_settings_can_request(desc) ||
|
| 2305 | WARN_ON(irq_settings_is_per_cpu_devid(desc)) ||
|
| 2306 | !irq_supports_nmi(desc))
|
| 2307 | return -EINVAL;
|
| 2308 |
|
| 2309 | action = kzalloc_obj(struct irqaction);
|
| 2310 | if (!action)
|
| 2311 | return -ENOMEM;
|
| 2312 |
|
| 2313 | action->handler = handler;
|
| 2314 | action->flags = irqflags | IRQF_NO_THREAD | IRQF_NOBALANCING;
|
| 2315 | action->name = name;
|
| 2316 | action->dev_id = dev_id;
|
| 2317 |
|
| 2318 | retval = irq_chip_pm_get(&desc->irq_data);
|
| 2319 | if (retval < 0)
|
| 2320 | goto err_out;
|
| 2321 |
|
| 2322 | retval = __setup_irq(irq, desc, action);
|
| 2323 | if (retval)
|
| 2324 | goto err_irq_setup;
|
| 2325 |
|
| 2326 | scoped_guard(raw_spinlock_irqsave, &desc->lock) {
|
| 2327 |
|
| 2328 | desc->istate |= IRQS_NMI;
|
| 2329 | retval = irq_nmi_setup(desc);
|
| 2330 | }
|
| 2331 |
|
| 2332 | if (retval) {
|
| 2333 | __cleanup_nmi(irq, desc);
|
| 2334 | return -EINVAL;
|
| 2335 | }
|
| 2336 | return 0;
|
| 2337 |
|
| 2338 | err_irq_setup:
|
| 2339 | irq_chip_pm_put(&desc->irq_data);
|
| 2340 | err_out:
|
| 2341 | kfree(action);
|
| 2342 |
|
| 2343 | return retval;
|
| 2344 | }
|
| 2345 |
|
| 2346 | void enable_percpu_irq(unsigned int irq, unsigned int type)
|
| 2347 | {
|
| 2348 | scoped_irqdesc_get_and_lock(irq, IRQ_GET_DESC_CHECK_PERCPU) {
|
| 2349 | struct irq_desc *desc = scoped_irqdesc;
|
| 2350 |
|
| 2351 | |
| 2352 | * If the trigger type is not specified by the caller, then
|
| 2353 | * use the default for this interrupt.
|
| 2354 |
|
| 2355 | type &= IRQ_TYPE_SENSE_MASK;
|
| 2356 | if (type == IRQ_TYPE_NONE)
|
| 2357 | type = irqd_get_trigger_type(&desc->irq_data);
|
| 2358 |
|
| 2359 | if (type != IRQ_TYPE_NONE) {
|
| 2360 | if (__irq_set_trigger(desc, type)) {
|
| 2361 | WARN(1, "failed to set type for IRQ%d\n", irq);
|
| 2362 | return;
|
| 2363 | }
|
| 2364 | }
|
| 2365 | irq_percpu_enable(desc, smp_processor_id());
|
| 2366 | }
|
| 2367 | }
|
| 2368 | EXPORT_SYMBOL_GPL(enable_percpu_irq);
|
| 2369 |
|
| 2370 | void enable_percpu_nmi(unsigned int irq, unsigned int type)
|
| 2371 | {
|
| 2372 | enable_percpu_irq(irq, type);
|
| 2373 | }
|
| 2374 |
|
| 2375 | |
| 2376 | * irq_percpu_is_enabled - Check whether the per cpu irq is enabled
|
| 2377 | * @irq: Linux irq number to check for
|
| 2378 | *
|
| 2379 | * Must be called from a non migratable context. Returns the enable
|
| 2380 | * state of a per cpu interrupt on the current cpu.
|
| 2381 |
|
| 2382 | bool irq_percpu_is_enabled(unsigned int irq)
|
| 2383 | {
|
| 2384 | scoped_irqdesc_get_and_lock(irq, IRQ_GET_DESC_CHECK_PERCPU)
|
| 2385 | return cpumask_test_cpu(smp_processor_id(), scoped_irqdesc->percpu_enabled);
|
| 2386 | return false;
|
| 2387 | }
|
| 2388 | EXPORT_SYMBOL_GPL(irq_percpu_is_enabled);
|
| 2389 |
|
| 2390 | void disable_percpu_irq(unsigned int irq)
|
| 2391 | {
|
| 2392 | scoped_irqdesc_get_and_lock(irq, IRQ_GET_DESC_CHECK_PERCPU)
|
| 2393 | irq_percpu_disable(scoped_irqdesc, smp_processor_id());
|
| 2394 | }
|
| 2395 | EXPORT_SYMBOL_GPL(disable_percpu_irq);
|
| 2396 |
|
| 2397 | void disable_percpu_nmi(unsigned int irq)
|
| 2398 | {
|
| 2399 | disable_percpu_irq(irq);
|
| 2400 | }
|
| 2401 |
|
| 2402 | |
| 2403 | * Internal function to unregister a percpu irqaction.
|
| 2404 |
|
| 2405 | static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_id)
|
| 2406 | {
|
| 2407 | struct irq_desc *desc = irq_to_desc(irq);
|
| 2408 | struct irqaction *action, **action_ptr;
|
| 2409 |
|
| 2410 | WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
|
| 2411 |
|
| 2412 | if (!desc)
|
| 2413 | return NULL;
|
| 2414 |
|
| 2415 | scoped_guard(raw_spinlock_irqsave, &desc->lock) {
|
| 2416 | action_ptr = &desc->action;
|
| 2417 | for (;;) {
|
| 2418 | action = *action_ptr;
|
| 2419 |
|
| 2420 | if (!action) {
|
| 2421 | WARN(1, "Trying to free already-free IRQ %d\n", irq);
|
| 2422 | return NULL;
|
| 2423 | }
|
| 2424 |
|
| 2425 | if (action->percpu_dev_id == dev_id)
|
| 2426 | break;
|
| 2427 |
|
| 2428 | action_ptr = &action->next;
|
| 2429 | }
|
| 2430 |
|
| 2431 | if (cpumask_intersects(desc->percpu_enabled, action->affinity)) {
|
| 2432 | WARN(1, "percpu IRQ %d still enabled on CPU%d!\n", irq,
|
| 2433 | cpumask_first_and(desc->percpu_enabled, action->affinity));
|
| 2434 | return NULL;
|
| 2435 | }
|
| 2436 |
|
| 2437 |
|
| 2438 | *action_ptr = action->next;
|
| 2439 |
|
| 2440 |
|
| 2441 | if (!desc->action) {
|
| 2442 | desc->istate &= ~IRQS_NMI;
|
| 2443 | irq_proc_update_valid(desc);
|
| 2444 | }
|
| 2445 | }
|
| 2446 |
|
| 2447 | unregister_handler_proc(irq, action);
|
| 2448 | irq_chip_pm_put(&desc->irq_data);
|
| 2449 | module_put(desc->owner);
|
| 2450 | return action;
|
| 2451 | }
|
| 2452 |
|
| 2453 | |
| 2454 | * free_percpu_irq - free an interrupt allocated with request_percpu_irq
|
| 2455 | * @irq: Interrupt line to free
|
| 2456 | * @dev_id: Device identity to free
|
| 2457 | *
|
| 2458 | * Remove a percpu interrupt handler. The handler is removed, but the
|
| 2459 | * interrupt line is not disabled. This must be done on each CPU before
|
| 2460 | * calling this function. The function does not return until any executing
|
| 2461 | * interrupts for this IRQ have completed.
|
| 2462 | *
|
| 2463 | * This function must not be called from interrupt context.
|
| 2464 |
|
| 2465 | void free_percpu_irq(unsigned int irq, void __percpu *dev_id)
|
| 2466 | {
|
| 2467 | struct irq_desc *desc = irq_to_desc(irq);
|
| 2468 |
|
| 2469 | if (!desc || !irq_settings_is_per_cpu_devid(desc))
|
| 2470 | return;
|
| 2471 |
|
| 2472 | chip_bus_lock(desc);
|
| 2473 | kfree(__free_percpu_irq(irq, dev_id));
|
| 2474 | chip_bus_sync_unlock(desc);
|
| 2475 | }
|
| 2476 | EXPORT_SYMBOL_GPL(free_percpu_irq);
|
| 2477 |
|
| 2478 | void free_percpu_nmi(unsigned int irq, void __percpu *dev_id)
|
| 2479 | {
|
| 2480 | struct irq_desc *desc = irq_to_desc(irq);
|
| 2481 |
|
| 2482 | if (!desc || !irq_settings_is_per_cpu_devid(desc))
|
| 2483 | return;
|
| 2484 |
|
| 2485 | if (WARN_ON(!irq_is_nmi(desc)))
|
| 2486 | return;
|
| 2487 |
|
| 2488 | kfree(__free_percpu_irq(irq, dev_id));
|
| 2489 | }
|
| 2490 |
|
| 2491 | static
|
| 2492 | struct irqaction *create_percpu_irqaction(irq_handler_t handler, unsigned long flags,
|
| 2493 | const char *devname, const cpumask_t *affinity,
|
| 2494 | void __percpu *dev_id)
|
| 2495 | {
|
| 2496 | struct irqaction *action;
|
| 2497 |
|
| 2498 | if (!affinity)
|
| 2499 | affinity = cpu_possible_mask;
|
| 2500 |
|
| 2501 | action = kzalloc_obj(struct irqaction);
|
| 2502 | if (!action)
|
| 2503 | return NULL;
|
| 2504 |
|
| 2505 | action->handler = handler;
|
| 2506 | action->flags = flags | IRQF_PERCPU | IRQF_NO_SUSPEND;
|
| 2507 | action->name = devname;
|
| 2508 | action->percpu_dev_id = dev_id;
|
| 2509 | action->affinity = affinity;
|
| 2510 |
|
| 2511 | |
| 2512 | * We allow some form of sharing for non-overlapping affinity
|
| 2513 | * masks. Obviously, covering all CPUs prevents any sharing in
|
| 2514 | * the first place.
|
| 2515 |
|
| 2516 | if (!cpumask_equal(affinity, cpu_possible_mask))
|
| 2517 | action->flags |= IRQF_SHARED;
|
| 2518 |
|
| 2519 | return action;
|
| 2520 | }
|
| 2521 |
|
| 2522 | |
| 2523 | * request_percpu_irq_affinity - allocate a percpu interrupt line
|
| 2524 | * @irq: Interrupt line to allocate
|
| 2525 | * @handler: Function to be called when the IRQ occurs.
|
| 2526 | * @devname: An ascii name for the claiming device
|
| 2527 | * @affinity: A cpumask describing the target CPUs for this interrupt
|
| 2528 | * @dev_id: A percpu cookie passed back to the handler function
|
| 2529 | *
|
| 2530 | * This call allocates interrupt resources, but doesn't enable the interrupt
|
| 2531 | * on any CPU, as all percpu-devid interrupts are flagged with IRQ_NOAUTOEN.
|
| 2532 | * It has to be done on each CPU using enable_percpu_irq().
|
| 2533 | *
|
| 2534 | * @dev_id must be globally unique. It is a per-cpu variable, and
|
| 2535 | * the handler gets called with the interrupted CPU's instance of
|
| 2536 | * that variable.
|
| 2537 |
|
| 2538 | int request_percpu_irq_affinity(unsigned int irq, irq_handler_t handler, const char *devname,
|
| 2539 | const cpumask_t *affinity, void __percpu *dev_id)
|
| 2540 | {
|
| 2541 | struct irqaction *action;
|
| 2542 | struct irq_desc *desc;
|
| 2543 | int retval;
|
| 2544 |
|
| 2545 | if (!dev_id)
|
| 2546 | return -EINVAL;
|
| 2547 |
|
| 2548 | desc = irq_to_desc(irq);
|
| 2549 | if (!desc || !irq_settings_can_request(desc) ||
|
| 2550 | !irq_settings_is_per_cpu_devid(desc))
|
| 2551 | return -EINVAL;
|
| 2552 |
|
| 2553 | action = create_percpu_irqaction(handler, 0, devname, affinity, dev_id);
|
| 2554 | if (!action)
|
| 2555 | return -ENOMEM;
|
| 2556 |
|
| 2557 | retval = irq_chip_pm_get(&desc->irq_data);
|
| 2558 | if (retval < 0) {
|
| 2559 | kfree(action);
|
| 2560 | return retval;
|
| 2561 | }
|
| 2562 |
|
| 2563 | retval = __setup_irq(irq, desc, action);
|
| 2564 |
|
| 2565 | if (retval) {
|
| 2566 | irq_chip_pm_put(&desc->irq_data);
|
| 2567 | kfree(action);
|
| 2568 | }
|
| 2569 |
|
| 2570 | return retval;
|
| 2571 | }
|
| 2572 | EXPORT_SYMBOL_GPL(request_percpu_irq_affinity);
|
| 2573 |
|
| 2574 | |
| 2575 | * request_percpu_nmi - allocate a percpu interrupt line for NMI delivery
|
| 2576 | * @irq: Interrupt line to allocate
|
| 2577 | * @handler: Function to be called when the IRQ occurs.
|
| 2578 | * @name: An ascii name for the claiming device
|
| 2579 | * @affinity: A cpumask describing the target CPUs for this interrupt
|
| 2580 | * @dev_id: A percpu cookie passed back to the handler function
|
| 2581 | *
|
| 2582 | * This call allocates interrupt resources for a per CPU NMI. Per CPU NMIs
|
| 2583 | * have to be setup on each CPU by calling prepare_percpu_nmi() before
|
| 2584 | * being enabled on the same CPU by using enable_percpu_nmi().
|
| 2585 | *
|
| 2586 | * @dev_id must be globally unique. It is a per-cpu variable, and the
|
| 2587 | * handler gets called with the interrupted CPU's instance of that
|
| 2588 | * variable.
|
| 2589 | *
|
| 2590 | * Interrupt lines requested for NMI delivering should have auto enabling
|
| 2591 | * setting disabled.
|
| 2592 | *
|
| 2593 | * If the interrupt line cannot be used to deliver NMIs, function
|
| 2594 | * will fail returning a negative value.
|
| 2595 |
|
| 2596 | int request_percpu_nmi(unsigned int irq, irq_handler_t handler, const char *name,
|
| 2597 | const struct cpumask *affinity, void __percpu *dev_id)
|
| 2598 | {
|
| 2599 | struct irqaction *action;
|
| 2600 | struct irq_desc *desc;
|
| 2601 | int retval;
|
| 2602 |
|
| 2603 | if (!handler)
|
| 2604 | return -EINVAL;
|
| 2605 |
|
| 2606 | desc = irq_to_desc(irq);
|
| 2607 |
|
| 2608 | if (!desc || !irq_settings_can_request(desc) ||
|
| 2609 | !irq_settings_is_per_cpu_devid(desc) ||
|
| 2610 | irq_settings_can_autoenable(desc) ||
|
| 2611 | !irq_supports_nmi(desc))
|
| 2612 | return -EINVAL;
|
| 2613 |
|
| 2614 |
|
| 2615 | if (irq_is_nmi(desc) &&
|
| 2616 | (!affinity || cpumask_equal(affinity, cpu_possible_mask)))
|
| 2617 | return -EINVAL;
|
| 2618 |
|
| 2619 | action = create_percpu_irqaction(handler, IRQF_NO_THREAD | IRQF_NOBALANCING,
|
| 2620 | name, affinity, dev_id);
|
| 2621 | if (!action)
|
| 2622 | return -ENOMEM;
|
| 2623 |
|
| 2624 | retval = irq_chip_pm_get(&desc->irq_data);
|
| 2625 | if (retval < 0)
|
| 2626 | goto err_out;
|
| 2627 |
|
| 2628 | retval = __setup_irq(irq, desc, action);
|
| 2629 | if (retval)
|
| 2630 | goto err_irq_setup;
|
| 2631 |
|
| 2632 | scoped_guard(raw_spinlock_irqsave, &desc->lock)
|
| 2633 | desc->istate |= IRQS_NMI;
|
| 2634 | return 0;
|
| 2635 |
|
| 2636 | err_irq_setup:
|
| 2637 | irq_chip_pm_put(&desc->irq_data);
|
| 2638 | err_out:
|
| 2639 | kfree(action);
|
| 2640 |
|
| 2641 | return retval;
|
| 2642 | }
|
| 2643 |
|
| 2644 | |
| 2645 | * prepare_percpu_nmi - performs CPU local setup for NMI delivery
|
| 2646 | * @irq: Interrupt line to prepare for NMI delivery
|
| 2647 | *
|
| 2648 | * This call prepares an interrupt line to deliver NMI on the current CPU,
|
| 2649 | * before that interrupt line gets enabled with enable_percpu_nmi().
|
| 2650 | *
|
| 2651 | * As a CPU local operation, this should be called from non-preemptible
|
| 2652 | * context.
|
| 2653 | *
|
| 2654 | * If the interrupt line cannot be used to deliver NMIs, function will fail
|
| 2655 | * returning a negative value.
|
| 2656 |
|
| 2657 | int prepare_percpu_nmi(unsigned int irq)
|
| 2658 | {
|
| 2659 | int ret = -EINVAL;
|
| 2660 |
|
| 2661 | WARN_ON(preemptible());
|
| 2662 |
|
| 2663 | scoped_irqdesc_get_and_lock(irq, IRQ_GET_DESC_CHECK_PERCPU) {
|
| 2664 | if (WARN(!irq_is_nmi(scoped_irqdesc),
|
| 2665 | "prepare_percpu_nmi called for a non-NMI interrupt: irq %u\n", irq))
|
| 2666 | return -EINVAL;
|
| 2667 |
|
| 2668 | ret = irq_nmi_setup(scoped_irqdesc);
|
| 2669 | if (ret)
|
| 2670 | pr_err("Failed to setup NMI delivery: irq %u\n", irq);
|
| 2671 | }
|
| 2672 | return ret;
|
| 2673 | }
|
| 2674 |
|
| 2675 | |
| 2676 | * teardown_percpu_nmi - undoes NMI setup of IRQ line
|
| 2677 | * @irq: Interrupt line from which CPU local NMI configuration should be removed
|
| 2678 | *
|
| 2679 | * This call undoes the setup done by prepare_percpu_nmi().
|
| 2680 | *
|
| 2681 | * IRQ line should not be enabled for the current CPU.
|
| 2682 | * As a CPU local operation, this should be called from non-preemptible
|
| 2683 | * context.
|
| 2684 |
|
| 2685 | void teardown_percpu_nmi(unsigned int irq)
|
| 2686 | {
|
| 2687 | WARN_ON(preemptible());
|
| 2688 |
|
| 2689 | scoped_irqdesc_get_and_lock(irq, IRQ_GET_DESC_CHECK_PERCPU) {
|
| 2690 | if (WARN_ON(!irq_is_nmi(scoped_irqdesc)))
|
| 2691 | return;
|
| 2692 | irq_nmi_teardown(scoped_irqdesc);
|
| 2693 | }
|
| 2694 | }
|
| 2695 |
|
| 2696 | static int __irq_get_irqchip_state(struct irq_data *data, enum irqchip_irq_state which, bool *state)
|
| 2697 | {
|
| 2698 | struct irq_chip *chip;
|
| 2699 | int err = -EINVAL;
|
| 2700 |
|
| 2701 | do {
|
| 2702 | chip = irq_data_get_irq_chip(data);
|
| 2703 | if (WARN_ON_ONCE(!chip))
|
| 2704 | return -ENODEV;
|
| 2705 | if (chip->irq_get_irqchip_state)
|
| 2706 | break;
|
| 2707 | data = irqd_get_parent_data(data);
|
| 2708 | } while (data);
|
| 2709 |
|
| 2710 | if (data)
|
| 2711 | err = chip->irq_get_irqchip_state(data, which, state);
|
| 2712 | return err;
|
| 2713 | }
|
| 2714 |
|
| 2715 | |
| 2716 | * irq_get_irqchip_state - returns the irqchip state of a interrupt.
|
| 2717 | * @irq: Interrupt line that is forwarded to a VM
|
| 2718 | * @which: One of IRQCHIP_STATE_* the caller wants to know about
|
| 2719 | * @state: a pointer to a boolean where the state is to be stored
|
| 2720 | *
|
| 2721 | * This call snapshots the internal irqchip state of an interrupt,
|
| 2722 | * returning into @state the bit corresponding to stage @which
|
| 2723 | *
|
| 2724 | * This function should be called with preemption disabled if the interrupt
|
| 2725 | * controller has per-cpu registers.
|
| 2726 |
|
| 2727 | int irq_get_irqchip_state(unsigned int irq, enum irqchip_irq_state which, bool *state)
|
| 2728 | {
|
| 2729 | scoped_irqdesc_get_and_buslock(irq, 0) {
|
| 2730 | struct irq_data *data = irq_desc_get_irq_data(scoped_irqdesc);
|
| 2731 |
|
| 2732 | return __irq_get_irqchip_state(data, which, state);
|
| 2733 | }
|
| 2734 | return -EINVAL;
|
| 2735 | }
|
| 2736 | EXPORT_SYMBOL_GPL(irq_get_irqchip_state);
|
| 2737 |
|
| 2738 | |
| 2739 | * irq_set_irqchip_state - set the state of a forwarded interrupt.
|
| 2740 | * @irq: Interrupt line that is forwarded to a VM
|
| 2741 | * @which: State to be restored (one of IRQCHIP_STATE_*)
|
| 2742 | * @val: Value corresponding to @which
|
| 2743 | *
|
| 2744 | * This call sets the internal irqchip state of an interrupt, depending on
|
| 2745 | * the value of @which.
|
| 2746 | *
|
| 2747 | * This function should be called with migration disabled if the interrupt
|
| 2748 | * controller has per-cpu registers.
|
| 2749 |
|
| 2750 | int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which, bool val)
|
| 2751 | {
|
| 2752 | scoped_irqdesc_get_and_buslock(irq, 0) {
|
| 2753 | struct irq_data *data = irq_desc_get_irq_data(scoped_irqdesc);
|
| 2754 | struct irq_chip *chip;
|
| 2755 |
|
| 2756 | do {
|
| 2757 | chip = irq_data_get_irq_chip(data);
|
| 2758 |
|
| 2759 | if (WARN_ON_ONCE(!chip))
|
| 2760 | return -ENODEV;
|
| 2761 |
|
| 2762 | if (chip->irq_set_irqchip_state)
|
| 2763 | break;
|
| 2764 |
|
| 2765 | data = irqd_get_parent_data(data);
|
| 2766 | } while (data);
|
| 2767 |
|
| 2768 | if (data)
|
| 2769 | return chip->irq_set_irqchip_state(data, which, val);
|
| 2770 | }
|
| 2771 | return -EINVAL;
|
| 2772 | }
|
| 2773 | EXPORT_SYMBOL_GPL(irq_set_irqchip_state);
|
| 2774 |
|
| 2775 | |
| 2776 | * irq_has_action - Check whether an interrupt is requested
|
| 2777 | * @irq: The linux irq number
|
| 2778 | *
|
| 2779 | * Returns: A snapshot of the current state
|
| 2780 |
|
| 2781 | bool irq_has_action(unsigned int irq)
|
| 2782 | {
|
| 2783 | bool res;
|
| 2784 |
|
| 2785 | rcu_read_lock();
|
| 2786 | res = irq_desc_has_action(irq_to_desc(irq));
|
| 2787 | rcu_read_unlock();
|
| 2788 | return res;
|
| 2789 | }
|
| 2790 | EXPORT_SYMBOL_GPL(irq_has_action);
|
| 2791 |
|
| 2792 | |
| 2793 | * irq_check_status_bit - Check whether bits in the irq descriptor status are set
|
| 2794 | * @irq: The linux irq number
|
| 2795 | * @bitmask: The bitmask to evaluate
|
| 2796 | *
|
| 2797 | * Returns: True if one of the bits in @bitmask is set
|
| 2798 |
|
| 2799 | bool irq_check_status_bit(unsigned int irq, unsigned int bitmask)
|
| 2800 | {
|
| 2801 | struct irq_desc *desc;
|
| 2802 | bool res = false;
|
| 2803 |
|
| 2804 | rcu_read_lock();
|
| 2805 | desc = irq_to_desc(irq);
|
| 2806 | if (desc)
|
| 2807 | res = !!(desc->status_use_accessors & bitmask);
|
| 2808 | rcu_read_unlock();
|
| 2809 | return res;
|
| 2810 | }
|
| 2811 | EXPORT_SYMBOL_GPL(irq_check_status_bit);
|