| 1 | |
| 2 | |
| 3 | * SMP related functions
|
| 4 | *
|
| 5 | * Copyright IBM Corp. 1999, 2012
|
| 6 | * Author(s): Denis Joseph Barrow,
|
| 7 | * Martin Schwidefsky <[email protected]>,
|
| 8 | *
|
| 9 | * based on other smp stuff by
|
| 10 | * (c) 1995 Alan Cox, CymruNET Ltd <[email protected]>
|
| 11 | * (c) 1998 Ingo Molnar
|
| 12 | *
|
| 13 | * The code outside of smp.c uses logical cpu numbers, only smp.c does
|
| 14 | * the translation of logical to physical cpu ids. All new code that
|
| 15 | * operates on physical cpu numbers needs to go into smp.c.
|
| 16 |
|
| 17 |
|
| 18 | #define pr_fmt(fmt) "cpu: " fmt
|
| 19 |
|
| 20 | #include <linux/cpufeature.h>
|
| 21 | #include <linux/workqueue.h>
|
| 22 | #include <linux/memblock.h>
|
| 23 | #include <linux/export.h>
|
| 24 | #include <linux/init.h>
|
| 25 | #include <linux/mm.h>
|
| 26 | #include <linux/err.h>
|
| 27 | #include <linux/spinlock.h>
|
| 28 | #include <linux/kernel_stat.h>
|
| 29 | #include <linux/delay.h>
|
| 30 | #include <linux/interrupt.h>
|
| 31 | #include <linux/irqflags.h>
|
| 32 | #include <linux/irq_work.h>
|
| 33 | #include <linux/cpu.h>
|
| 34 | #include <linux/slab.h>
|
| 35 | #include <linux/sched/hotplug.h>
|
| 36 | #include <linux/sched/task_stack.h>
|
| 37 | #include <linux/crash_dump.h>
|
| 38 | #include <linux/kprobes.h>
|
| 39 | #include <asm/access-regs.h>
|
| 40 | #include <asm/asm-offsets.h>
|
| 41 | #include <asm/machine.h>
|
| 42 | #include <asm/ctlreg.h>
|
| 43 | #include <asm/pfault.h>
|
| 44 | #include <asm/diag.h>
|
| 45 | #include <asm/facility.h>
|
| 46 | #include <asm/fpu.h>
|
| 47 | #include <asm/ipl.h>
|
| 48 | #include <asm/setup.h>
|
| 49 | #include <asm/irq.h>
|
| 50 | #include <asm/tlbflush.h>
|
| 51 | #include <asm/vtime.h>
|
| 52 | #include <asm/abs_lowcore.h>
|
| 53 | #include <asm/sclp.h>
|
| 54 | #include <asm/debug.h>
|
| 55 | #include <asm/os_info.h>
|
| 56 | #include <asm/sigp.h>
|
| 57 | #include <asm/nmi.h>
|
| 58 | #include <asm/stacktrace.h>
|
| 59 | #include <asm/topology.h>
|
| 60 | #include <asm/vdso.h>
|
| 61 | #include <asm/maccess.h>
|
| 62 | #include "entry.h"
|
| 63 |
|
| 64 | enum {
|
| 65 | ec_schedule = 0,
|
| 66 | ec_call_function_single,
|
| 67 | ec_stop_cpu,
|
| 68 | ec_mcck_pending,
|
| 69 | ec_irq_work,
|
| 70 | };
|
| 71 |
|
| 72 | enum {
|
| 73 | CPU_STATE_STANDBY,
|
| 74 | CPU_STATE_CONFIGURED,
|
| 75 | };
|
| 76 |
|
| 77 | static u8 boot_core_type;
|
| 78 | DEFINE_PER_CPU(struct pcpu, pcpu_devices);
|
| 79 | |
| 80 | * Pointer to the pcpu area of the boot CPU. This is required when a restart
|
| 81 | * interrupt is triggered on an offline CPU. For that case accessing percpu
|
| 82 | * data with the common primitives does not work, since the percpu offset is
|
| 83 | * stored in a non existent lowcore.
|
| 84 |
|
| 85 | static struct pcpu *ipl_pcpu;
|
| 86 |
|
| 87 | unsigned int smp_cpu_mt_shift;
|
| 88 | EXPORT_SYMBOL(smp_cpu_mt_shift);
|
| 89 |
|
| 90 | unsigned int smp_cpu_mtid;
|
| 91 | EXPORT_SYMBOL(smp_cpu_mtid);
|
| 92 |
|
| 93 | #ifdef CONFIG_CRASH_DUMP
|
| 94 | __vector128 __initdata boot_cpu_vector_save_area[__NUM_VXRS];
|
| 95 | #endif
|
| 96 |
|
| 97 | static unsigned int smp_max_threads __initdata = -1U;
|
| 98 | cpumask_t cpu_setup_mask;
|
| 99 |
|
| 100 | static int __init early_smt(char *s)
|
| 101 | {
|
| 102 | get_option(&s, &smp_max_threads);
|
| 103 | return 0;
|
| 104 | }
|
| 105 | early_param("smt", early_smt);
|
| 106 |
|
| 107 | |
| 108 | * The smp_cpu_state_mutex must be held when changing the state or polarization
|
| 109 | * member of a pcpu data structure within the pcpu_devices array.
|
| 110 |
|
| 111 | DEFINE_MUTEX(smp_cpu_state_mutex);
|
| 112 |
|
| 113 | |
| 114 | * Signal processor helper functions.
|
| 115 |
|
| 116 | static inline int __pcpu_sigp_relax(u16 addr, u8 order, unsigned long parm)
|
| 117 | {
|
| 118 | int cc;
|
| 119 |
|
| 120 | while (1) {
|
| 121 | cc = __pcpu_sigp(addr, order, parm, NULL);
|
| 122 | if (cc != SIGP_CC_BUSY)
|
| 123 | return cc;
|
| 124 | cpu_relax();
|
| 125 | }
|
| 126 | }
|
| 127 |
|
| 128 | static int pcpu_sigp_retry(struct pcpu *pcpu, u8 order, u32 parm)
|
| 129 | {
|
| 130 | int cc, retry;
|
| 131 |
|
| 132 | for (retry = 0; ; retry++) {
|
| 133 | cc = __pcpu_sigp(pcpu->address, order, parm, NULL);
|
| 134 | if (cc != SIGP_CC_BUSY)
|
| 135 | break;
|
| 136 | if (retry >= 3)
|
| 137 | udelay(10);
|
| 138 | }
|
| 139 | return cc;
|
| 140 | }
|
| 141 |
|
| 142 | static inline int pcpu_stopped(struct pcpu *pcpu)
|
| 143 | {
|
| 144 | u32 status;
|
| 145 |
|
| 146 | if (__pcpu_sigp(pcpu->address, SIGP_SENSE,
|
| 147 | 0, &status) != SIGP_CC_STATUS_STORED)
|
| 148 | return 0;
|
| 149 | return !!(status & (SIGP_STATUS_CHECK_STOP|SIGP_STATUS_STOPPED));
|
| 150 | }
|
| 151 |
|
| 152 | static inline int pcpu_running(struct pcpu *pcpu)
|
| 153 | {
|
| 154 | if (__pcpu_sigp(pcpu->address, SIGP_SENSE_RUNNING,
|
| 155 | 0, NULL) != SIGP_CC_STATUS_STORED)
|
| 156 | return 1;
|
| 157 |
|
| 158 | return 0;
|
| 159 | }
|
| 160 |
|
| 161 | |
| 162 | * Find struct pcpu by cpu address.
|
| 163 |
|
| 164 | static struct pcpu *pcpu_find_address(const struct cpumask *mask, u16 address)
|
| 165 | {
|
| 166 | int cpu;
|
| 167 |
|
| 168 | for_each_cpu(cpu, mask)
|
| 169 | if (per_cpu(pcpu_devices, cpu).address == address)
|
| 170 | return &per_cpu(pcpu_devices, cpu);
|
| 171 | return NULL;
|
| 172 | }
|
| 173 |
|
| 174 | static void pcpu_ec_call(struct pcpu *pcpu, int ec_bit)
|
| 175 | {
|
| 176 | if (test_and_set_bit(ec_bit, &pcpu->ec_mask))
|
| 177 | return;
|
| 178 | pcpu->ec_clk = get_tod_clock_fast();
|
| 179 | pcpu_sigp_retry(pcpu, SIGP_EXTERNAL_CALL, 0);
|
| 180 | }
|
| 181 |
|
| 182 | static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu)
|
| 183 | {
|
| 184 | unsigned long async_stack, nodat_stack, mcck_stack;
|
| 185 | struct lowcore *lc;
|
| 186 |
|
| 187 | lc = (struct lowcore *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
|
| 188 | nodat_stack = __get_free_pages(GFP_KERNEL, THREAD_SIZE_ORDER);
|
| 189 | async_stack = stack_alloc();
|
| 190 | mcck_stack = stack_alloc();
|
| 191 | if (!lc || !nodat_stack || !async_stack || !mcck_stack)
|
| 192 | goto out;
|
| 193 | memcpy(lc, get_lowcore(), 512);
|
| 194 | memset((char *) lc + 512, 0, sizeof(*lc) - 512);
|
| 195 | lc->async_stack = async_stack + STACK_INIT_OFFSET;
|
| 196 | lc->nodat_stack = nodat_stack + STACK_INIT_OFFSET;
|
| 197 | lc->mcck_stack = mcck_stack + STACK_INIT_OFFSET;
|
| 198 | lc->cpu_nr = cpu;
|
| 199 | lc->spinlock_lockval = arch_spin_lockval(cpu);
|
| 200 | lc->spinlock_index = 0;
|
| 201 | lc->return_lpswe = gen_lpswe(__LC_RETURN_PSW);
|
| 202 | lc->return_mcck_lpswe = gen_lpswe(__LC_RETURN_MCCK_PSW);
|
| 203 | lc->preempt_count = PREEMPT_DISABLED;
|
| 204 | if (nmi_alloc_mcesa(&lc->mcesad))
|
| 205 | goto out;
|
| 206 | if (abs_lowcore_map(cpu, lc, true))
|
| 207 | goto out_mcesa;
|
| 208 | lowcore_ptr[cpu] = lc;
|
| 209 | pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, __pa(lc));
|
| 210 | return 0;
|
| 211 |
|
| 212 | out_mcesa:
|
| 213 | nmi_free_mcesa(&lc->mcesad);
|
| 214 | out:
|
| 215 | stack_free(mcck_stack);
|
| 216 | stack_free(async_stack);
|
| 217 | free_pages(nodat_stack, THREAD_SIZE_ORDER);
|
| 218 | free_pages((unsigned long) lc, LC_ORDER);
|
| 219 | return -ENOMEM;
|
| 220 | }
|
| 221 |
|
| 222 | static void pcpu_free_lowcore(struct pcpu *pcpu, int cpu)
|
| 223 | {
|
| 224 | unsigned long async_stack, nodat_stack, mcck_stack;
|
| 225 | struct lowcore *lc;
|
| 226 |
|
| 227 | lc = lowcore_ptr[cpu];
|
| 228 | nodat_stack = lc->nodat_stack - STACK_INIT_OFFSET;
|
| 229 | async_stack = lc->async_stack - STACK_INIT_OFFSET;
|
| 230 | mcck_stack = lc->mcck_stack - STACK_INIT_OFFSET;
|
| 231 | pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, 0);
|
| 232 | lowcore_ptr[cpu] = NULL;
|
| 233 | abs_lowcore_unmap(cpu);
|
| 234 | nmi_free_mcesa(&lc->mcesad);
|
| 235 | stack_free(async_stack);
|
| 236 | stack_free(mcck_stack);
|
| 237 | free_pages(nodat_stack, THREAD_SIZE_ORDER);
|
| 238 | free_pages((unsigned long) lc, LC_ORDER);
|
| 239 | }
|
| 240 |
|
| 241 | static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu)
|
| 242 | {
|
| 243 | struct lowcore *lc, *abs_lc;
|
| 244 |
|
| 245 | lc = lowcore_ptr[cpu];
|
| 246 | cpumask_set_cpu(cpu, &init_mm.context.cpu_attach_mask);
|
| 247 | cpumask_set_cpu(cpu, mm_cpumask(&init_mm));
|
| 248 | lc->cpu_nr = cpu;
|
| 249 | lc->pcpu = (unsigned long)pcpu;
|
| 250 | lc->restart_flags = RESTART_FLAG_CTLREGS;
|
| 251 | lc->spinlock_lockval = arch_spin_lockval(cpu);
|
| 252 | lc->spinlock_index = 0;
|
| 253 | lc->percpu_offset = __per_cpu_offset[cpu];
|
| 254 | lc->kernel_asce = get_lowcore()->kernel_asce;
|
| 255 | lc->user_asce = s390_invalid_asce;
|
| 256 | lc->user_timer = lc->system_timer =
|
| 257 | lc->steal_timer = lc->avg_steal_timer = 0;
|
| 258 | abs_lc = get_abs_lowcore();
|
| 259 | memcpy(lc->cregs_save_area, abs_lc->cregs_save_area, sizeof(lc->cregs_save_area));
|
| 260 | put_abs_lowcore(abs_lc);
|
| 261 | lc->cregs_save_area[1] = lc->user_asce;
|
| 262 | lc->cregs_save_area[7] = lc->user_asce;
|
| 263 | save_access_regs((unsigned int *) lc->access_regs_save_area);
|
| 264 | arch_spin_lock_setup(cpu);
|
| 265 | }
|
| 266 |
|
| 267 | static void pcpu_attach_task(int cpu, struct task_struct *tsk)
|
| 268 | {
|
| 269 | struct lowcore *lc;
|
| 270 |
|
| 271 | lc = lowcore_ptr[cpu];
|
| 272 | lc->kernel_stack = (unsigned long)task_stack_page(tsk) + STACK_INIT_OFFSET;
|
| 273 | lc->current_task = (unsigned long)tsk;
|
| 274 | lc->lpp = LPP_MAGIC;
|
| 275 | lc->current_pid = tsk->pid;
|
| 276 | lc->user_timer = tsk->thread.user_timer;
|
| 277 | lc->guest_timer = tsk->thread.guest_timer;
|
| 278 | lc->system_timer = tsk->thread.system_timer;
|
| 279 | lc->hardirq_timer = tsk->thread.hardirq_timer;
|
| 280 | lc->softirq_timer = tsk->thread.softirq_timer;
|
| 281 | lc->steal_timer = 0;
|
| 282 | #ifdef CONFIG_STACKPROTECTOR
|
| 283 | lc->stack_canary = tsk->stack_canary;
|
| 284 | #endif
|
| 285 | }
|
| 286 |
|
| 287 | static void pcpu_start_fn(int cpu, void (*func)(void *), void *data)
|
| 288 | {
|
| 289 | struct lowcore *lc;
|
| 290 |
|
| 291 | lc = lowcore_ptr[cpu];
|
| 292 | lc->restart_stack = lc->kernel_stack;
|
| 293 | lc->restart_fn = (unsigned long) func;
|
| 294 | lc->restart_data = (unsigned long) data;
|
| 295 | lc->restart_source = -1U;
|
| 296 | pcpu_sigp_retry(per_cpu_ptr(&pcpu_devices, cpu), SIGP_RESTART, 0);
|
| 297 | }
|
| 298 |
|
| 299 | typedef void (pcpu_delegate_fn)(void *);
|
| 300 |
|
| 301 | |
| 302 | * Call function via PSW restart on pcpu and stop the current cpu.
|
| 303 |
|
| 304 | static void __pcpu_delegate(pcpu_delegate_fn *func, void *data)
|
| 305 | {
|
| 306 | func(data);
|
| 307 | }
|
| 308 |
|
| 309 | static void __noreturn pcpu_delegate(struct pcpu *pcpu, int cpu,
|
| 310 | pcpu_delegate_fn *func,
|
| 311 | void *data, unsigned long stack)
|
| 312 | {
|
| 313 | struct lowcore *lc, *abs_lc;
|
| 314 | unsigned int source_cpu;
|
| 315 |
|
| 316 | lc = lowcore_ptr[cpu];
|
| 317 | source_cpu = stap();
|
| 318 |
|
| 319 | if (pcpu->address == source_cpu) {
|
| 320 | call_on_stack(2, stack, void, __pcpu_delegate,
|
| 321 | pcpu_delegate_fn *, func, void *, data);
|
| 322 | }
|
| 323 |
|
| 324 | pcpu_sigp_retry(pcpu, SIGP_STOP, 0);
|
| 325 | pcpu_sigp_retry(pcpu, SIGP_CPU_RESET, 0);
|
| 326 |
|
| 327 | if (lc) {
|
| 328 | lc->restart_stack = stack;
|
| 329 | lc->restart_fn = (unsigned long)func;
|
| 330 | lc->restart_data = (unsigned long)data;
|
| 331 | lc->restart_source = source_cpu;
|
| 332 | } else {
|
| 333 | abs_lc = get_abs_lowcore();
|
| 334 | abs_lc->restart_stack = stack;
|
| 335 | abs_lc->restart_fn = (unsigned long)func;
|
| 336 | abs_lc->restart_data = (unsigned long)data;
|
| 337 | abs_lc->restart_source = source_cpu;
|
| 338 | put_abs_lowcore(abs_lc);
|
| 339 | }
|
| 340 | asm volatile(
|
| 341 | "0: sigp 0,%0,%2 # sigp restart to target cpu\n"
|
| 342 | " brc 2,0b # busy, try again\n"
|
| 343 | "1: sigp 0,%1,%3 # sigp stop to current cpu\n"
|
| 344 | " brc 2,1b # busy, try again"
|
| 345 | : : "d" (pcpu->address), "d" (source_cpu),
|
| 346 | "K" (SIGP_RESTART), "K" (SIGP_STOP)
|
| 347 | : "0", "1", "cc");
|
| 348 | for (;;) ;
|
| 349 | }
|
| 350 |
|
| 351 | |
| 352 | * Enable additional logical cpus for multi-threading.
|
| 353 |
|
| 354 | static int pcpu_set_smt(unsigned int mtid)
|
| 355 | {
|
| 356 | int cc;
|
| 357 |
|
| 358 | if (smp_cpu_mtid == mtid)
|
| 359 | return 0;
|
| 360 | cc = __pcpu_sigp(0, SIGP_SET_MULTI_THREADING, mtid, NULL);
|
| 361 | if (cc == 0) {
|
| 362 | smp_cpu_mtid = mtid;
|
| 363 | smp_cpu_mt_shift = 0;
|
| 364 | while (smp_cpu_mtid >= (1U << smp_cpu_mt_shift))
|
| 365 | smp_cpu_mt_shift++;
|
| 366 | per_cpu(pcpu_devices, 0).address = stap();
|
| 367 | }
|
| 368 | return cc;
|
| 369 | }
|
| 370 |
|
| 371 | |
| 372 | * Call function on the ipl CPU.
|
| 373 |
|
| 374 | void __noreturn smp_call_ipl_cpu(void (*func)(void *), void *data)
|
| 375 | {
|
| 376 | struct lowcore *lc = lowcore_ptr[0];
|
| 377 |
|
| 378 | if (ipl_pcpu->address == stap())
|
| 379 | lc = get_lowcore();
|
| 380 |
|
| 381 | pcpu_delegate(ipl_pcpu, 0, func, data, lc->nodat_stack);
|
| 382 | }
|
| 383 |
|
| 384 | int smp_find_processor_id(u16 address)
|
| 385 | {
|
| 386 | int cpu;
|
| 387 |
|
| 388 | for_each_present_cpu(cpu)
|
| 389 | if (per_cpu(pcpu_devices, cpu).address == address)
|
| 390 | return cpu;
|
| 391 | return -1;
|
| 392 | }
|
| 393 |
|
| 394 | void schedule_mcck_handler(void)
|
| 395 | {
|
| 396 | pcpu_ec_call(this_cpu_ptr(&pcpu_devices), ec_mcck_pending);
|
| 397 | }
|
| 398 |
|
| 399 | bool notrace arch_vcpu_is_preempted(int cpu)
|
| 400 | {
|
| 401 | if (test_cpu_flag_of(CIF_ENABLED_WAIT, cpu))
|
| 402 | return false;
|
| 403 | if (pcpu_running(per_cpu_ptr(&pcpu_devices, cpu)))
|
| 404 | return false;
|
| 405 | return true;
|
| 406 | }
|
| 407 | EXPORT_SYMBOL(arch_vcpu_is_preempted);
|
| 408 |
|
| 409 | void notrace smp_yield_cpu(int cpu)
|
| 410 | {
|
| 411 | if (!machine_has_diag9c())
|
| 412 | return;
|
| 413 | diag_stat_inc_norecursion(DIAG_STAT_X09C);
|
| 414 | asm volatile("diag %0,0,0x9c"
|
| 415 | : : "d" (per_cpu(pcpu_devices, cpu).address));
|
| 416 | }
|
| 417 | EXPORT_SYMBOL_GPL(smp_yield_cpu);
|
| 418 |
|
| 419 | |
| 420 | * Send cpus emergency shutdown signal. This gives the cpus the
|
| 421 | * opportunity to complete outstanding interrupts.
|
| 422 |
|
| 423 | void notrace smp_emergency_stop(void)
|
| 424 | {
|
| 425 | static arch_spinlock_t lock = __ARCH_SPIN_LOCK_UNLOCKED;
|
| 426 | static cpumask_t cpumask;
|
| 427 | u64 end;
|
| 428 | int cpu;
|
| 429 |
|
| 430 | arch_spin_lock(&lock);
|
| 431 | cpumask_copy(&cpumask, cpu_online_mask);
|
| 432 | cpumask_clear_cpu(smp_processor_id(), &cpumask);
|
| 433 |
|
| 434 | end = get_tod_clock_monotonic() + (1000000UL << 12);
|
| 435 | for_each_cpu(cpu, &cpumask) {
|
| 436 | struct pcpu *pcpu = per_cpu_ptr(&pcpu_devices, cpu);
|
| 437 | set_bit(ec_stop_cpu, &pcpu->ec_mask);
|
| 438 | while (__pcpu_sigp(pcpu->address, SIGP_EMERGENCY_SIGNAL,
|
| 439 | 0, NULL) == SIGP_CC_BUSY &&
|
| 440 | get_tod_clock_monotonic() < end)
|
| 441 | cpu_relax();
|
| 442 | }
|
| 443 | while (get_tod_clock_monotonic() < end) {
|
| 444 | for_each_cpu(cpu, &cpumask)
|
| 445 | if (pcpu_stopped(per_cpu_ptr(&pcpu_devices, cpu)))
|
| 446 | cpumask_clear_cpu(cpu, &cpumask);
|
| 447 | if (cpumask_empty(&cpumask))
|
| 448 | break;
|
| 449 | cpu_relax();
|
| 450 | }
|
| 451 | arch_spin_unlock(&lock);
|
| 452 | }
|
| 453 | NOKPROBE_SYMBOL(smp_emergency_stop);
|
| 454 |
|
| 455 | |
| 456 | * Stop all cpus but the current one.
|
| 457 |
|
| 458 | void smp_send_stop(void)
|
| 459 | {
|
| 460 | struct pcpu *pcpu;
|
| 461 | int cpu;
|
| 462 |
|
| 463 |
|
| 464 | __load_psw_mask(PSW_KERNEL_BITS);
|
| 465 | trace_hardirqs_off();
|
| 466 |
|
| 467 | debug_set_critical();
|
| 468 |
|
| 469 | if (oops_in_progress)
|
| 470 | smp_emergency_stop();
|
| 471 |
|
| 472 |
|
| 473 | for_each_online_cpu(cpu) {
|
| 474 | if (cpu == smp_processor_id())
|
| 475 | continue;
|
| 476 | pcpu = per_cpu_ptr(&pcpu_devices, cpu);
|
| 477 | pcpu_sigp_retry(pcpu, SIGP_STOP, 0);
|
| 478 | while (!pcpu_stopped(pcpu))
|
| 479 | cpu_relax();
|
| 480 | }
|
| 481 | }
|
| 482 |
|
| 483 | |
| 484 | * This is the main routine where commands issued by other
|
| 485 | * cpus are handled.
|
| 486 |
|
| 487 | static void smp_handle_ext_call(void)
|
| 488 | {
|
| 489 | unsigned long bits;
|
| 490 |
|
| 491 |
|
| 492 | bits = this_cpu_xchg(pcpu_devices.ec_mask, 0);
|
| 493 | if (test_bit(ec_stop_cpu, &bits))
|
| 494 | smp_stop_cpu();
|
| 495 | if (test_bit(ec_schedule, &bits))
|
| 496 | scheduler_ipi();
|
| 497 | if (test_bit(ec_call_function_single, &bits))
|
| 498 | generic_smp_call_function_single_interrupt();
|
| 499 | if (test_bit(ec_mcck_pending, &bits))
|
| 500 | s390_handle_mcck();
|
| 501 | if (test_bit(ec_irq_work, &bits))
|
| 502 | irq_work_run();
|
| 503 | }
|
| 504 |
|
| 505 | static void do_ext_call_interrupt(struct ext_code ext_code,
|
| 506 | unsigned int param32, unsigned long param64)
|
| 507 | {
|
| 508 | inc_irq_stat(ext_code.code == 0x1202 ? IRQEXT_EXC : IRQEXT_EMS);
|
| 509 | smp_handle_ext_call();
|
| 510 | }
|
| 511 |
|
| 512 | void arch_send_call_function_ipi_mask(const struct cpumask *mask)
|
| 513 | {
|
| 514 | int cpu;
|
| 515 |
|
| 516 | for_each_cpu(cpu, mask)
|
| 517 | pcpu_ec_call(per_cpu_ptr(&pcpu_devices, cpu), ec_call_function_single);
|
| 518 | }
|
| 519 |
|
| 520 | void arch_send_call_function_single_ipi(int cpu)
|
| 521 | {
|
| 522 | pcpu_ec_call(per_cpu_ptr(&pcpu_devices, cpu), ec_call_function_single);
|
| 523 | }
|
| 524 |
|
| 525 | |
| 526 | * this function sends a 'reschedule' IPI to another CPU.
|
| 527 | * it goes straight through and wastes no time serializing
|
| 528 | * anything. Worst case is that we lose a reschedule ...
|
| 529 |
|
| 530 | void arch_smp_send_reschedule(int cpu)
|
| 531 | {
|
| 532 | pcpu_ec_call(per_cpu_ptr(&pcpu_devices, cpu), ec_schedule);
|
| 533 | }
|
| 534 |
|
| 535 | #ifdef CONFIG_IRQ_WORK
|
| 536 | void arch_irq_work_raise(void)
|
| 537 | {
|
| 538 | pcpu_ec_call(this_cpu_ptr(&pcpu_devices), ec_irq_work);
|
| 539 | }
|
| 540 | #endif
|
| 541 |
|
| 542 | #ifdef CONFIG_CRASH_DUMP
|
| 543 |
|
| 544 | int smp_store_status(int cpu)
|
| 545 | {
|
| 546 | struct lowcore *lc;
|
| 547 | struct pcpu *pcpu;
|
| 548 | unsigned long pa;
|
| 549 |
|
| 550 | pcpu = per_cpu_ptr(&pcpu_devices, cpu);
|
| 551 | lc = lowcore_ptr[cpu];
|
| 552 | pa = __pa(&lc->floating_pt_save_area);
|
| 553 | if (__pcpu_sigp_relax(pcpu->address, SIGP_STORE_STATUS_AT_ADDRESS,
|
| 554 | pa) != SIGP_CC_ORDER_CODE_ACCEPTED)
|
| 555 | return -EIO;
|
| 556 | if (!cpu_has_vx() && !cpu_has_gs())
|
| 557 | return 0;
|
| 558 | pa = lc->mcesad & MCESA_ORIGIN_MASK;
|
| 559 | if (cpu_has_gs())
|
| 560 | pa |= lc->mcesad & MCESA_LC_MASK;
|
| 561 | if (__pcpu_sigp_relax(pcpu->address, SIGP_STORE_ADDITIONAL_STATUS,
|
| 562 | pa) != SIGP_CC_ORDER_CODE_ACCEPTED)
|
| 563 | return -EIO;
|
| 564 | return 0;
|
| 565 | }
|
| 566 |
|
| 567 | |
| 568 | * Collect CPU state of the previous, crashed system.
|
| 569 | * There are three cases:
|
| 570 | * 1) standard zfcp/nvme dump
|
| 571 | * condition: OLDMEM_BASE == NULL && is_ipl_type_dump() == true
|
| 572 | * The state for all CPUs except the boot CPU needs to be collected
|
| 573 | * with sigp stop-and-store-status. The boot CPU state is located in
|
| 574 | * the absolute lowcore of the memory stored in the HSA. The zcore code
|
| 575 | * will copy the boot CPU state from the HSA.
|
| 576 | * 2) stand-alone kdump for SCSI/NVMe (zfcp/nvme dump with swapped memory)
|
| 577 | * condition: OLDMEM_BASE != NULL && is_ipl_type_dump() == true
|
| 578 | * The state for all CPUs except the boot CPU needs to be collected
|
| 579 | * with sigp stop-and-store-status. The firmware or the boot-loader
|
| 580 | * stored the registers of the boot CPU in the absolute lowcore in the
|
| 581 | * memory of the old system.
|
| 582 | * 3) kdump or stand-alone kdump for DASD
|
| 583 | * condition: OLDMEM_BASE != NULL && is_ipl_type_dump() == false
|
| 584 | * The state for all CPUs except the boot CPU needs to be collected
|
| 585 | * with sigp stop-and-store-status. The kexec code or the boot-loader
|
| 586 | * stored the registers of the boot CPU in the memory of the old system.
|
| 587 | *
|
| 588 | * Note that the legacy kdump mode where the old kernel stored the CPU states
|
| 589 | * does no longer exist: setup_arch() explicitly deactivates the elfcorehdr=
|
| 590 | * kernel parameter. The is_kdump_kernel() implementation on s390 is independent
|
| 591 | * of the elfcorehdr= parameter.
|
| 592 |
|
| 593 | static bool dump_available(void)
|
| 594 | {
|
| 595 | return oldmem_data.start || is_ipl_type_dump();
|
| 596 | }
|
| 597 |
|
| 598 | void __init smp_save_dump_ipl_cpu(void)
|
| 599 | {
|
| 600 | struct save_area *sa;
|
| 601 | void *regs;
|
| 602 |
|
| 603 | if (!dump_available())
|
| 604 | return;
|
| 605 | sa = save_area_alloc(true);
|
| 606 | regs = memblock_alloc_or_panic(512, 8);
|
| 607 | copy_oldmem_kernel(regs, __LC_FPREGS_SAVE_AREA, 512);
|
| 608 | save_area_add_regs(sa, regs);
|
| 609 | memblock_free(regs, 512);
|
| 610 | if (cpu_has_vx())
|
| 611 | save_area_add_vxrs(sa, boot_cpu_vector_save_area);
|
| 612 | }
|
| 613 |
|
| 614 | void __init smp_save_dump_secondary_cpus(void)
|
| 615 | {
|
| 616 | int addr, boot_cpu_addr, max_cpu_addr;
|
| 617 | struct save_area *sa;
|
| 618 | void *page;
|
| 619 |
|
| 620 | if (!dump_available())
|
| 621 | return;
|
| 622 |
|
| 623 | page = memblock_alloc_low(PAGE_SIZE, PAGE_SIZE);
|
| 624 | if (!page)
|
| 625 | panic("ERROR: Failed to allocate %lx bytes below %lx\n",
|
| 626 | PAGE_SIZE, 1UL << 31);
|
| 627 |
|
| 628 |
|
| 629 | pcpu_set_smt(sclp.mtid_prev);
|
| 630 | boot_cpu_addr = stap();
|
| 631 | max_cpu_addr = SCLP_MAX_CORES << sclp.mtid_prev;
|
| 632 | for (addr = 0; addr <= max_cpu_addr; addr++) {
|
| 633 | if (addr == boot_cpu_addr)
|
| 634 | continue;
|
| 635 | if (__pcpu_sigp_relax(addr, SIGP_SENSE, 0) ==
|
| 636 | SIGP_CC_NOT_OPERATIONAL)
|
| 637 | continue;
|
| 638 | sa = save_area_alloc(false);
|
| 639 | __pcpu_sigp_relax(addr, SIGP_STORE_STATUS_AT_ADDRESS, __pa(page));
|
| 640 | save_area_add_regs(sa, page);
|
| 641 | if (cpu_has_vx()) {
|
| 642 | __pcpu_sigp_relax(addr, SIGP_STORE_ADDITIONAL_STATUS, __pa(page));
|
| 643 | save_area_add_vxrs(sa, page);
|
| 644 | }
|
| 645 | }
|
| 646 | memblock_free(page, PAGE_SIZE);
|
| 647 | diag_amode31_ops.diag308_reset();
|
| 648 | pcpu_set_smt(0);
|
| 649 | }
|
| 650 | #endif
|
| 651 |
|
| 652 | void smp_cpu_set_polarization(int cpu, int val)
|
| 653 | {
|
| 654 | per_cpu(pcpu_devices, cpu).polarization = val;
|
| 655 | }
|
| 656 |
|
| 657 | int smp_cpu_get_polarization(int cpu)
|
| 658 | {
|
| 659 | return per_cpu(pcpu_devices, cpu).polarization;
|
| 660 | }
|
| 661 |
|
| 662 | void smp_set_core_capacity(int cpu, unsigned long val)
|
| 663 | {
|
| 664 | int i;
|
| 665 |
|
| 666 | cpu = smp_get_base_cpu(cpu);
|
| 667 | for (i = cpu; (i <= cpu + smp_cpu_mtid) && (i < nr_cpu_ids); i++)
|
| 668 | topology_set_cpu_scale(i, val);
|
| 669 | }
|
| 670 |
|
| 671 | int smp_cpu_get_cpu_address(int cpu)
|
| 672 | {
|
| 673 | return per_cpu(pcpu_devices, cpu).address;
|
| 674 | }
|
| 675 |
|
| 676 | static void __ref smp_get_core_info(struct sclp_core_info *info, int early)
|
| 677 | {
|
| 678 | static int use_sigp_detection;
|
| 679 | int address;
|
| 680 |
|
| 681 | if (use_sigp_detection || sclp_get_core_info(info, early)) {
|
| 682 | use_sigp_detection = 1;
|
| 683 | for (address = 0;
|
| 684 | address < (SCLP_MAX_CORES << smp_cpu_mt_shift);
|
| 685 | address += (1U << smp_cpu_mt_shift)) {
|
| 686 | if (__pcpu_sigp_relax(address, SIGP_SENSE, 0) ==
|
| 687 | SIGP_CC_NOT_OPERATIONAL)
|
| 688 | continue;
|
| 689 | info->core[info->configured].core_id =
|
| 690 | address >> smp_cpu_mt_shift;
|
| 691 | info->core[info->configured].type = boot_core_type;
|
| 692 | info->configured++;
|
| 693 | }
|
| 694 | info->combined = info->configured;
|
| 695 | }
|
| 696 | }
|
| 697 |
|
| 698 | static int smp_add_core(struct sclp_core_entry *core, cpumask_t *avail,
|
| 699 | bool configured, bool early)
|
| 700 | {
|
| 701 | struct pcpu *pcpu;
|
| 702 | int cpu, nr, i;
|
| 703 | u16 address;
|
| 704 |
|
| 705 | nr = 0;
|
| 706 | if (sclp.has_core_type && core->type != boot_core_type)
|
| 707 | return nr;
|
| 708 | cpu = cpumask_first(avail);
|
| 709 | address = core->core_id << smp_cpu_mt_shift;
|
| 710 | for (i = 0; (i <= smp_cpu_mtid) && (cpu < nr_cpu_ids); i++) {
|
| 711 | if (pcpu_find_address(cpu_present_mask, address + i))
|
| 712 | continue;
|
| 713 | pcpu = per_cpu_ptr(&pcpu_devices, cpu);
|
| 714 | pcpu->address = address + i;
|
| 715 | if (configured)
|
| 716 | pcpu->state = CPU_STATE_CONFIGURED;
|
| 717 | else
|
| 718 | pcpu->state = CPU_STATE_STANDBY;
|
| 719 | smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
|
| 720 | topology_set_cpu_scale(cpu, CPU_CAPACITY_HIGH);
|
| 721 | set_cpu_present(cpu, true);
|
| 722 | if (!early && arch_register_cpu(cpu))
|
| 723 | set_cpu_present(cpu, false);
|
| 724 | else
|
| 725 | nr++;
|
| 726 | cpumask_clear_cpu(cpu, avail);
|
| 727 | cpu = cpumask_next(cpu, avail);
|
| 728 | }
|
| 729 | return nr;
|
| 730 | }
|
| 731 |
|
| 732 | static int __smp_rescan_cpus(struct sclp_core_info *info, bool early)
|
| 733 | {
|
| 734 | struct sclp_core_entry *core;
|
| 735 | static cpumask_t avail;
|
| 736 | bool configured;
|
| 737 | u16 core_id;
|
| 738 | int nr, i;
|
| 739 |
|
| 740 | cpus_read_lock();
|
| 741 | mutex_lock(&smp_cpu_state_mutex);
|
| 742 | nr = 0;
|
| 743 | cpumask_xor(&avail, cpu_possible_mask, cpu_present_mask);
|
| 744 | |
| 745 | * Add IPL core first (which got logical CPU number 0) to make sure
|
| 746 | * that all SMT threads get subsequent logical CPU numbers.
|
| 747 |
|
| 748 | if (early) {
|
| 749 | core_id = per_cpu(pcpu_devices, 0).address >> smp_cpu_mt_shift;
|
| 750 | for (i = 0; i < info->configured; i++) {
|
| 751 | core = &info->core[i];
|
| 752 | if (core->core_id == core_id) {
|
| 753 | nr += smp_add_core(core, &avail, true, early);
|
| 754 | break;
|
| 755 | }
|
| 756 | }
|
| 757 | }
|
| 758 | for (i = 0; i < info->combined; i++) {
|
| 759 | configured = i < info->configured;
|
| 760 | nr += smp_add_core(&info->core[i], &avail, configured, early);
|
| 761 | }
|
| 762 | mutex_unlock(&smp_cpu_state_mutex);
|
| 763 | cpus_read_unlock();
|
| 764 | return nr;
|
| 765 | }
|
| 766 |
|
| 767 | void __init smp_detect_cpus(void)
|
| 768 | {
|
| 769 | unsigned int cpu, mtid, c_cpus, s_cpus;
|
| 770 | struct sclp_core_info *info;
|
| 771 | u16 address;
|
| 772 |
|
| 773 |
|
| 774 | info = memblock_alloc_or_panic(sizeof(*info), 8);
|
| 775 | smp_get_core_info(info, 1);
|
| 776 |
|
| 777 | if (sclp.has_core_type) {
|
| 778 | address = stap();
|
| 779 | for (cpu = 0; cpu < info->combined; cpu++)
|
| 780 | if (info->core[cpu].core_id == address) {
|
| 781 |
|
| 782 | boot_core_type = info->core[cpu].type;
|
| 783 | break;
|
| 784 | }
|
| 785 | if (cpu >= info->combined)
|
| 786 | panic("Could not find boot CPU type");
|
| 787 | }
|
| 788 |
|
| 789 |
|
| 790 | mtid = boot_core_type ? sclp.mtid : sclp.mtid_cp;
|
| 791 | mtid = (mtid < smp_max_threads) ? mtid : smp_max_threads - 1;
|
| 792 | pcpu_set_smt(mtid);
|
| 793 | cpu_smt_set_num_threads(smp_cpu_mtid + 1, smp_cpu_mtid + 1);
|
| 794 |
|
| 795 |
|
| 796 | c_cpus = s_cpus = 0;
|
| 797 | for (cpu = 0; cpu < info->combined; cpu++) {
|
| 798 | if (sclp.has_core_type &&
|
| 799 | info->core[cpu].type != boot_core_type)
|
| 800 | continue;
|
| 801 | if (cpu < info->configured)
|
| 802 | c_cpus += smp_cpu_mtid + 1;
|
| 803 | else
|
| 804 | s_cpus += smp_cpu_mtid + 1;
|
| 805 | }
|
| 806 | pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
|
| 807 | memblock_free(info, sizeof(*info));
|
| 808 | }
|
| 809 |
|
| 810 | |
| 811 | * Activate a secondary processor.
|
| 812 |
|
| 813 | static void smp_start_secondary(void *cpuvoid)
|
| 814 | {
|
| 815 | struct lowcore *lc = get_lowcore();
|
| 816 | int cpu = raw_smp_processor_id();
|
| 817 |
|
| 818 | lc->last_update_clock = get_tod_clock();
|
| 819 | lc->restart_stack = (unsigned long)restart_stack;
|
| 820 | lc->restart_fn = (unsigned long)do_restart;
|
| 821 | lc->restart_data = 0;
|
| 822 | lc->restart_source = -1U;
|
| 823 | lc->restart_flags = 0;
|
| 824 | restore_access_regs(lc->access_regs_save_area);
|
| 825 | cpu_init();
|
| 826 | rcutree_report_cpu_starting(cpu);
|
| 827 | init_cpu_timer();
|
| 828 | vtime_init();
|
| 829 | vdso_getcpu_init();
|
| 830 | pfault_init();
|
| 831 | cpumask_set_cpu(cpu, &cpu_setup_mask);
|
| 832 | update_cpu_masks();
|
| 833 | notify_cpu_starting(cpu);
|
| 834 | if (topology_cpu_dedicated(cpu))
|
| 835 | set_cpu_flag(CIF_DEDICATED_CPU);
|
| 836 | else
|
| 837 | clear_cpu_flag(CIF_DEDICATED_CPU);
|
| 838 | set_cpu_online(cpu, true);
|
| 839 | inc_irq_stat(CPU_RST);
|
| 840 | local_irq_enable();
|
| 841 | cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
|
| 842 | }
|
| 843 |
|
| 844 |
|
| 845 | int __cpu_up(unsigned int cpu, struct task_struct *tidle)
|
| 846 | {
|
| 847 | struct pcpu *pcpu = per_cpu_ptr(&pcpu_devices, cpu);
|
| 848 | int rc;
|
| 849 |
|
| 850 | if (pcpu->state != CPU_STATE_CONFIGURED)
|
| 851 | return -EIO;
|
| 852 | if (pcpu_sigp_retry(pcpu, SIGP_INITIAL_CPU_RESET, 0) !=
|
| 853 | SIGP_CC_ORDER_CODE_ACCEPTED)
|
| 854 | return -EIO;
|
| 855 |
|
| 856 | rc = pcpu_alloc_lowcore(pcpu, cpu);
|
| 857 | if (rc)
|
| 858 | return rc;
|
| 859 | |
| 860 | * Make sure global control register contents do not change
|
| 861 | * until new CPU has initialized control registers.
|
| 862 |
|
| 863 | system_ctlreg_lock();
|
| 864 | pcpu_prepare_secondary(pcpu, cpu);
|
| 865 | pcpu_attach_task(cpu, tidle);
|
| 866 | pcpu_start_fn(cpu, smp_start_secondary, NULL);
|
| 867 |
|
| 868 | while (!cpu_online(cpu))
|
| 869 | cpu_relax();
|
| 870 | system_ctlreg_unlock();
|
| 871 | return 0;
|
| 872 | }
|
| 873 |
|
| 874 | static unsigned int setup_possible_cpus __initdata;
|
| 875 |
|
| 876 | static int __init _setup_possible_cpus(char *s)
|
| 877 | {
|
| 878 | get_option(&s, &setup_possible_cpus);
|
| 879 | return 0;
|
| 880 | }
|
| 881 | early_param("possible_cpus", _setup_possible_cpus);
|
| 882 |
|
| 883 | int __cpu_disable(void)
|
| 884 | {
|
| 885 | struct ctlreg cregs[16];
|
| 886 | int cpu;
|
| 887 |
|
| 888 |
|
| 889 | smp_handle_ext_call();
|
| 890 | cpu = smp_processor_id();
|
| 891 | set_cpu_online(cpu, false);
|
| 892 | cpumask_clear_cpu(cpu, &cpu_setup_mask);
|
| 893 | update_cpu_masks();
|
| 894 |
|
| 895 | pfault_fini();
|
| 896 |
|
| 897 | __local_ctl_store(0, 15, cregs);
|
| 898 | cregs[0].val &= ~0x0000ee70UL;
|
| 899 | cregs[6].val &= ~0xff000000UL;
|
| 900 | cregs[14].val &= ~0x1f000000UL;
|
| 901 | __local_ctl_load(0, 15, cregs);
|
| 902 | return 0;
|
| 903 | }
|
| 904 |
|
| 905 | void __cpu_die(unsigned int cpu)
|
| 906 | {
|
| 907 | struct pcpu *pcpu;
|
| 908 |
|
| 909 |
|
| 910 | pcpu = per_cpu_ptr(&pcpu_devices, cpu);
|
| 911 | while (!pcpu_stopped(pcpu))
|
| 912 | cpu_relax();
|
| 913 | pcpu_free_lowcore(pcpu, cpu);
|
| 914 | cpumask_clear_cpu(cpu, mm_cpumask(&init_mm));
|
| 915 | cpumask_clear_cpu(cpu, &init_mm.context.cpu_attach_mask);
|
| 916 | pcpu->flags = 0;
|
| 917 | }
|
| 918 |
|
| 919 | void __noreturn cpu_die(void)
|
| 920 | {
|
| 921 | idle_task_exit();
|
| 922 | pcpu_sigp_retry(this_cpu_ptr(&pcpu_devices), SIGP_STOP, 0);
|
| 923 | for (;;) ;
|
| 924 | }
|
| 925 |
|
| 926 | void __init smp_fill_possible_mask(void)
|
| 927 | {
|
| 928 | unsigned int possible, sclp_max, cpu;
|
| 929 |
|
| 930 | sclp_max = max(sclp.mtid, sclp.mtid_cp) + 1;
|
| 931 | sclp_max = min(smp_max_threads, sclp_max);
|
| 932 | sclp_max = (sclp.max_cores * sclp_max) ?: nr_cpu_ids;
|
| 933 | possible = setup_possible_cpus ?: nr_cpu_ids;
|
| 934 | possible = min(possible, sclp_max);
|
| 935 | for (cpu = 0; cpu < possible && cpu < nr_cpu_ids; cpu++)
|
| 936 | set_cpu_possible(cpu, true);
|
| 937 | }
|
| 938 |
|
| 939 | void __init smp_prepare_cpus(unsigned int max_cpus)
|
| 940 | {
|
| 941 | if (register_external_irq(EXT_IRQ_EMERGENCY_SIG, do_ext_call_interrupt))
|
| 942 | panic("Couldn't request external interrupt 0x1201");
|
| 943 | system_ctl_set_bit(0, 14);
|
| 944 | if (register_external_irq(EXT_IRQ_EXTERNAL_CALL, do_ext_call_interrupt))
|
| 945 | panic("Couldn't request external interrupt 0x1202");
|
| 946 | system_ctl_set_bit(0, 13);
|
| 947 | smp_rescan_cpus(true);
|
| 948 | }
|
| 949 |
|
| 950 | void __init smp_prepare_boot_cpu(void)
|
| 951 | {
|
| 952 | struct lowcore *lc = get_lowcore();
|
| 953 |
|
| 954 | WARN_ON(!cpu_present(0) || !cpu_online(0));
|
| 955 | lc->percpu_offset = __per_cpu_offset[0];
|
| 956 | ipl_pcpu = per_cpu_ptr(&pcpu_devices, 0);
|
| 957 | ipl_pcpu->state = CPU_STATE_CONFIGURED;
|
| 958 | lc->pcpu = (unsigned long)ipl_pcpu;
|
| 959 | smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN);
|
| 960 | topology_set_cpu_scale(0, CPU_CAPACITY_HIGH);
|
| 961 | }
|
| 962 |
|
| 963 | void __init smp_setup_processor_id(void)
|
| 964 | {
|
| 965 | struct lowcore *lc = get_lowcore();
|
| 966 |
|
| 967 | lc->cpu_nr = 0;
|
| 968 | per_cpu(pcpu_devices, 0).address = stap();
|
| 969 | lc->spinlock_lockval = arch_spin_lockval(0);
|
| 970 | lc->spinlock_index = 0;
|
| 971 | }
|
| 972 |
|
| 973 | |
| 974 | * the frequency of the profiling timer can be changed
|
| 975 | * by writing a multiplier value into /proc/profile.
|
| 976 | *
|
| 977 | * usually you want to run this on all CPUs ;)
|
| 978 |
|
| 979 | int setup_profiling_timer(unsigned int multiplier)
|
| 980 | {
|
| 981 | return 0;
|
| 982 | }
|
| 983 |
|
| 984 | static ssize_t cpu_configure_show(struct device *dev,
|
| 985 | struct device_attribute *attr, char *buf)
|
| 986 | {
|
| 987 | ssize_t count;
|
| 988 |
|
| 989 | mutex_lock(&smp_cpu_state_mutex);
|
| 990 | count = sysfs_emit(buf, "%d\n", per_cpu(pcpu_devices, dev->id).state);
|
| 991 | mutex_unlock(&smp_cpu_state_mutex);
|
| 992 | return count;
|
| 993 | }
|
| 994 |
|
| 995 | static ssize_t cpu_configure_store(struct device *dev,
|
| 996 | struct device_attribute *attr,
|
| 997 | const char *buf, size_t count)
|
| 998 | {
|
| 999 | struct pcpu *pcpu;
|
| 1000 | int cpu, val, rc, i;
|
| 1001 | char delim;
|
| 1002 |
|
| 1003 | if (sscanf(buf, "%d %c", &val, &delim) != 1)
|
| 1004 | return -EINVAL;
|
| 1005 | if (val != 0 && val != 1)
|
| 1006 | return -EINVAL;
|
| 1007 | cpus_read_lock();
|
| 1008 | mutex_lock(&smp_cpu_state_mutex);
|
| 1009 | rc = -EBUSY;
|
| 1010 |
|
| 1011 | cpu = dev->id;
|
| 1012 | cpu = smp_get_base_cpu(cpu);
|
| 1013 | for (i = 0; i <= smp_cpu_mtid; i++)
|
| 1014 | if (cpu_online(cpu + i))
|
| 1015 | goto out;
|
| 1016 | pcpu = per_cpu_ptr(&pcpu_devices, cpu);
|
| 1017 | rc = 0;
|
| 1018 | switch (val) {
|
| 1019 | case 0:
|
| 1020 | if (pcpu->state != CPU_STATE_CONFIGURED)
|
| 1021 | break;
|
| 1022 | rc = sclp_core_deconfigure(pcpu->address >> smp_cpu_mt_shift);
|
| 1023 | if (rc)
|
| 1024 | break;
|
| 1025 | for (i = 0; i <= smp_cpu_mtid; i++) {
|
| 1026 | if (cpu + i >= nr_cpu_ids || !cpu_present(cpu + i))
|
| 1027 | continue;
|
| 1028 | per_cpu(pcpu_devices, cpu + i).state = CPU_STATE_STANDBY;
|
| 1029 | smp_cpu_set_polarization(cpu + i,
|
| 1030 | POLARIZATION_UNKNOWN);
|
| 1031 | set_cpu_enabled(cpu + i, false);
|
| 1032 | }
|
| 1033 | topology_expect_change();
|
| 1034 | break;
|
| 1035 | case 1:
|
| 1036 | if (pcpu->state != CPU_STATE_STANDBY)
|
| 1037 | break;
|
| 1038 | rc = sclp_core_configure(pcpu->address >> smp_cpu_mt_shift);
|
| 1039 | if (rc)
|
| 1040 | break;
|
| 1041 | for (i = 0; i <= smp_cpu_mtid; i++) {
|
| 1042 | if (cpu + i >= nr_cpu_ids || !cpu_present(cpu + i))
|
| 1043 | continue;
|
| 1044 | per_cpu(pcpu_devices, cpu + i).state = CPU_STATE_CONFIGURED;
|
| 1045 | smp_cpu_set_polarization(cpu + i,
|
| 1046 | POLARIZATION_UNKNOWN);
|
| 1047 | set_cpu_enabled(cpu + i, true);
|
| 1048 | }
|
| 1049 | topology_expect_change();
|
| 1050 | break;
|
| 1051 | default:
|
| 1052 | break;
|
| 1053 | }
|
| 1054 | out:
|
| 1055 | mutex_unlock(&smp_cpu_state_mutex);
|
| 1056 | cpus_read_unlock();
|
| 1057 | return rc ? rc : count;
|
| 1058 | }
|
| 1059 | static DEVICE_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
|
| 1060 |
|
| 1061 | static ssize_t show_cpu_address(struct device *dev,
|
| 1062 | struct device_attribute *attr, char *buf)
|
| 1063 | {
|
| 1064 | return sysfs_emit(buf, "%d\n", per_cpu(pcpu_devices, dev->id).address);
|
| 1065 | }
|
| 1066 | static DEVICE_ATTR(address, 0444, show_cpu_address, NULL);
|
| 1067 |
|
| 1068 | static struct attribute *cpu_common_attrs[] = {
|
| 1069 | &dev_attr_configure.attr,
|
| 1070 | &dev_attr_address.attr,
|
| 1071 | NULL,
|
| 1072 | };
|
| 1073 |
|
| 1074 | static struct attribute_group cpu_common_attr_group = {
|
| 1075 | .attrs = cpu_common_attrs,
|
| 1076 | };
|
| 1077 |
|
| 1078 | bool arch_cpu_is_hotpluggable(int cpu)
|
| 1079 | {
|
| 1080 | return !!cpu;
|
| 1081 | }
|
| 1082 |
|
| 1083 | int arch_register_cpu(int cpu)
|
| 1084 | {
|
| 1085 | struct pcpu *pcpu = per_cpu_ptr(&pcpu_devices, cpu);
|
| 1086 | struct cpu *c = per_cpu_ptr(&cpu_devices, cpu);
|
| 1087 | int rc;
|
| 1088 |
|
| 1089 | c->hotpluggable = arch_cpu_is_hotpluggable(cpu);
|
| 1090 | rc = register_cpu(c, cpu);
|
| 1091 | if (rc)
|
| 1092 | goto out;
|
| 1093 | rc = sysfs_create_group(&c->dev.kobj, &cpu_common_attr_group);
|
| 1094 | if (rc)
|
| 1095 | goto out_cpu;
|
| 1096 | rc = topology_cpu_init(c);
|
| 1097 | if (rc)
|
| 1098 | goto out_topology;
|
| 1099 | if (pcpu->state != CPU_STATE_CONFIGURED)
|
| 1100 | set_cpu_enabled(cpu, false);
|
| 1101 | return 0;
|
| 1102 |
|
| 1103 | out_topology:
|
| 1104 | sysfs_remove_group(&c->dev.kobj, &cpu_common_attr_group);
|
| 1105 | out_cpu:
|
| 1106 | unregister_cpu(c);
|
| 1107 | out:
|
| 1108 | return rc;
|
| 1109 | }
|
| 1110 |
|
| 1111 | int __ref smp_rescan_cpus(bool early)
|
| 1112 | {
|
| 1113 | struct sclp_core_info *info;
|
| 1114 | int nr;
|
| 1115 |
|
| 1116 | info = kzalloc_obj(*info);
|
| 1117 | if (!info)
|
| 1118 | return -ENOMEM;
|
| 1119 | smp_get_core_info(info, 0);
|
| 1120 | nr = __smp_rescan_cpus(info, early);
|
| 1121 | kfree(info);
|
| 1122 | if (nr && !early)
|
| 1123 | topology_schedule_update();
|
| 1124 | return 0;
|
| 1125 | }
|
| 1126 |
|
| 1127 | static ssize_t __ref rescan_store(struct device *dev,
|
| 1128 | struct device_attribute *attr,
|
| 1129 | const char *buf,
|
| 1130 | size_t count)
|
| 1131 | {
|
| 1132 | int rc;
|
| 1133 |
|
| 1134 | rc = lock_device_hotplug_sysfs();
|
| 1135 | if (rc)
|
| 1136 | return rc;
|
| 1137 | rc = smp_rescan_cpus(false);
|
| 1138 | unlock_device_hotplug();
|
| 1139 | return rc ? rc : count;
|
| 1140 | }
|
| 1141 | static DEVICE_ATTR_WO(rescan);
|
| 1142 |
|
| 1143 | static int __init s390_smp_init(void)
|
| 1144 | {
|
| 1145 | struct device *dev_root;
|
| 1146 | int rc = 0;
|
| 1147 |
|
| 1148 | dev_root = bus_get_dev_root(&cpu_subsys);
|
| 1149 | if (dev_root) {
|
| 1150 | rc = device_create_file(dev_root, &dev_attr_rescan);
|
| 1151 | put_device(dev_root);
|
| 1152 | }
|
| 1153 | return rc;
|
| 1154 | }
|
| 1155 | subsys_initcall(s390_smp_init);
|