| 1 |
|
| 2 | |
| 3 | * padata.h - header for the padata parallelization interface
|
| 4 | *
|
| 5 | * Copyright (C) 2008, 2009 secunet Security Networks AG
|
| 6 | * Copyright (C) 2008, 2009 Steffen Klassert <[email protected]>
|
| 7 | *
|
| 8 | * Copyright (c) 2020 Oracle and/or its affiliates.
|
| 9 | * Author: Daniel Jordan <[email protected]>
|
| 10 |
|
| 11 |
|
| 12 | #ifndef PADATA_H
|
| 13 | #define PADATA_H
|
| 14 |
|
| 15 | #include <linux/refcount.h>
|
| 16 | #include <linux/compiler_types.h>
|
| 17 | #include <linux/workqueue.h>
|
| 18 | #include <linux/spinlock.h>
|
| 19 | #include <linux/list.h>
|
| 20 | #include <linux/kobject.h>
|
| 21 |
|
| 22 | #define PADATA_CPU_SERIAL 0x01
|
| 23 | #define PADATA_CPU_PARALLEL 0x02
|
| 24 |
|
| 25 | |
| 26 | * struct padata_priv - Represents one job
|
| 27 | *
|
| 28 | * @list: List entry, to attach to the padata lists.
|
| 29 | * @pd: Pointer to the internal control structure.
|
| 30 | * @cb_cpu: Callback cpu for serializatioon.
|
| 31 | * @seq_nr: Sequence number of the parallelized data object.
|
| 32 | * @info: Used to pass information from the parallel to the serial function.
|
| 33 | * @parallel: Parallel execution function.
|
| 34 | * @serial: Serial complete function.
|
| 35 |
|
| 36 | struct padata_priv {
|
| 37 | struct list_head list;
|
| 38 | struct parallel_data *pd;
|
| 39 | int cb_cpu;
|
| 40 | unsigned int seq_nr;
|
| 41 | int info;
|
| 42 | void (*parallel)(struct padata_priv *padata);
|
| 43 | void (*serial)(struct padata_priv *padata);
|
| 44 | };
|
| 45 |
|
| 46 | |
| 47 | * struct padata_list - one per work type per CPU
|
| 48 | *
|
| 49 | * @list: List head.
|
| 50 | * @lock: List lock.
|
| 51 |
|
| 52 | struct padata_list {
|
| 53 | struct list_head list;
|
| 54 | spinlock_t lock;
|
| 55 | };
|
| 56 |
|
| 57 | |
| 58 | * struct padata_serial_queue - The percpu padata serial queue
|
| 59 | *
|
| 60 | * @serial: List to wait for serialization after reordering.
|
| 61 | * @work: work struct for serialization.
|
| 62 | * @pd: Backpointer to the internal control structure.
|
| 63 |
|
| 64 | struct padata_serial_queue {
|
| 65 | struct padata_list serial;
|
| 66 | struct work_struct work;
|
| 67 | struct parallel_data *pd;
|
| 68 | };
|
| 69 |
|
| 70 | |
| 71 | * struct padata_cpumask - The cpumasks for the parallel/serial workers
|
| 72 | *
|
| 73 | * @pcpu: cpumask for the parallel workers.
|
| 74 | * @cbcpu: cpumask for the serial (callback) workers.
|
| 75 |
|
| 76 | struct padata_cpumask {
|
| 77 | cpumask_var_t pcpu;
|
| 78 | cpumask_var_t cbcpu;
|
| 79 | };
|
| 80 |
|
| 81 | |
| 82 | * struct parallel_data - Internal control structure, covers everything
|
| 83 | * that depends on the cpumask in use.
|
| 84 | *
|
| 85 | * @ps: padata_shell object.
|
| 86 | * @reorder_list: percpu reorder lists
|
| 87 | * @squeue: percpu padata queues used for serialuzation.
|
| 88 | * @refcnt: Number of objects holding a reference on this parallel_data.
|
| 89 | * @seq_nr: Sequence number of the parallelized data object.
|
| 90 | * @processed: Number of already processed objects.
|
| 91 | * @cpu: Next CPU to be processed.
|
| 92 | * @cpumask: The cpumasks in use for parallel and serial workers.
|
| 93 |
|
| 94 | struct parallel_data {
|
| 95 | struct padata_shell *ps;
|
| 96 | struct padata_list __percpu *reorder_list;
|
| 97 | struct padata_serial_queue __percpu *squeue;
|
| 98 | refcount_t refcnt;
|
| 99 | unsigned int seq_nr;
|
| 100 | unsigned int processed;
|
| 101 | int cpu;
|
| 102 | struct padata_cpumask cpumask;
|
| 103 | };
|
| 104 |
|
| 105 | |
| 106 | * struct padata_shell - Wrapper around struct parallel_data, its
|
| 107 | * purpose is to allow the underlying control structure to be replaced
|
| 108 | * on the fly using RCU.
|
| 109 | *
|
| 110 | * @pinst: padat instance.
|
| 111 | * @pd: Actual parallel_data structure which may be substituted on the fly.
|
| 112 | * @opd: Pointer to old pd to be freed by padata_replace.
|
| 113 | * @list: List entry in padata_instance list.
|
| 114 |
|
| 115 | struct padata_shell {
|
| 116 | struct padata_instance *pinst;
|
| 117 | struct parallel_data __rcu *pd;
|
| 118 | struct parallel_data *opd;
|
| 119 | struct list_head list;
|
| 120 | };
|
| 121 |
|
| 122 | |
| 123 | * struct padata_mt_job - represents one multithreaded job
|
| 124 | *
|
| 125 | * @thread_fn: Called for each chunk of work that a padata thread does.
|
| 126 | * @fn_arg: The thread function argument.
|
| 127 | * @start: The start of the job (units are job-specific).
|
| 128 | * @size: size of this node's work (units are job-specific).
|
| 129 | * @align: Ranges passed to the thread function fall on this boundary, with the
|
| 130 | * possible exceptions of the beginning and end of the job.
|
| 131 | * @min_chunk: The minimum chunk size in job-specific units. This allows
|
| 132 | * the client to communicate the minimum amount of work that's
|
| 133 | * appropriate for one worker thread to do at once.
|
| 134 | * @max_threads: Max threads to use for the job, actual number may be less
|
| 135 | * depending on task size and minimum chunk size.
|
| 136 | * @numa_aware: Distribute jobs to different nodes with CPU in a round robin fashion.
|
| 137 |
|
| 138 | struct padata_mt_job {
|
| 139 | void (*thread_fn)(unsigned long start, unsigned long end, void *arg);
|
| 140 | void *fn_arg;
|
| 141 | unsigned long start;
|
| 142 | unsigned long size;
|
| 143 | unsigned long align;
|
| 144 | unsigned long min_chunk;
|
| 145 | int max_threads;
|
| 146 | bool numa_aware;
|
| 147 | };
|
| 148 |
|
| 149 | |
| 150 | * struct padata_instance - The overall control structure.
|
| 151 | *
|
| 152 | * @cpuhp_node: Linkage for CPU hotplug callbacks.
|
| 153 | * @parallel_wq: The workqueue used for parallel work.
|
| 154 | * @serial_wq: The workqueue used for serial work.
|
| 155 | * @pslist: List of padata_shell objects attached to this instance.
|
| 156 | * @cpumask: User supplied cpumasks for parallel and serial works.
|
| 157 | * @validate_cpumask: Internal cpumask used to validate @cpumask during hotplug.
|
| 158 | * @kobj: padata instance kernel object.
|
| 159 | * @lock: padata instance lock.
|
| 160 | * @flags: padata flags.
|
| 161 |
|
| 162 | struct padata_instance {
|
| 163 | struct hlist_node cpuhp_node;
|
| 164 | struct workqueue_struct *parallel_wq;
|
| 165 | struct workqueue_struct *serial_wq;
|
| 166 | struct list_head pslist;
|
| 167 | struct padata_cpumask cpumask;
|
| 168 | cpumask_var_t validate_cpumask;
|
| 169 | struct kobject kobj;
|
| 170 | struct mutex lock;
|
| 171 | u8 flags;
|
| 172 | #define PADATA_INIT 1
|
| 173 | #define PADATA_RESET 2
|
| 174 | #define PADATA_INVALID 4
|
| 175 | };
|
| 176 |
|
| 177 | #ifdef CONFIG_PADATA
|
| 178 | extern void __init padata_init(void);
|
| 179 | extern struct padata_instance *padata_alloc(const char *name);
|
| 180 | extern void padata_free(struct padata_instance *pinst);
|
| 181 | extern struct padata_shell *padata_alloc_shell(struct padata_instance *pinst);
|
| 182 | extern void padata_free_shell(struct padata_shell *ps);
|
| 183 | extern int padata_do_parallel(struct padata_shell *ps,
|
| 184 | struct padata_priv *padata, int *cb_cpu);
|
| 185 | extern void padata_do_serial(struct padata_priv *padata);
|
| 186 | extern void __init padata_do_multithreaded(struct padata_mt_job *job);
|
| 187 | extern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type,
|
| 188 | cpumask_var_t cpumask);
|
| 189 | #else
|
| 190 | static inline void __init padata_init(void) {}
|
| 191 | static inline void __init padata_do_multithreaded(struct padata_mt_job *job)
|
| 192 | {
|
| 193 | job->thread_fn(job->start, job->start + job->size, job->fn_arg);
|
| 194 | }
|
| 195 | #endif
|
| 196 |
|
| 197 | #endif
|