projects/linux·blob master · refs

336 lines · 8.3 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * SolidRun DPU driver for control plane
4 *
5 * Copyright (C) 2022-2023 SolidRun
6 *
7 * Author: Alvaro Karsz <[email protected]>
8 *
9 */
10
11#include <linux/iopoll.h>
12
13#include "snet_vdpa.h"
14
15enum snet_ctrl_opcodes {
16 SNET_CTRL_OP_DESTROY = 1,
17 SNET_CTRL_OP_READ_VQ_STATE,
18 SNET_CTRL_OP_SUSPEND,
19 SNET_CTRL_OP_RESUME,
20};
21
22#define SNET_CTRL_TIMEOUT 2000000
23
24#define SNET_CTRL_DATA_SIZE_MASK 0x0000FFFF
25#define SNET_CTRL_IN_PROCESS_MASK 0x00010000
26#define SNET_CTRL_CHUNK_RDY_MASK 0x00020000
27#define SNET_CTRL_ERROR_MASK 0x0FFC0000
28
29#define SNET_VAL_TO_ERR(val) (-(((val) & SNET_CTRL_ERROR_MASK) >> 18))
30#define SNET_EMPTY_CTRL(val) (((val) & SNET_CTRL_ERROR_MASK) || \
31 !((val) & SNET_CTRL_IN_PROCESS_MASK))
32#define SNET_DATA_READY(val) ((val) & (SNET_CTRL_ERROR_MASK | SNET_CTRL_CHUNK_RDY_MASK))
33
34/* Control register used to read data from the DPU */
35struct snet_ctrl_reg_ctrl {
36 /* Chunk size in 4B words */
37 u16 data_size;
38 /* We are in the middle of a command */
39 u16 in_process:1;
40 /* A data chunk is ready and can be consumed */
41 u16 chunk_ready:1;
42 /* Error code */
43 u16 error:10;
44 /* Saved for future usage */
45 u16 rsvd:4;
46};
47
48/* Opcode register */
49struct snet_ctrl_reg_op {
50 u16 opcode;
51 /* Only if VQ index is relevant for the command */
52 u16 vq_idx;
53};
54
55struct snet_ctrl_regs {
56 struct snet_ctrl_reg_op op;
57 struct snet_ctrl_reg_ctrl ctrl;
58 u32 rsvd;
59 u32 data[];
60};
61
62static struct snet_ctrl_regs __iomem *snet_get_ctrl(struct snet *snet)
63{
64 return snet->bar + snet->psnet->cfg.ctrl_off;
65}
66
67static int snet_wait_for_empty_ctrl(struct snet_ctrl_regs __iomem *regs)
68{
69 u32 val;
70
71 return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_EMPTY_CTRL(val), 10,
72 SNET_CTRL_TIMEOUT);
73}
74
75static int snet_wait_for_empty_op(struct snet_ctrl_regs __iomem *regs)
76{
77 u32 val;
78
79 return readx_poll_timeout(ioread32, &regs->op, val, !val, 10, SNET_CTRL_TIMEOUT);
80}
81
82static int snet_wait_for_data(struct snet_ctrl_regs __iomem *regs)
83{
84 u32 val;
85
86 return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_DATA_READY(val), 10,
87 SNET_CTRL_TIMEOUT);
88}
89
90static u32 snet_read32_word(struct snet_ctrl_regs __iomem *ctrl_regs, u16 word_idx)
91{
92 return ioread32(&ctrl_regs->data[word_idx]);
93}
94
95static u32 snet_read_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs)
96{
97 return ioread32(&ctrl_regs->ctrl);
98}
99
100static void snet_write_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
101{
102 iowrite32(val, &ctrl_regs->ctrl);
103}
104
105static void snet_write_op(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
106{
107 iowrite32(val, &ctrl_regs->op);
108}
109
110static int snet_wait_for_dpu_completion(struct snet_ctrl_regs __iomem *ctrl_regs)
111{
112 /* Wait until the DPU finishes completely.
113 * It will clear the opcode register.
114 */
115 return snet_wait_for_empty_op(ctrl_regs);
116}
117
118/* Reading ctrl from the DPU:
119 * buf_size must be 4B aligned
120 *
121 * Steps:
122 *
123 * (1) Verify that the DPU is not in the middle of another operation by
124 * reading the in_process and error bits in the control register.
125 * (2) Write the request opcode and the VQ idx in the opcode register
126 * and write the buffer size in the control register.
127 * (3) Start reading chunks of data, chunk_ready bit indicates that a
128 * data chunk is available, we signal that we read the data by clearing the bit.
129 * (4) Detect that the transfer is completed when the in_process bit
130 * in the control register is cleared or when an error appears.
131 */
132static int snet_ctrl_read_from_dpu(struct snet *snet, u16 opcode, u16 vq_idx, void *buffer,
133 u32 buf_size)
134{
135 struct pci_dev *pdev = snet->pdev;
136 struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
137 u32 *bfr_ptr = (u32 *)buffer;
138 u32 val;
139 u16 buf_words;
140 int ret;
141 u16 words, i, tot_words = 0;
142
143 /* Supported for config 2+ */
144 if (!SNET_CFG_VER(snet, 2))
145 return -EOPNOTSUPP;
146
147 if (!IS_ALIGNED(buf_size, 4))
148 return -EINVAL;
149
150 mutex_lock(&snet->ctrl_lock);
151
152 buf_words = buf_size / 4;
153
154 /* Make sure control register is empty */
155 ret = snet_wait_for_empty_ctrl(regs);
156 if (ret) {
157 SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
158 goto exit;
159 }
160
161 /* We need to write the buffer size in the control register, and the opcode + vq index in
162 * the opcode register.
163 * We use a spinlock to serialize the writes.
164 */
165 spin_lock(&snet->ctrl_spinlock);
166
167 snet_write_ctrl(regs, buf_words);
168 snet_write_op(regs, opcode | (vq_idx << 16));
169
170 spin_unlock(&snet->ctrl_spinlock);
171
172 while (buf_words != tot_words) {
173 ret = snet_wait_for_data(regs);
174 if (ret) {
175 SNET_WARN(pdev, "Timeout waiting for control data\n");
176 goto exit;
177 }
178
179 val = snet_read_ctrl(regs);
180
181 /* Error? */
182 if (val & SNET_CTRL_ERROR_MASK) {
183 ret = SNET_VAL_TO_ERR(val);
184 SNET_WARN(pdev, "Error while reading control data from DPU, err %d\n", ret);
185 goto exit;
186 }
187
188 words = min_t(u16, val & SNET_CTRL_DATA_SIZE_MASK, buf_words - tot_words);
189
190 for (i = 0; i < words; i++) {
191 *bfr_ptr = snet_read32_word(regs, i);
192 bfr_ptr++;
193 }
194
195 tot_words += words;
196
197 /* Is the job completed? */
198 if (!(val & SNET_CTRL_IN_PROCESS_MASK))
199 break;
200
201 /* Clear the chunk ready bit and continue */
202 val &= ~SNET_CTRL_CHUNK_RDY_MASK;
203 snet_write_ctrl(regs, val);
204 }
205
206 ret = snet_wait_for_dpu_completion(regs);
207 if (ret)
208 SNET_WARN(pdev, "Timeout waiting for the DPU to complete a control command\n");
209
210exit:
211 mutex_unlock(&snet->ctrl_lock);
212 return ret;
213}
214
215/* Send a control message to the DPU using the old mechanism
216 * used with config version 1.
217 */
218static int snet_send_ctrl_msg_old(struct snet *snet, u32 opcode)
219{
220 struct pci_dev *pdev = snet->pdev;
221 struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
222 int ret;
223
224 mutex_lock(&snet->ctrl_lock);
225
226 /* Old mechanism uses just 1 register, the opcode register.
227 * Make sure that the opcode register is empty, and that the DPU isn't
228 * processing an old message.
229 */
230 ret = snet_wait_for_empty_op(regs);
231 if (ret) {
232 SNET_WARN(pdev, "Timeout waiting for previous control message to be ACKed\n");
233 goto exit;
234 }
235
236 /* Write the message */
237 snet_write_op(regs, opcode);
238
239 /* DPU ACKs the message by clearing the opcode register */
240 ret = snet_wait_for_empty_op(regs);
241 if (ret)
242 SNET_WARN(pdev, "Timeout waiting for a control message to be ACKed\n");
243
244exit:
245 mutex_unlock(&snet->ctrl_lock);
246 return ret;
247}
248
249/* Send a control message to the DPU.
250 * A control message is a message without payload.
251 */
252static int snet_send_ctrl_msg(struct snet *snet, u16 opcode, u16 vq_idx)
253{
254 struct pci_dev *pdev = snet->pdev;
255 struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
256 u32 val;
257 int ret;
258
259 /* If config version is not 2+, use the old mechanism */
260 if (!SNET_CFG_VER(snet, 2))
261 return snet_send_ctrl_msg_old(snet, opcode);
262
263 mutex_lock(&snet->ctrl_lock);
264
265 /* Make sure control register is empty */
266 ret = snet_wait_for_empty_ctrl(regs);
267 if (ret) {
268 SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
269 goto exit;
270 }
271
272 /* We need to clear the control register and write the opcode + vq index in the opcode
273 * register.
274 * We use a spinlock to serialize the writes.
275 */
276 spin_lock(&snet->ctrl_spinlock);
277
278 snet_write_ctrl(regs, 0);
279 snet_write_op(regs, opcode | (vq_idx << 16));
280
281 spin_unlock(&snet->ctrl_spinlock);
282
283 /* The DPU ACKs control messages by setting the chunk ready bit
284 * without data.
285 */
286 ret = snet_wait_for_data(regs);
287 if (ret) {
288 SNET_WARN(pdev, "Timeout waiting for control message to be ACKed\n");
289 goto exit;
290 }
291
292 /* Check for errors */
293 val = snet_read_ctrl(regs);
294 ret = SNET_VAL_TO_ERR(val);
295
296 /* Clear the chunk ready bit */
297 val &= ~SNET_CTRL_CHUNK_RDY_MASK;
298 snet_write_ctrl(regs, val);
299
300 ret = snet_wait_for_dpu_completion(regs);
301 if (ret)
302 SNET_WARN(pdev, "Timeout waiting for DPU to complete a control command, err %d\n",
303 ret);
304
305exit:
306 mutex_unlock(&snet->ctrl_lock);
307 return ret;
308}
309
310void snet_ctrl_clear(struct snet *snet)
311{
312 struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
313
314 snet_write_op(regs, 0);
315}
316
317int snet_destroy_dev(struct snet *snet)
318{
319 return snet_send_ctrl_msg(snet, SNET_CTRL_OP_DESTROY, 0);
320}
321
322int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state)
323{
324 return snet_ctrl_read_from_dpu(snet, SNET_CTRL_OP_READ_VQ_STATE, idx, state,
325 sizeof(*state));
326}
327
328int snet_suspend_dev(struct snet *snet)
329{
330 return snet_send_ctrl_msg(snet, SNET_CTRL_OP_SUSPEND, 0);
331}
332
333int snet_resume_dev(struct snet *snet)
334{
335 return snet_send_ctrl_msg(snet, SNET_CTRL_OP_RESUME, 0);
336}