| 1 | |
| 2 |
|
| 3 |
|
| 4 | #include <linux/kernel.h>
|
| 5 | #include <linux/nospec.h>
|
| 6 | #include "cc_driver.h"
|
| 7 | #include "cc_buffer_mgr.h"
|
| 8 | #include "cc_request_mgr.h"
|
| 9 | #include "cc_pm.h"
|
| 10 |
|
| 11 | #define CC_MAX_POLL_ITER 10
|
| 12 |
|
| 13 | #define CC_MAX_DESC_SEQ_LEN 23
|
| 14 |
|
| 15 | struct cc_req_mgr_handle {
|
| 16 |
|
| 17 | unsigned int hw_queue_size;
|
| 18 | unsigned int min_free_hw_slots;
|
| 19 | unsigned int max_used_sw_slots;
|
| 20 | struct cc_crypto_req req_queue[MAX_REQUEST_QUEUE_SIZE];
|
| 21 | u32 req_queue_head;
|
| 22 | u32 req_queue_tail;
|
| 23 | u32 axi_completed;
|
| 24 | u32 q_free_slots;
|
| 25 | |
| 26 | * that must be single request at a time
|
| 27 |
|
| 28 | spinlock_t hw_lock;
|
| 29 | struct cc_hw_desc compl_desc;
|
| 30 | u8 *dummy_comp_buff;
|
| 31 | dma_addr_t dummy_comp_buff_dma;
|
| 32 |
|
| 33 |
|
| 34 | struct list_head backlog;
|
| 35 | unsigned int bl_len;
|
| 36 | spinlock_t bl_lock;
|
| 37 |
|
| 38 | #ifdef COMP_IN_WQ
|
| 39 | struct workqueue_struct *workq;
|
| 40 | struct delayed_work compwork;
|
| 41 | #else
|
| 42 | struct tasklet_struct comptask;
|
| 43 | #endif
|
| 44 | };
|
| 45 |
|
| 46 | struct cc_bl_item {
|
| 47 | struct cc_crypto_req creq;
|
| 48 | struct cc_hw_desc desc[CC_MAX_DESC_SEQ_LEN];
|
| 49 | unsigned int len;
|
| 50 | struct list_head list;
|
| 51 | bool notif;
|
| 52 | };
|
| 53 |
|
| 54 | static const u32 cc_cpp_int_masks[CC_CPP_NUM_ALGS][CC_CPP_NUM_SLOTS] = {
|
| 55 | { BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_0_INT_BIT_SHIFT),
|
| 56 | BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_1_INT_BIT_SHIFT),
|
| 57 | BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_2_INT_BIT_SHIFT),
|
| 58 | BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_3_INT_BIT_SHIFT),
|
| 59 | BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_4_INT_BIT_SHIFT),
|
| 60 | BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_5_INT_BIT_SHIFT),
|
| 61 | BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_6_INT_BIT_SHIFT),
|
| 62 | BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_7_INT_BIT_SHIFT) },
|
| 63 | { BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_0_INT_BIT_SHIFT),
|
| 64 | BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_1_INT_BIT_SHIFT),
|
| 65 | BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_2_INT_BIT_SHIFT),
|
| 66 | BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_3_INT_BIT_SHIFT),
|
| 67 | BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_4_INT_BIT_SHIFT),
|
| 68 | BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_5_INT_BIT_SHIFT),
|
| 69 | BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_6_INT_BIT_SHIFT),
|
| 70 | BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_7_INT_BIT_SHIFT) }
|
| 71 | };
|
| 72 |
|
| 73 | static void comp_handler(unsigned long devarg);
|
| 74 | #ifdef COMP_IN_WQ
|
| 75 | static void comp_work_handler(struct work_struct *work);
|
| 76 | #endif
|
| 77 |
|
| 78 | static inline u32 cc_cpp_int_mask(enum cc_cpp_alg alg, int slot)
|
| 79 | {
|
| 80 | alg = array_index_nospec(alg, CC_CPP_NUM_ALGS);
|
| 81 | slot = array_index_nospec(slot, CC_CPP_NUM_SLOTS);
|
| 82 |
|
| 83 | return cc_cpp_int_masks[alg][slot];
|
| 84 | }
|
| 85 |
|
| 86 | void cc_req_mgr_fini(struct cc_drvdata *drvdata)
|
| 87 | {
|
| 88 | struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
|
| 89 | struct device *dev = drvdata_to_dev(drvdata);
|
| 90 |
|
| 91 | if (!req_mgr_h)
|
| 92 | return;
|
| 93 |
|
| 94 | if (req_mgr_h->dummy_comp_buff_dma) {
|
| 95 | dma_free_coherent(dev, sizeof(u32), req_mgr_h->dummy_comp_buff,
|
| 96 | req_mgr_h->dummy_comp_buff_dma);
|
| 97 | }
|
| 98 |
|
| 99 | dev_dbg(dev, "max_used_hw_slots=%d\n", (req_mgr_h->hw_queue_size -
|
| 100 | req_mgr_h->min_free_hw_slots));
|
| 101 | dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots);
|
| 102 |
|
| 103 | #ifdef COMP_IN_WQ
|
| 104 | destroy_workqueue(req_mgr_h->workq);
|
| 105 | #else
|
| 106 |
|
| 107 | tasklet_kill(&req_mgr_h->comptask);
|
| 108 | #endif
|
| 109 | kfree_sensitive(req_mgr_h);
|
| 110 | drvdata->request_mgr_handle = NULL;
|
| 111 | }
|
| 112 |
|
| 113 | int cc_req_mgr_init(struct cc_drvdata *drvdata)
|
| 114 | {
|
| 115 | struct cc_req_mgr_handle *req_mgr_h;
|
| 116 | struct device *dev = drvdata_to_dev(drvdata);
|
| 117 | int rc = 0;
|
| 118 |
|
| 119 | req_mgr_h = kzalloc_obj(*req_mgr_h);
|
| 120 | if (!req_mgr_h) {
|
| 121 | rc = -ENOMEM;
|
| 122 | goto req_mgr_init_err;
|
| 123 | }
|
| 124 |
|
| 125 | drvdata->request_mgr_handle = req_mgr_h;
|
| 126 |
|
| 127 | spin_lock_init(&req_mgr_h->hw_lock);
|
| 128 | spin_lock_init(&req_mgr_h->bl_lock);
|
| 129 | INIT_LIST_HEAD(&req_mgr_h->backlog);
|
| 130 |
|
| 131 | #ifdef COMP_IN_WQ
|
| 132 | dev_dbg(dev, "Initializing completion workqueue\n");
|
| 133 | req_mgr_h->workq = create_singlethread_workqueue("ccree");
|
| 134 | if (!req_mgr_h->workq) {
|
| 135 | dev_err(dev, "Failed creating work queue\n");
|
| 136 | rc = -ENOMEM;
|
| 137 | goto req_mgr_init_err;
|
| 138 | }
|
| 139 | INIT_DELAYED_WORK(&req_mgr_h->compwork, comp_work_handler);
|
| 140 | #else
|
| 141 | dev_dbg(dev, "Initializing completion tasklet\n");
|
| 142 | tasklet_init(&req_mgr_h->comptask, comp_handler,
|
| 143 | (unsigned long)drvdata);
|
| 144 | #endif
|
| 145 | req_mgr_h->hw_queue_size = cc_ioread(drvdata,
|
| 146 | CC_REG(DSCRPTR_QUEUE_SRAM_SIZE));
|
| 147 | dev_dbg(dev, "hw_queue_size=0x%08X\n", req_mgr_h->hw_queue_size);
|
| 148 | if (req_mgr_h->hw_queue_size < MIN_HW_QUEUE_SIZE) {
|
| 149 | dev_err(dev, "Invalid HW queue size = %u (Min. required is %u)\n",
|
| 150 | req_mgr_h->hw_queue_size, MIN_HW_QUEUE_SIZE);
|
| 151 | rc = -ENOMEM;
|
| 152 | goto req_mgr_init_err;
|
| 153 | }
|
| 154 | req_mgr_h->min_free_hw_slots = req_mgr_h->hw_queue_size;
|
| 155 | req_mgr_h->max_used_sw_slots = 0;
|
| 156 |
|
| 157 |
|
| 158 | req_mgr_h->dummy_comp_buff =
|
| 159 | dma_alloc_coherent(dev, sizeof(u32),
|
| 160 | &req_mgr_h->dummy_comp_buff_dma,
|
| 161 | GFP_KERNEL);
|
| 162 | if (!req_mgr_h->dummy_comp_buff) {
|
| 163 | dev_err(dev, "Not enough memory to allocate DMA (%zu) dropped buffer\n",
|
| 164 | sizeof(u32));
|
| 165 | rc = -ENOMEM;
|
| 166 | goto req_mgr_init_err;
|
| 167 | }
|
| 168 |
|
| 169 |
|
| 170 | hw_desc_init(&req_mgr_h->compl_desc);
|
| 171 | set_din_const(&req_mgr_h->compl_desc, 0, sizeof(u32));
|
| 172 | set_dout_dlli(&req_mgr_h->compl_desc, req_mgr_h->dummy_comp_buff_dma,
|
| 173 | sizeof(u32), NS_BIT, 1);
|
| 174 | set_flow_mode(&req_mgr_h->compl_desc, BYPASS);
|
| 175 | set_queue_last_ind(drvdata, &req_mgr_h->compl_desc);
|
| 176 |
|
| 177 | return 0;
|
| 178 |
|
| 179 | req_mgr_init_err:
|
| 180 | cc_req_mgr_fini(drvdata);
|
| 181 | return rc;
|
| 182 | }
|
| 183 |
|
| 184 | static void enqueue_seq(struct cc_drvdata *drvdata, struct cc_hw_desc seq[],
|
| 185 | unsigned int seq_len)
|
| 186 | {
|
| 187 | int i, w;
|
| 188 | void __iomem *reg = drvdata->cc_base + CC_REG(DSCRPTR_QUEUE_WORD0);
|
| 189 | struct device *dev = drvdata_to_dev(drvdata);
|
| 190 |
|
| 191 | |
| 192 | * We do indeed write all 6 command words to the same
|
| 193 | * register. The HW supports this.
|
| 194 |
|
| 195 |
|
| 196 | for (i = 0; i < seq_len; i++) {
|
| 197 | for (w = 0; w <= 5; w++)
|
| 198 | writel_relaxed(seq[i].word[w], reg);
|
| 199 |
|
| 200 | if (cc_dump_desc)
|
| 201 | dev_dbg(dev, "desc[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
|
| 202 | i, seq[i].word[0], seq[i].word[1],
|
| 203 | seq[i].word[2], seq[i].word[3],
|
| 204 | seq[i].word[4], seq[i].word[5]);
|
| 205 | }
|
| 206 | }
|
| 207 |
|
| 208 | |
| 209 | * request_mgr_complete() - Completion will take place if and only if user
|
| 210 | * requested completion by cc_send_sync_request().
|
| 211 | *
|
| 212 | * @dev: Device pointer
|
| 213 | * @dx_compl_h: The completion event to signal
|
| 214 | * @dummy: unused error code
|
| 215 |
|
| 216 | static void request_mgr_complete(struct device *dev, void *dx_compl_h,
|
| 217 | int dummy)
|
| 218 | {
|
| 219 | struct completion *this_compl = dx_compl_h;
|
| 220 |
|
| 221 | complete(this_compl);
|
| 222 | }
|
| 223 |
|
| 224 | static int cc_queues_status(struct cc_drvdata *drvdata,
|
| 225 | struct cc_req_mgr_handle *req_mgr_h,
|
| 226 | unsigned int total_seq_len)
|
| 227 | {
|
| 228 | unsigned long poll_queue;
|
| 229 | struct device *dev = drvdata_to_dev(drvdata);
|
| 230 |
|
| 231 | |
| 232 | * be changed during the poll because the spinlock_bh
|
| 233 | * is held by the thread
|
| 234 |
|
| 235 | if (((req_mgr_h->req_queue_head + 1) & (MAX_REQUEST_QUEUE_SIZE - 1)) ==
|
| 236 | req_mgr_h->req_queue_tail) {
|
| 237 | dev_err(dev, "SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n",
|
| 238 | req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE);
|
| 239 | return -ENOSPC;
|
| 240 | }
|
| 241 |
|
| 242 | if (req_mgr_h->q_free_slots >= total_seq_len)
|
| 243 | return 0;
|
| 244 |
|
| 245 |
|
| 246 | for (poll_queue = 0; poll_queue < CC_MAX_POLL_ITER ; poll_queue++) {
|
| 247 | req_mgr_h->q_free_slots =
|
| 248 | cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));
|
| 249 | if (req_mgr_h->q_free_slots < req_mgr_h->min_free_hw_slots)
|
| 250 | req_mgr_h->min_free_hw_slots = req_mgr_h->q_free_slots;
|
| 251 |
|
| 252 | if (req_mgr_h->q_free_slots >= total_seq_len) {
|
| 253 |
|
| 254 | return 0;
|
| 255 | }
|
| 256 |
|
| 257 | dev_dbg(dev, "HW FIFO is full. q_free_slots=%d total_seq_len=%d\n",
|
| 258 | req_mgr_h->q_free_slots, total_seq_len);
|
| 259 | }
|
| 260 |
|
| 261 | dev_dbg(dev, "HW FIFO full, timeout. req_queue_head=%d sw_fifo_len=%d q_free_slots=%d total_seq_len=%d\n",
|
| 262 | req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE,
|
| 263 | req_mgr_h->q_free_slots, total_seq_len);
|
| 264 | return -ENOSPC;
|
| 265 | }
|
| 266 |
|
| 267 | |
| 268 | * cc_do_send_request() - Enqueue caller request to crypto hardware.
|
| 269 | * Need to be called with HW lock held and PM running
|
| 270 | *
|
| 271 | * @drvdata: Associated device driver context
|
| 272 | * @cc_req: The request to enqueue
|
| 273 | * @desc: The crypto sequence
|
| 274 | * @len: The crypto sequence length
|
| 275 | * @add_comp: If "true": add an artificial dout DMA to mark completion
|
| 276 | *
|
| 277 |
|
| 278 | static void cc_do_send_request(struct cc_drvdata *drvdata,
|
| 279 | struct cc_crypto_req *cc_req,
|
| 280 | struct cc_hw_desc *desc, unsigned int len,
|
| 281 | bool add_comp)
|
| 282 | {
|
| 283 | struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
|
| 284 | unsigned int used_sw_slots;
|
| 285 | unsigned int total_seq_len = len;
|
| 286 | struct device *dev = drvdata_to_dev(drvdata);
|
| 287 |
|
| 288 | used_sw_slots = ((req_mgr_h->req_queue_head -
|
| 289 | req_mgr_h->req_queue_tail) &
|
| 290 | (MAX_REQUEST_QUEUE_SIZE - 1));
|
| 291 | if (used_sw_slots > req_mgr_h->max_used_sw_slots)
|
| 292 | req_mgr_h->max_used_sw_slots = used_sw_slots;
|
| 293 |
|
| 294 |
|
| 295 | req_mgr_h->req_queue[req_mgr_h->req_queue_head] = *cc_req;
|
| 296 | req_mgr_h->req_queue_head = (req_mgr_h->req_queue_head + 1) &
|
| 297 | (MAX_REQUEST_QUEUE_SIZE - 1);
|
| 298 |
|
| 299 | dev_dbg(dev, "Enqueue request head=%u\n", req_mgr_h->req_queue_head);
|
| 300 |
|
| 301 | |
| 302 | * We are about to push command to the HW via the command registers
|
| 303 | * that may reference host memory. We need to issue a memory barrier
|
| 304 | * to make sure there are no outstanding memory writes
|
| 305 |
|
| 306 | wmb();
|
| 307 |
|
| 308 |
|
| 309 |
|
| 310 | enqueue_seq(drvdata, desc, len);
|
| 311 |
|
| 312 | if (add_comp) {
|
| 313 | enqueue_seq(drvdata, &req_mgr_h->compl_desc, 1);
|
| 314 | total_seq_len++;
|
| 315 | }
|
| 316 |
|
| 317 | if (req_mgr_h->q_free_slots < total_seq_len) {
|
| 318 | |
| 319 | * with resuming power. Set the free slot count to 0 and hope
|
| 320 | * for the best.
|
| 321 |
|
| 322 | dev_err(dev, "HW free slot count mismatch.");
|
| 323 | req_mgr_h->q_free_slots = 0;
|
| 324 | } else {
|
| 325 |
|
| 326 | req_mgr_h->q_free_slots -= total_seq_len;
|
| 327 | }
|
| 328 | }
|
| 329 |
|
| 330 | static void cc_enqueue_backlog(struct cc_drvdata *drvdata,
|
| 331 | struct cc_bl_item *bli)
|
| 332 | {
|
| 333 | struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
|
| 334 | struct device *dev = drvdata_to_dev(drvdata);
|
| 335 |
|
| 336 | spin_lock_bh(&mgr->bl_lock);
|
| 337 | list_add_tail(&bli->list, &mgr->backlog);
|
| 338 | ++mgr->bl_len;
|
| 339 | dev_dbg(dev, "+++bl len: %d\n", mgr->bl_len);
|
| 340 | spin_unlock_bh(&mgr->bl_lock);
|
| 341 | tasklet_schedule(&mgr->comptask);
|
| 342 | }
|
| 343 |
|
| 344 | static void cc_proc_backlog(struct cc_drvdata *drvdata)
|
| 345 | {
|
| 346 | struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
|
| 347 | struct cc_bl_item *bli;
|
| 348 | struct cc_crypto_req *creq;
|
| 349 | void *req;
|
| 350 | struct device *dev = drvdata_to_dev(drvdata);
|
| 351 | int rc;
|
| 352 |
|
| 353 | spin_lock(&mgr->bl_lock);
|
| 354 |
|
| 355 | while (mgr->bl_len) {
|
| 356 | bli = list_first_entry(&mgr->backlog, struct cc_bl_item, list);
|
| 357 | dev_dbg(dev, "---bl len: %d\n", mgr->bl_len);
|
| 358 |
|
| 359 | spin_unlock(&mgr->bl_lock);
|
| 360 |
|
| 361 |
|
| 362 | creq = &bli->creq;
|
| 363 | req = creq->user_arg;
|
| 364 |
|
| 365 | |
| 366 | * Notify the request we're moving out of the backlog
|
| 367 | * but only if we haven't done so already.
|
| 368 |
|
| 369 | if (!bli->notif) {
|
| 370 | creq->user_cb(dev, req, -EINPROGRESS);
|
| 371 | bli->notif = true;
|
| 372 | }
|
| 373 |
|
| 374 | spin_lock(&mgr->hw_lock);
|
| 375 |
|
| 376 | rc = cc_queues_status(drvdata, mgr, bli->len);
|
| 377 | if (rc) {
|
| 378 | |
| 379 | * There is still no room in the FIFO for
|
| 380 | * this request. Bail out. We'll return here
|
| 381 | * on the next completion irq.
|
| 382 |
|
| 383 | spin_unlock(&mgr->hw_lock);
|
| 384 | return;
|
| 385 | }
|
| 386 |
|
| 387 | cc_do_send_request(drvdata, &bli->creq, bli->desc, bli->len,
|
| 388 | false);
|
| 389 | spin_unlock(&mgr->hw_lock);
|
| 390 |
|
| 391 |
|
| 392 | spin_lock(&mgr->bl_lock);
|
| 393 | list_del(&bli->list);
|
| 394 | --mgr->bl_len;
|
| 395 | kfree(bli);
|
| 396 | }
|
| 397 |
|
| 398 | spin_unlock(&mgr->bl_lock);
|
| 399 | }
|
| 400 |
|
| 401 | int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req,
|
| 402 | struct cc_hw_desc *desc, unsigned int len,
|
| 403 | struct crypto_async_request *req)
|
| 404 | {
|
| 405 | int rc;
|
| 406 | struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
|
| 407 | struct device *dev = drvdata_to_dev(drvdata);
|
| 408 | bool backlog_ok = req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG;
|
| 409 | gfp_t flags = cc_gfp_flags(req);
|
| 410 | struct cc_bl_item *bli;
|
| 411 |
|
| 412 | rc = cc_pm_get(dev);
|
| 413 | if (rc) {
|
| 414 | dev_err(dev, "cc_pm_get returned %x\n", rc);
|
| 415 | return rc;
|
| 416 | }
|
| 417 |
|
| 418 | spin_lock_bh(&mgr->hw_lock);
|
| 419 | rc = cc_queues_status(drvdata, mgr, len);
|
| 420 |
|
| 421 | #ifdef CC_DEBUG_FORCE_BACKLOG
|
| 422 | if (backlog_ok)
|
| 423 | rc = -ENOSPC;
|
| 424 | #endif
|
| 425 |
|
| 426 | if (rc == -ENOSPC && backlog_ok) {
|
| 427 | spin_unlock_bh(&mgr->hw_lock);
|
| 428 |
|
| 429 | bli = kmalloc_obj(*bli, flags);
|
| 430 | if (!bli) {
|
| 431 | cc_pm_put_suspend(dev);
|
| 432 | return -ENOMEM;
|
| 433 | }
|
| 434 |
|
| 435 | memcpy(&bli->creq, cc_req, sizeof(*cc_req));
|
| 436 | memcpy(&bli->desc, desc, len * sizeof(*desc));
|
| 437 | bli->len = len;
|
| 438 | bli->notif = false;
|
| 439 | cc_enqueue_backlog(drvdata, bli);
|
| 440 | return -EBUSY;
|
| 441 | }
|
| 442 |
|
| 443 | if (!rc) {
|
| 444 | cc_do_send_request(drvdata, cc_req, desc, len, false);
|
| 445 | rc = -EINPROGRESS;
|
| 446 | }
|
| 447 |
|
| 448 | spin_unlock_bh(&mgr->hw_lock);
|
| 449 | return rc;
|
| 450 | }
|
| 451 |
|
| 452 | int cc_send_sync_request(struct cc_drvdata *drvdata,
|
| 453 | struct cc_crypto_req *cc_req, struct cc_hw_desc *desc,
|
| 454 | unsigned int len)
|
| 455 | {
|
| 456 | int rc;
|
| 457 | struct device *dev = drvdata_to_dev(drvdata);
|
| 458 | struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
|
| 459 |
|
| 460 | init_completion(&cc_req->seq_compl);
|
| 461 | cc_req->user_cb = request_mgr_complete;
|
| 462 | cc_req->user_arg = &cc_req->seq_compl;
|
| 463 |
|
| 464 | rc = cc_pm_get(dev);
|
| 465 | if (rc) {
|
| 466 | dev_err(dev, "cc_pm_get returned %x\n", rc);
|
| 467 | return rc;
|
| 468 | }
|
| 469 |
|
| 470 | while (true) {
|
| 471 | spin_lock_bh(&mgr->hw_lock);
|
| 472 | rc = cc_queues_status(drvdata, mgr, len + 1);
|
| 473 |
|
| 474 | if (!rc)
|
| 475 | break;
|
| 476 |
|
| 477 | spin_unlock_bh(&mgr->hw_lock);
|
| 478 | wait_for_completion_interruptible(&drvdata->hw_queue_avail);
|
| 479 | reinit_completion(&drvdata->hw_queue_avail);
|
| 480 | }
|
| 481 |
|
| 482 | cc_do_send_request(drvdata, cc_req, desc, len, true);
|
| 483 | spin_unlock_bh(&mgr->hw_lock);
|
| 484 | wait_for_completion(&cc_req->seq_compl);
|
| 485 | return 0;
|
| 486 | }
|
| 487 |
|
| 488 | |
| 489 | * send_request_init() - Enqueue caller request to crypto hardware during init
|
| 490 | * process.
|
| 491 | * Assume this function is not called in the middle of a flow,
|
| 492 | * since we set QUEUE_LAST_IND flag in the last descriptor.
|
| 493 | *
|
| 494 | * @drvdata: Associated device driver context
|
| 495 | * @desc: The crypto sequence
|
| 496 | * @len: The crypto sequence length
|
| 497 | *
|
| 498 | * Return:
|
| 499 | * Returns "0" upon success
|
| 500 |
|
| 501 | int send_request_init(struct cc_drvdata *drvdata, struct cc_hw_desc *desc,
|
| 502 | unsigned int len)
|
| 503 | {
|
| 504 | struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
|
| 505 | unsigned int total_seq_len = len;
|
| 506 | int rc = 0;
|
| 507 |
|
| 508 | |
| 509 |
|
| 510 | rc = cc_queues_status(drvdata, req_mgr_h, total_seq_len);
|
| 511 | if (rc)
|
| 512 | return rc;
|
| 513 |
|
| 514 | set_queue_last_ind(drvdata, &desc[(len - 1)]);
|
| 515 |
|
| 516 | |
| 517 | * We are about to push command to the HW via the command registers
|
| 518 | * that may reference host memory. We need to issue a memory barrier
|
| 519 | * to make sure there are no outstanding memory writes
|
| 520 |
|
| 521 | wmb();
|
| 522 | enqueue_seq(drvdata, desc, len);
|
| 523 |
|
| 524 |
|
| 525 | req_mgr_h->q_free_slots =
|
| 526 | cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));
|
| 527 |
|
| 528 | return 0;
|
| 529 | }
|
| 530 |
|
| 531 | void complete_request(struct cc_drvdata *drvdata)
|
| 532 | {
|
| 533 | struct cc_req_mgr_handle *request_mgr_handle =
|
| 534 | drvdata->request_mgr_handle;
|
| 535 |
|
| 536 | complete(&drvdata->hw_queue_avail);
|
| 537 | #ifdef COMP_IN_WQ
|
| 538 | queue_delayed_work(request_mgr_handle->workq,
|
| 539 | &request_mgr_handle->compwork, 0);
|
| 540 | #else
|
| 541 | tasklet_schedule(&request_mgr_handle->comptask);
|
| 542 | #endif
|
| 543 | }
|
| 544 |
|
| 545 | #ifdef COMP_IN_WQ
|
| 546 | static void comp_work_handler(struct work_struct *work)
|
| 547 | {
|
| 548 | struct cc_drvdata *drvdata =
|
| 549 | container_of(work, struct cc_drvdata, compwork.work);
|
| 550 |
|
| 551 | comp_handler((unsigned long)drvdata);
|
| 552 | }
|
| 553 | #endif
|
| 554 |
|
| 555 | static void proc_completions(struct cc_drvdata *drvdata)
|
| 556 | {
|
| 557 | struct cc_crypto_req *cc_req;
|
| 558 | struct device *dev = drvdata_to_dev(drvdata);
|
| 559 | struct cc_req_mgr_handle *request_mgr_handle =
|
| 560 | drvdata->request_mgr_handle;
|
| 561 | unsigned int *tail = &request_mgr_handle->req_queue_tail;
|
| 562 | unsigned int *head = &request_mgr_handle->req_queue_head;
|
| 563 | int rc;
|
| 564 | u32 mask;
|
| 565 |
|
| 566 | while (request_mgr_handle->axi_completed) {
|
| 567 | request_mgr_handle->axi_completed--;
|
| 568 |
|
| 569 |
|
| 570 | if (*head == *tail) {
|
| 571 | |
| 572 | * queue is empty. This is not normal. Return and
|
| 573 | * hope for the best.
|
| 574 |
|
| 575 | dev_err(dev, "Request queue is empty head == tail %u\n",
|
| 576 | *head);
|
| 577 | break;
|
| 578 | }
|
| 579 |
|
| 580 | cc_req = &request_mgr_handle->req_queue[*tail];
|
| 581 |
|
| 582 | if (cc_req->cpp.is_cpp) {
|
| 583 |
|
| 584 | dev_dbg(dev, "CPP request completion slot: %d alg:%d\n",
|
| 585 | cc_req->cpp.slot, cc_req->cpp.alg);
|
| 586 | mask = cc_cpp_int_mask(cc_req->cpp.alg,
|
| 587 | cc_req->cpp.slot);
|
| 588 | rc = (drvdata->irq & mask ? -EPERM : 0);
|
| 589 | dev_dbg(dev, "Got mask: %x irq: %x rc: %d\n", mask,
|
| 590 | drvdata->irq, rc);
|
| 591 | } else {
|
| 592 | dev_dbg(dev, "None CPP request completion\n");
|
| 593 | rc = 0;
|
| 594 | }
|
| 595 |
|
| 596 | if (cc_req->user_cb)
|
| 597 | cc_req->user_cb(dev, cc_req->user_arg, rc);
|
| 598 | *tail = (*tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1);
|
| 599 | dev_dbg(dev, "Dequeue request tail=%u\n", *tail);
|
| 600 | dev_dbg(dev, "Request completed. axi_completed=%d\n",
|
| 601 | request_mgr_handle->axi_completed);
|
| 602 | cc_pm_put_suspend(dev);
|
| 603 | }
|
| 604 | }
|
| 605 |
|
| 606 | static inline u32 cc_axi_comp_count(struct cc_drvdata *drvdata)
|
| 607 | {
|
| 608 | return FIELD_GET(AXIM_MON_COMP_VALUE,
|
| 609 | cc_ioread(drvdata, drvdata->axim_mon_offset));
|
| 610 | }
|
| 611 |
|
| 612 |
|
| 613 | static void comp_handler(unsigned long devarg)
|
| 614 | {
|
| 615 | struct cc_drvdata *drvdata = (struct cc_drvdata *)devarg;
|
| 616 | struct cc_req_mgr_handle *request_mgr_handle =
|
| 617 | drvdata->request_mgr_handle;
|
| 618 | struct device *dev = drvdata_to_dev(drvdata);
|
| 619 | u32 irq;
|
| 620 |
|
| 621 | dev_dbg(dev, "Completion handler called!\n");
|
| 622 | irq = (drvdata->irq & drvdata->comp_mask);
|
| 623 |
|
| 624 | |
| 625 | * we clear it now
|
| 626 |
|
| 627 | cc_iowrite(drvdata, CC_REG(HOST_ICR), irq);
|
| 628 |
|
| 629 |
|
| 630 |
|
| 631 | request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata);
|
| 632 |
|
| 633 | dev_dbg(dev, "AXI completion after updated: %d\n",
|
| 634 | request_mgr_handle->axi_completed);
|
| 635 |
|
| 636 | while (request_mgr_handle->axi_completed) {
|
| 637 | do {
|
| 638 | drvdata->irq |= cc_ioread(drvdata, CC_REG(HOST_IRR));
|
| 639 | irq = (drvdata->irq & drvdata->comp_mask);
|
| 640 | proc_completions(drvdata);
|
| 641 |
|
| 642 | |
| 643 | * request_mgr_handle->axi_completed is 0.
|
| 644 |
|
| 645 | request_mgr_handle->axi_completed +=
|
| 646 | cc_axi_comp_count(drvdata);
|
| 647 | } while (request_mgr_handle->axi_completed > 0);
|
| 648 |
|
| 649 | cc_iowrite(drvdata, CC_REG(HOST_ICR), irq);
|
| 650 |
|
| 651 | request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata);
|
| 652 | }
|
| 653 |
|
| 654 | |
| 655 | * unmask AXI completion interrupt
|
| 656 |
|
| 657 | cc_iowrite(drvdata, CC_REG(HOST_IMR),
|
| 658 | cc_ioread(drvdata, CC_REG(HOST_IMR)) & ~drvdata->comp_mask);
|
| 659 |
|
| 660 | cc_proc_backlog(drvdata);
|
| 661 | dev_dbg(dev, "Comp. handler done.\n");
|
| 662 | }
|