projects/linux·blob master · refs

489 lines · 12.7 KiB

1// SPDX-License-Identifier: GPL-2.0-only OR MIT
2/* Copyright (c) 2023 Imagination Technologies Ltd. */
3
4#include "pvr_cccb.h"
5#include "pvr_context.h"
6#include "pvr_device.h"
7#include "pvr_drv.h"
8#include "pvr_gem.h"
9#include "pvr_job.h"
10#include "pvr_power.h"
11#include "pvr_rogue_fwif.h"
12#include "pvr_rogue_fwif_common.h"
13#include "pvr_rogue_fwif_resetframework.h"
14#include "pvr_stream.h"
15#include "pvr_stream_defs.h"
16#include "pvr_vm.h"
17
18#include <drm/drm_auth.h>
19#include <drm/drm_managed.h>
20
21#include <linux/bug.h>
22#include <linux/errno.h>
23#include <linux/kernel.h>
24#include <linux/list.h>
25#include <linux/sched.h>
26#include <linux/slab.h>
27#include <linux/spinlock.h>
28#include <linux/string.h>
29#include <linux/types.h>
30#include <linux/xarray.h>
31
32static int
33remap_priority(struct pvr_file *pvr_file, s32 uapi_priority,
34 enum pvr_context_priority *priority_out)
35{
36 switch (uapi_priority) {
37 case DRM_PVR_CTX_PRIORITY_LOW:
38 *priority_out = PVR_CTX_PRIORITY_LOW;
39 break;
40 case DRM_PVR_CTX_PRIORITY_NORMAL:
41 *priority_out = PVR_CTX_PRIORITY_MEDIUM;
42 break;
43 case DRM_PVR_CTX_PRIORITY_HIGH:
44 if (!capable(CAP_SYS_NICE) && !drm_is_current_master(from_pvr_file(pvr_file)))
45 return -EACCES;
46 *priority_out = PVR_CTX_PRIORITY_HIGH;
47 break;
48 default:
49 return -EINVAL;
50 }
51
52 return 0;
53}
54
55static int get_fw_obj_size(enum drm_pvr_ctx_type type)
56{
57 switch (type) {
58 case DRM_PVR_CTX_TYPE_RENDER:
59 return sizeof(struct rogue_fwif_fwrendercontext);
60 case DRM_PVR_CTX_TYPE_COMPUTE:
61 return sizeof(struct rogue_fwif_fwcomputecontext);
62 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG:
63 return sizeof(struct rogue_fwif_fwtransfercontext);
64 }
65
66 return -EINVAL;
67}
68
69static int
70process_static_context_state(struct pvr_device *pvr_dev, const struct pvr_stream_cmd_defs *cmd_defs,
71 u64 stream_user_ptr, u32 stream_size, void *dest)
72{
73 void *stream;
74 int err;
75
76 stream = memdup_user(u64_to_user_ptr(stream_user_ptr), stream_size);
77 if (IS_ERR(stream))
78 return PTR_ERR(stream);
79
80 err = pvr_stream_process(pvr_dev, cmd_defs, stream, stream_size, dest);
81
82 kfree(stream);
83
84 return err;
85}
86
87static int init_render_fw_objs(struct pvr_context *ctx,
88 struct drm_pvr_ioctl_create_context_args *args,
89 void *fw_ctx_map)
90{
91 struct rogue_fwif_static_rendercontext_state *static_rendercontext_state;
92 struct rogue_fwif_fwrendercontext *fw_render_context = fw_ctx_map;
93
94 if (!args->static_context_state_len)
95 return -EINVAL;
96
97 static_rendercontext_state = &fw_render_context->static_render_context_state;
98
99 /* Copy static render context state from userspace. */
100 return process_static_context_state(ctx->pvr_dev,
101 &pvr_static_render_context_state_stream,
102 args->static_context_state,
103 args->static_context_state_len,
104 &static_rendercontext_state->ctxswitch_regs[0]);
105}
106
107static int init_compute_fw_objs(struct pvr_context *ctx,
108 struct drm_pvr_ioctl_create_context_args *args,
109 void *fw_ctx_map)
110{
111 struct rogue_fwif_fwcomputecontext *fw_compute_context = fw_ctx_map;
112 struct rogue_fwif_cdm_registers_cswitch *ctxswitch_regs;
113
114 if (!args->static_context_state_len)
115 return -EINVAL;
116
117 ctxswitch_regs = &fw_compute_context->static_compute_context_state.ctxswitch_regs;
118
119 /* Copy static render context state from userspace. */
120 return process_static_context_state(ctx->pvr_dev,
121 &pvr_static_compute_context_state_stream,
122 args->static_context_state,
123 args->static_context_state_len,
124 ctxswitch_regs);
125}
126
127static int init_transfer_fw_objs(struct pvr_context *ctx,
128 struct drm_pvr_ioctl_create_context_args *args,
129 void *fw_ctx_map)
130{
131 if (args->static_context_state_len)
132 return -EINVAL;
133
134 return 0;
135}
136
137static int init_fw_objs(struct pvr_context *ctx,
138 struct drm_pvr_ioctl_create_context_args *args,
139 void *fw_ctx_map)
140{
141 switch (ctx->type) {
142 case DRM_PVR_CTX_TYPE_RENDER:
143 return init_render_fw_objs(ctx, args, fw_ctx_map);
144 case DRM_PVR_CTX_TYPE_COMPUTE:
145 return init_compute_fw_objs(ctx, args, fw_ctx_map);
146 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG:
147 return init_transfer_fw_objs(ctx, args, fw_ctx_map);
148 }
149
150 return -EINVAL;
151}
152
153static void
154ctx_fw_data_init(void *cpu_ptr, void *priv)
155{
156 struct pvr_context *ctx = priv;
157
158 memcpy(cpu_ptr, ctx->data, ctx->data_size);
159}
160
161/**
162 * pvr_context_destroy_queues() - Destroy all queues attached to a context.
163 * @ctx: Context to destroy queues on.
164 * @cleanup_queue_entity: Whether to cleanup the queue entity e.g. context
165 * creation failure path.
166 *
167 * Should be called when the last reference to a context object is dropped.
168 * It releases all resources attached to the queues bound to this context.
169 */
170static void pvr_context_destroy_queues(struct pvr_context *ctx, bool cleanup_queue_entity)
171{
172 switch (ctx->type) {
173 case DRM_PVR_CTX_TYPE_RENDER:
174 pvr_queue_destroy(ctx->queues.fragment, cleanup_queue_entity);
175 pvr_queue_destroy(ctx->queues.geometry, cleanup_queue_entity);
176 break;
177 case DRM_PVR_CTX_TYPE_COMPUTE:
178 pvr_queue_destroy(ctx->queues.compute, cleanup_queue_entity);
179 break;
180 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG:
181 pvr_queue_destroy(ctx->queues.transfer, cleanup_queue_entity);
182 break;
183 }
184}
185
186/**
187 * pvr_context_create_queues() - Create all queues attached to a context.
188 * @ctx: Context to create queues on.
189 * @args: Context creation arguments passed by userspace.
190 * @fw_ctx_map: CPU mapping of the FW context object.
191 *
192 * Return:
193 * * 0 on success, or
194 * * A negative error code otherwise.
195 */
196static int pvr_context_create_queues(struct pvr_context *ctx,
197 struct drm_pvr_ioctl_create_context_args *args,
198 void *fw_ctx_map)
199{
200 int err;
201
202 switch (ctx->type) {
203 case DRM_PVR_CTX_TYPE_RENDER:
204 ctx->queues.geometry = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_GEOMETRY,
205 args, fw_ctx_map);
206 if (IS_ERR(ctx->queues.geometry)) {
207 err = PTR_ERR(ctx->queues.geometry);
208 ctx->queues.geometry = NULL;
209 goto err_destroy_queues;
210 }
211
212 ctx->queues.fragment = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_FRAGMENT,
213 args, fw_ctx_map);
214 if (IS_ERR(ctx->queues.fragment)) {
215 err = PTR_ERR(ctx->queues.fragment);
216 ctx->queues.fragment = NULL;
217 goto err_destroy_queues;
218 }
219 return 0;
220
221 case DRM_PVR_CTX_TYPE_COMPUTE:
222 ctx->queues.compute = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_COMPUTE,
223 args, fw_ctx_map);
224 if (IS_ERR(ctx->queues.compute)) {
225 err = PTR_ERR(ctx->queues.compute);
226 ctx->queues.compute = NULL;
227 goto err_destroy_queues;
228 }
229 return 0;
230
231 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG:
232 ctx->queues.transfer = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_TRANSFER_FRAG,
233 args, fw_ctx_map);
234 if (IS_ERR(ctx->queues.transfer)) {
235 err = PTR_ERR(ctx->queues.transfer);
236 ctx->queues.transfer = NULL;
237 goto err_destroy_queues;
238 }
239 return 0;
240 }
241
242 return -EINVAL;
243
244err_destroy_queues:
245 pvr_context_destroy_queues(ctx, true);
246 return err;
247}
248
249/**
250 * pvr_context_kill_queues() - Kill queues attached to context.
251 * @ctx: Context to kill queues on.
252 *
253 * Killing the queues implies making them unusable for future jobs, while still
254 * letting the currently submitted jobs a chance to finish. Queue resources will
255 * stay around until pvr_context_destroy_queues() is called.
256 */
257static void pvr_context_kill_queues(struct pvr_context *ctx)
258{
259 switch (ctx->type) {
260 case DRM_PVR_CTX_TYPE_RENDER:
261 pvr_queue_kill(ctx->queues.fragment);
262 pvr_queue_kill(ctx->queues.geometry);
263 break;
264 case DRM_PVR_CTX_TYPE_COMPUTE:
265 pvr_queue_kill(ctx->queues.compute);
266 break;
267 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG:
268 pvr_queue_kill(ctx->queues.transfer);
269 break;
270 }
271}
272
273/**
274 * pvr_context_create() - Create a context.
275 * @pvr_file: File to attach the created context to.
276 * @args: Context creation arguments.
277 *
278 * Return:
279 * * 0 on success, or
280 * * A negative error code on failure.
281 */
282int pvr_context_create(struct pvr_file *pvr_file, struct drm_pvr_ioctl_create_context_args *args)
283{
284 struct pvr_device *pvr_dev = pvr_file->pvr_dev;
285 struct pvr_context *ctx;
286 int ctx_size;
287 int err;
288
289 /* Context creation flags are currently unused and must be zero. */
290 if (args->flags)
291 return -EINVAL;
292
293 ctx_size = get_fw_obj_size(args->type);
294 if (ctx_size < 0)
295 return ctx_size;
296
297 ctx = kzalloc_obj(*ctx);
298 if (!ctx)
299 return -ENOMEM;
300
301 ctx->data_size = ctx_size;
302 ctx->type = args->type;
303 ctx->flags = args->flags;
304 ctx->pvr_dev = pvr_dev;
305 kref_init(&ctx->ref_count);
306
307 err = remap_priority(pvr_file, args->priority, &ctx->priority);
308 if (err)
309 goto err_free_ctx;
310
311 ctx->vm_ctx = pvr_vm_context_lookup(pvr_file, args->vm_context_handle);
312 if (!ctx->vm_ctx) {
313 err = -EINVAL;
314 goto err_free_ctx;
315 }
316
317 ctx->data = kzalloc(ctx_size, GFP_KERNEL);
318 if (!ctx->data) {
319 err = -ENOMEM;
320 goto err_put_vm;
321 }
322
323 err = xa_alloc(&pvr_dev->ctx_ids, &ctx->ctx_id, ctx, xa_limit_32b, GFP_KERNEL);
324 if (err)
325 goto err_free_ctx_data;
326
327 err = pvr_context_create_queues(ctx, args, ctx->data);
328 if (err)
329 goto err_free_ctx_id;
330
331 err = init_fw_objs(ctx, args, ctx->data);
332 if (err)
333 goto err_destroy_queues;
334
335 err = pvr_fw_object_create(pvr_dev, ctx_size, PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
336 ctx_fw_data_init, ctx, &ctx->fw_obj);
337 if (err)
338 goto err_destroy_queues;
339
340 err = xa_alloc(&pvr_file->ctx_handles, &args->handle, ctx, xa_limit_32b, GFP_KERNEL);
341 if (err)
342 goto err_destroy_fw_obj;
343
344 spin_lock(&pvr_dev->ctx_list_lock);
345 list_add_tail(&ctx->file_link, &pvr_file->contexts);
346 spin_unlock(&pvr_dev->ctx_list_lock);
347
348 return 0;
349
350err_destroy_fw_obj:
351 pvr_fw_object_destroy(ctx->fw_obj);
352
353err_destroy_queues:
354 pvr_context_destroy_queues(ctx, true);
355
356err_free_ctx_id:
357 /*
358 * Ctx_id is not exposed to userspace and not visible yet within
359 * the kernel/FW, plus a matching context handle (exposed to userspace)
360 * hasn't been allocated yet, so it is safe to remove ctx_id
361 * from the ctx_ids xarray.
362 */
363 xa_erase(&pvr_dev->ctx_ids, ctx->ctx_id);
364
365err_free_ctx_data:
366 kfree(ctx->data);
367
368err_put_vm:
369 pvr_vm_context_put(ctx->vm_ctx);
370
371err_free_ctx:
372 kfree(ctx);
373 return err;
374}
375
376static void
377pvr_context_release(struct kref *ref_count)
378{
379 struct pvr_context *ctx =
380 container_of(ref_count, struct pvr_context, ref_count);
381 struct pvr_device *pvr_dev = ctx->pvr_dev;
382
383 WARN_ON(in_interrupt());
384 spin_lock(&pvr_dev->ctx_list_lock);
385 list_del(&ctx->file_link);
386 spin_unlock(&pvr_dev->ctx_list_lock);
387
388 xa_erase(&pvr_dev->ctx_ids, ctx->ctx_id);
389 pvr_context_destroy_queues(ctx, false);
390 pvr_fw_object_destroy(ctx->fw_obj);
391 kfree(ctx->data);
392 pvr_vm_context_put(ctx->vm_ctx);
393 kfree(ctx);
394}
395
396/**
397 * pvr_context_put() - Release reference on context
398 * @ctx: Target context.
399 */
400void
401pvr_context_put(struct pvr_context *ctx)
402{
403 if (ctx)
404 kref_put(&ctx->ref_count, pvr_context_release);
405}
406
407/**
408 * pvr_context_destroy() - Destroy context
409 * @pvr_file: Pointer to pvr_file structure.
410 * @handle: Userspace context handle.
411 *
412 * Removes context from context list and drops initial reference. Context will
413 * then be destroyed once all outstanding references are dropped.
414 *
415 * Return:
416 * * 0 on success, or
417 * * -%EINVAL if context not in context list.
418 */
419int
420pvr_context_destroy(struct pvr_file *pvr_file, u32 handle)
421{
422 struct pvr_context *ctx = xa_erase(&pvr_file->ctx_handles, handle);
423
424 if (!ctx)
425 return -EINVAL;
426
427 /* Make sure nothing can be queued to the queues after that point. */
428 pvr_context_kill_queues(ctx);
429
430 /* Release the reference held by the handle set. */
431 pvr_context_put(ctx);
432
433 return 0;
434}
435
436/**
437 * pvr_destroy_contexts_for_file: Destroy any contexts associated with the given file
438 * @pvr_file: Pointer to pvr_file structure.
439 *
440 * Removes all contexts associated with @pvr_file from the device context list and drops initial
441 * references. Contexts will then be destroyed once all outstanding references are dropped.
442 */
443void pvr_destroy_contexts_for_file(struct pvr_file *pvr_file)
444{
445 struct pvr_device *pvr_dev = pvr_file->pvr_dev;
446 struct pvr_context *ctx;
447 unsigned long handle;
448
449 xa_for_each(&pvr_file->ctx_handles, handle, ctx)
450 pvr_context_destroy(pvr_file, handle);
451
452 spin_lock(&pvr_dev->ctx_list_lock);
453 ctx = list_first_entry(&pvr_file->contexts, struct pvr_context, file_link);
454
455 while (!list_entry_is_head(ctx, &pvr_file->contexts, file_link)) {
456 list_del_init(&ctx->file_link);
457
458 if (pvr_context_get_if_referenced(ctx)) {
459 spin_unlock(&pvr_dev->ctx_list_lock);
460
461 pvr_vm_unmap_all(ctx->vm_ctx);
462
463 pvr_context_put(ctx);
464 spin_lock(&pvr_dev->ctx_list_lock);
465 }
466 ctx = list_first_entry(&pvr_file->contexts, struct pvr_context, file_link);
467 }
468 spin_unlock(&pvr_dev->ctx_list_lock);
469}
470
471/**
472 * pvr_context_device_init() - Device level initialization for queue related resources.
473 * @pvr_dev: The device to initialize.
474 */
475void pvr_context_device_init(struct pvr_device *pvr_dev)
476{
477 xa_init_flags(&pvr_dev->ctx_ids, XA_FLAGS_ALLOC1);
478 spin_lock_init(&pvr_dev->ctx_list_lock);
479}
480
481/**
482 * pvr_context_device_fini() - Device level cleanup for queue related resources.
483 * @pvr_dev: The device to cleanup.
484 */
485void pvr_context_device_fini(struct pvr_device *pvr_dev)
486{
487 WARN_ON(!xa_empty(&pvr_dev->ctx_ids));
488 xa_destroy(&pvr_dev->ctx_ids);
489}