projects/linux·blob master · refs

540 lines · 12.4 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Tegra host1x Syncpoints
4 *
5 * Copyright (c) 2010-2015, NVIDIA Corporation.
6 */
7
8#include <linux/module.h>
9#include <linux/device.h>
10#include <linux/dma-fence.h>
11#include <linux/overflow.h>
12#include <linux/slab.h>
13
14#include <trace/events/host1x.h>
15
16#include "syncpt.h"
17#include "dev.h"
18#include "intr.h"
19#include "debug.h"
20
21#define SYNCPT_CHECK_PERIOD (2 * HZ)
22#define MAX_STUCK_CHECK_COUNT 15
23
24static struct host1x_syncpt_base *
25host1x_syncpt_base_request(struct host1x *host)
26{
27 struct host1x_syncpt_base *bases = host->bases;
28 unsigned int i;
29
30 for (i = 0; i < host->info->nb_bases; i++)
31 if (!bases[i].requested)
32 break;
33
34 if (i >= host->info->nb_bases)
35 return NULL;
36
37 bases[i].requested = true;
38 return &bases[i];
39}
40
41static void host1x_syncpt_base_free(struct host1x_syncpt_base *base)
42{
43 if (base)
44 base->requested = false;
45}
46
47/**
48 * host1x_syncpt_alloc() - allocate a syncpoint
49 * @host: host1x device data
50 * @flags: bitfield of HOST1X_SYNCPT_* flags
51 * @name: name for the syncpoint for use in debug prints
52 *
53 * Allocates a hardware syncpoint for the caller's use. The caller then has
54 * the sole authority to mutate the syncpoint's value until it is freed again.
55 *
56 * If no free syncpoints are available, or a NULL name was specified, returns
57 * NULL.
58 */
59struct host1x_syncpt *host1x_syncpt_alloc(struct host1x *host,
60 unsigned long flags,
61 const char *name)
62{
63 struct host1x_syncpt *sp = host->syncpt;
64 char *full_name;
65 unsigned int i;
66
67 if (!name)
68 return NULL;
69
70 mutex_lock(&host->syncpt_mutex);
71
72 for (i = 0; i < host->info->nb_pts && kref_read(&sp->ref); i++, sp++)
73 ;
74
75 if (i >= host->info->nb_pts)
76 goto unlock;
77
78 if (flags & HOST1X_SYNCPT_HAS_BASE) {
79 sp->base = host1x_syncpt_base_request(host);
80 if (!sp->base)
81 goto unlock;
82 }
83
84 full_name = kasprintf(GFP_KERNEL, "%u-%s", sp->id, name);
85 if (!full_name)
86 goto free_base;
87
88 sp->name = full_name;
89
90 if (flags & HOST1X_SYNCPT_CLIENT_MANAGED)
91 sp->client_managed = true;
92 else
93 sp->client_managed = false;
94
95 kref_init(&sp->ref);
96
97 mutex_unlock(&host->syncpt_mutex);
98 return sp;
99
100free_base:
101 host1x_syncpt_base_free(sp->base);
102 sp->base = NULL;
103unlock:
104 mutex_unlock(&host->syncpt_mutex);
105 return NULL;
106}
107EXPORT_SYMBOL(host1x_syncpt_alloc);
108
109/**
110 * host1x_syncpt_id() - retrieve syncpoint ID
111 * @sp: host1x syncpoint
112 *
113 * Given a pointer to a struct host1x_syncpt, retrieves its ID. This ID is
114 * often used as a value to program into registers that control how hardware
115 * blocks interact with syncpoints.
116 */
117u32 host1x_syncpt_id(struct host1x_syncpt *sp)
118{
119 return sp->id;
120}
121EXPORT_SYMBOL(host1x_syncpt_id);
122
123/**
124 * host1x_syncpt_incr_max() - update the value sent to hardware
125 * @sp: host1x syncpoint
126 * @incrs: number of increments
127 */
128u32 host1x_syncpt_incr_max(struct host1x_syncpt *sp, u32 incrs)
129{
130 /*
131 * Syncpoint values are intended to be modulo 2^32, so overflow
132 * here is intended.
133 */
134 return (u32)atomic_add_return(incrs, &sp->max_val);
135}
136EXPORT_SYMBOL(host1x_syncpt_incr_max);
137
138 /*
139 * Write cached syncpoint and waitbase values to hardware.
140 */
141void host1x_syncpt_restore(struct host1x *host)
142{
143 struct host1x_syncpt *sp_base = host->syncpt;
144 unsigned int i;
145
146 for (i = 0; i < host1x_syncpt_nb_pts(host); i++) {
147 /*
148 * Unassign syncpt from channels for purposes of Tegra186
149 * syncpoint protection. This prevents any channel from
150 * accessing it until it is reassigned.
151 */
152 host1x_hw_syncpt_assign_to_channel(host, sp_base + i, NULL);
153 host1x_hw_syncpt_restore(host, sp_base + i);
154 }
155
156 for (i = 0; i < host1x_syncpt_nb_bases(host); i++)
157 host1x_hw_syncpt_restore_wait_base(host, sp_base + i);
158
159 host1x_hw_syncpt_enable_protection(host);
160
161 wmb();
162}
163
164/*
165 * Update the cached syncpoint and waitbase values by reading them
166 * from the registers.
167 */
168void host1x_syncpt_save(struct host1x *host)
169{
170 struct host1x_syncpt *sp_base = host->syncpt;
171 unsigned int i;
172
173 for (i = 0; i < host1x_syncpt_nb_pts(host); i++) {
174 if (host1x_syncpt_client_managed(sp_base + i))
175 host1x_hw_syncpt_load(host, sp_base + i);
176 else
177 WARN_ON(!host1x_syncpt_idle(sp_base + i));
178 }
179
180 for (i = 0; i < host1x_syncpt_nb_bases(host); i++)
181 host1x_hw_syncpt_load_wait_base(host, sp_base + i);
182}
183
184/*
185 * Updates the cached syncpoint value by reading a new value from the hardware
186 * register
187 */
188u32 host1x_syncpt_load(struct host1x_syncpt *sp)
189{
190 u32 val;
191
192 val = host1x_hw_syncpt_load(sp->host, sp);
193 trace_host1x_syncpt_load_min(sp->id, val);
194
195 return val;
196}
197
198/*
199 * Get the current syncpoint base
200 */
201u32 host1x_syncpt_load_wait_base(struct host1x_syncpt *sp)
202{
203 host1x_hw_syncpt_load_wait_base(sp->host, sp);
204
205 return sp->base_val;
206}
207
208/**
209 * host1x_syncpt_incr() - increment syncpoint value from CPU, updating cache
210 * @sp: host1x syncpoint
211 */
212int host1x_syncpt_incr(struct host1x_syncpt *sp)
213{
214 return host1x_hw_syncpt_cpu_incr(sp->host, sp);
215}
216EXPORT_SYMBOL(host1x_syncpt_incr);
217
218/**
219 * host1x_syncpt_wait() - wait for a syncpoint to reach a given value
220 * @sp: host1x syncpoint
221 * @thresh: threshold
222 * @timeout: maximum time to wait for the syncpoint to reach the given value
223 * @value: return location for the syncpoint value
224 */
225int host1x_syncpt_wait(struct host1x_syncpt *sp, u32 thresh, long timeout,
226 u32 *value)
227{
228 struct dma_fence *fence;
229 long wait_err;
230 u32 curr;
231
232 curr = host1x_syncpt_load(sp);
233
234 if (value)
235 *value = curr;
236
237 if (host1x_syncpt_is_expired(sp, thresh))
238 return 0;
239
240 if (timeout < 0)
241 timeout = LONG_MAX;
242 else if (timeout == 0)
243 return -EAGAIN;
244
245 fence = host1x_fence_create(sp, thresh, false);
246 if (IS_ERR(fence))
247 return PTR_ERR(fence);
248
249 wait_err = dma_fence_wait_timeout(fence, true, timeout);
250 if (wait_err == 0)
251 host1x_fence_cancel(fence);
252 dma_fence_put(fence);
253
254 /*
255 * Don't rely on dma_fence_wait_timeout return value,
256 * since it returns zero both on timeout and if the
257 * wait completed with 0 jiffies left.
258 */
259 if (wait_err == 0 && !host1x_syncpt_is_expired(sp, thresh)) {
260 if (value)
261 *value = host1x_syncpt_load(sp);
262
263 return -EAGAIN;
264 } else if (wait_err < 0) {
265 return wait_err;
266 } else {
267 /* Success, read the value cached by ISR */
268 if (value)
269 *value = host1x_syncpt_read_min(sp);
270
271 return 0;
272 }
273}
274EXPORT_SYMBOL(host1x_syncpt_wait);
275
276/*
277 * Returns true if syncpoint is expired, false if we may need to wait
278 */
279bool host1x_syncpt_is_expired(struct host1x_syncpt *sp, u32 thresh)
280{
281 u32 current_val;
282
283 smp_rmb();
284
285 current_val = (u32)atomic_read(&sp->min_val);
286
287 return (wrapping_sub(u32, current_val, thresh) & 0x80000000U) == 0U;
288}
289
290int host1x_syncpt_init(struct host1x *host)
291{
292 struct host1x_syncpt_base *bases;
293 struct host1x_syncpt *syncpt;
294 unsigned int i;
295
296 syncpt = devm_kcalloc(host->dev, host->info->nb_pts, sizeof(*syncpt),
297 GFP_KERNEL);
298 if (!syncpt)
299 return -ENOMEM;
300
301 bases = devm_kcalloc(host->dev, host->info->nb_bases, sizeof(*bases),
302 GFP_KERNEL);
303 if (!bases)
304 return -ENOMEM;
305
306 for (i = 0; i < host->info->nb_pts; i++) {
307 syncpt[i].id = i;
308 syncpt[i].host = host;
309 }
310
311 for (i = 0; i < host->info->nb_bases; i++)
312 bases[i].id = i;
313
314 mutex_init(&host->syncpt_mutex);
315 host->syncpt = syncpt;
316 host->bases = bases;
317
318 /* Allocate sync point to use for clearing waits for expired fences */
319 host->nop_sp = host1x_syncpt_alloc(host, 0, "reserved-nop");
320 if (!host->nop_sp)
321 return -ENOMEM;
322
323 if (host->info->reserve_vblank_syncpts) {
324 kref_init(&host->syncpt[26].ref);
325 kref_init(&host->syncpt[27].ref);
326 }
327
328 return 0;
329}
330
331/**
332 * host1x_syncpt_request() - request a syncpoint
333 * @client: client requesting the syncpoint
334 * @flags: flags
335 *
336 * host1x client drivers can use this function to allocate a syncpoint for
337 * subsequent use. A syncpoint returned by this function will be reserved for
338 * use by the client exclusively. When no longer using a syncpoint, a host1x
339 * client driver needs to release it using host1x_syncpt_put().
340 */
341struct host1x_syncpt *host1x_syncpt_request(struct host1x_client *client,
342 unsigned long flags)
343{
344 struct host1x *host = dev_get_drvdata(client->host->parent);
345
346 return host1x_syncpt_alloc(host, flags, dev_name(client->dev));
347}
348EXPORT_SYMBOL(host1x_syncpt_request);
349
350static void syncpt_release(struct kref *ref)
351{
352 struct host1x_syncpt *sp = container_of(ref, struct host1x_syncpt, ref);
353
354 atomic_set(&sp->max_val, host1x_syncpt_read(sp));
355
356 sp->locked = false;
357
358 host1x_syncpt_base_free(sp->base);
359 kfree(sp->name);
360 sp->base = NULL;
361 sp->name = NULL;
362 sp->client_managed = false;
363
364 mutex_unlock(&sp->host->syncpt_mutex);
365}
366
367/**
368 * host1x_syncpt_put() - free a requested syncpoint
369 * @sp: host1x syncpoint
370 *
371 * Release a syncpoint previously allocated using host1x_syncpt_request(). A
372 * host1x client driver should call this when the syncpoint is no longer in
373 * use.
374 */
375void host1x_syncpt_put(struct host1x_syncpt *sp)
376{
377 if (!sp)
378 return;
379
380 kref_put_mutex(&sp->ref, syncpt_release, &sp->host->syncpt_mutex);
381}
382EXPORT_SYMBOL(host1x_syncpt_put);
383
384void host1x_syncpt_deinit(struct host1x *host)
385{
386 struct host1x_syncpt *sp = host->syncpt;
387 unsigned int i;
388
389 for (i = 0; i < host->info->nb_pts; i++, sp++)
390 kfree(sp->name);
391}
392
393/**
394 * host1x_syncpt_read_max() - read maximum syncpoint value
395 * @sp: host1x syncpoint
396 *
397 * The maximum syncpoint value indicates how many operations there are in
398 * queue, either in channel or in a software thread.
399 */
400u32 host1x_syncpt_read_max(struct host1x_syncpt *sp)
401{
402 smp_rmb();
403
404 return (u32)atomic_read(&sp->max_val);
405}
406EXPORT_SYMBOL(host1x_syncpt_read_max);
407
408/**
409 * host1x_syncpt_read_min() - read minimum syncpoint value
410 * @sp: host1x syncpoint
411 *
412 * The minimum syncpoint value is a shadow of the current sync point value in
413 * hardware.
414 */
415u32 host1x_syncpt_read_min(struct host1x_syncpt *sp)
416{
417 smp_rmb();
418
419 return (u32)atomic_read(&sp->min_val);
420}
421EXPORT_SYMBOL(host1x_syncpt_read_min);
422
423/**
424 * host1x_syncpt_read() - read the current syncpoint value
425 * @sp: host1x syncpoint
426 */
427u32 host1x_syncpt_read(struct host1x_syncpt *sp)
428{
429 return host1x_syncpt_load(sp);
430}
431EXPORT_SYMBOL(host1x_syncpt_read);
432
433unsigned int host1x_syncpt_nb_pts(struct host1x *host)
434{
435 return host->info->nb_pts;
436}
437
438unsigned int host1x_syncpt_nb_bases(struct host1x *host)
439{
440 return host->info->nb_bases;
441}
442
443unsigned int host1x_syncpt_nb_mlocks(struct host1x *host)
444{
445 return host->info->nb_mlocks;
446}
447
448/**
449 * host1x_syncpt_get_by_id() - obtain a syncpoint by ID
450 * @host: host1x controller
451 * @id: syncpoint ID
452 */
453struct host1x_syncpt *host1x_syncpt_get_by_id(struct host1x *host,
454 unsigned int id)
455{
456 if (id >= host->info->nb_pts)
457 return NULL;
458
459 if (kref_get_unless_zero(&host->syncpt[id].ref))
460 return &host->syncpt[id];
461 else
462 return NULL;
463}
464EXPORT_SYMBOL(host1x_syncpt_get_by_id);
465
466/**
467 * host1x_syncpt_get_by_id_noref() - obtain a syncpoint by ID but don't
468 * increase the refcount.
469 * @host: host1x controller
470 * @id: syncpoint ID
471 */
472struct host1x_syncpt *host1x_syncpt_get_by_id_noref(struct host1x *host,
473 unsigned int id)
474{
475 if (id >= host->info->nb_pts)
476 return NULL;
477
478 return &host->syncpt[id];
479}
480EXPORT_SYMBOL(host1x_syncpt_get_by_id_noref);
481
482/**
483 * host1x_syncpt_get() - increment syncpoint refcount
484 * @sp: syncpoint
485 */
486struct host1x_syncpt *host1x_syncpt_get(struct host1x_syncpt *sp)
487{
488 kref_get(&sp->ref);
489
490 return sp;
491}
492EXPORT_SYMBOL(host1x_syncpt_get);
493
494/**
495 * host1x_syncpt_get_base() - obtain the wait base associated with a syncpoint
496 * @sp: host1x syncpoint
497 */
498struct host1x_syncpt_base *host1x_syncpt_get_base(struct host1x_syncpt *sp)
499{
500 return sp ? sp->base : NULL;
501}
502EXPORT_SYMBOL(host1x_syncpt_get_base);
503
504/**
505 * host1x_syncpt_base_id() - retrieve the ID of a syncpoint wait base
506 * @base: host1x syncpoint wait base
507 */
508u32 host1x_syncpt_base_id(struct host1x_syncpt_base *base)
509{
510 return base->id;
511}
512EXPORT_SYMBOL(host1x_syncpt_base_id);
513
514static void do_nothing(struct kref *ref)
515{
516}
517
518/**
519 * host1x_syncpt_release_vblank_reservation() - Make VBLANK syncpoint
520 * available for allocation
521 *
522 * @client: host1x bus client
523 * @syncpt_id: syncpoint ID to make available
524 *
525 * Makes VBLANK<i> syncpoint available for allocatation if it was
526 * reserved at initialization time. This should be called by the display
527 * driver after it has ensured that any VBLANK increment programming configured
528 * by the boot chain has been disabled.
529 */
530void host1x_syncpt_release_vblank_reservation(struct host1x_client *client,
531 u32 syncpt_id)
532{
533 struct host1x *host = dev_get_drvdata(client->host->parent);
534
535 if (!host->info->reserve_vblank_syncpts)
536 return;
537
538 kref_put(&host->syncpt[syncpt_id].ref, do_nothing);
539}
540EXPORT_SYMBOL(host1x_syncpt_release_vblank_reservation);