projects/linux·blob master · refs

320 lines · 8.4 KiB

1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2024 Intel Corporation
4 */
5
6#include <linux/shrinker.h>
7
8#include <drm/drm_managed.h>
9#include <drm/ttm/ttm_backup.h>
10#include <drm/ttm/ttm_bo.h>
11#include <drm/ttm/ttm_tt.h>
12
13#include "xe_bo.h"
14#include "xe_pm.h"
15#include "xe_shrinker.h"
16
17/**
18 * struct xe_shrinker - per-device shrinker
19 * @xe: Back pointer to the device.
20 * @lock: Lock protecting accounting.
21 * @shrinkable_pages: Number of pages that are currently shrinkable.
22 * @purgeable_pages: Number of pages that are currently purgeable.
23 * @shrink: Pointer to the mm shrinker.
24 * @pm_worker: Worker to wake up the device if required.
25 */
26struct xe_shrinker {
27 struct xe_device *xe;
28 rwlock_t lock;
29 long shrinkable_pages;
30 long purgeable_pages;
31 struct shrinker *shrink;
32 struct work_struct pm_worker;
33};
34
35static struct xe_shrinker *to_xe_shrinker(struct shrinker *shrink)
36{
37 return shrink->private_data;
38}
39
40/**
41 * xe_shrinker_mod_pages() - Modify shrinker page accounting
42 * @shrinker: Pointer to the struct xe_shrinker.
43 * @shrinkable: Shrinkable pages delta. May be negative.
44 * @purgeable: Purgeable page delta. May be negative.
45 *
46 * Modifies the shrinkable and purgeable pages accounting.
47 */
48void
49xe_shrinker_mod_pages(struct xe_shrinker *shrinker, long shrinkable, long purgeable)
50{
51 write_lock(&shrinker->lock);
52 shrinker->shrinkable_pages += shrinkable;
53 shrinker->purgeable_pages += purgeable;
54 write_unlock(&shrinker->lock);
55}
56
57static bool __xe_shrinker_runtime_pm_get(struct xe_shrinker *shrinker)
58{
59 struct xe_device *xe = shrinker->xe;
60
61 if (xe_pm_runtime_get_if_active(xe))
62 return true;
63
64 if (xe_rpm_reclaim_safe(xe) && !ttm_bo_shrink_avoid_wait()) {
65 xe_pm_runtime_get(xe);
66 return true;
67 }
68
69 queue_work(xe->unordered_wq, &shrinker->pm_worker);
70
71 return false;
72}
73
74static void xe_shrinker_runtime_pm_put(struct xe_shrinker *shrinker, bool runtime_pm)
75{
76 if (runtime_pm)
77 xe_pm_runtime_put(shrinker->xe);
78}
79
80static int __xe_shrinker_walk(struct xe_shrinker *shrinker,
81 struct ttm_operation_ctx *ctx,
82 const struct xe_bo_shrink_flags flags,
83 unsigned long to_scan, unsigned long *scanned,
84 unsigned long *freed)
85{
86 struct xe_device *xe = shrinker->xe;
87 unsigned int mem_type;
88 bool rpm = false;
89 int ret = 0;
90 s64 lret;
91
92 for (mem_type = XE_PL_SYSTEM; mem_type <= XE_PL_TT; ++mem_type) {
93 struct ttm_resource_manager *man = ttm_manager_type(&xe->ttm, mem_type);
94 struct ttm_bo_lru_cursor curs;
95 struct ttm_buffer_object *ttm_bo;
96 struct ttm_lru_walk_arg arg = {
97 .ctx = ctx,
98 .trylock_only = true,
99 };
100
101 if (!man || !man->use_tt)
102 continue;
103
104 if (mem_type != XE_PL_SYSTEM && !rpm &&
105 xe_device_is_l2_flush_optimized(xe)) {
106 if (!__xe_shrinker_runtime_pm_get(shrinker))
107 break;
108 rpm = true;
109 }
110
111 ttm_bo_lru_for_each_reserved_guarded(&curs, man, &arg, ttm_bo) {
112 if (!ttm_bo_shrink_suitable(ttm_bo, ctx))
113 continue;
114
115 lret = xe_bo_shrink(ctx, ttm_bo, flags, scanned);
116 if (lret < 0) {
117 ret = lret;
118 goto out;
119 }
120
121 *freed += lret;
122 if (*scanned >= to_scan)
123 goto out;
124 }
125 /* Trylocks should never error, just fail. */
126 xe_assert(xe, !IS_ERR(ttm_bo));
127 }
128
129out:
130 xe_shrinker_runtime_pm_put(shrinker, rpm);
131
132 return ret;
133}
134
135/*
136 * Try shrinking idle objects without writeback first, then if not sufficient,
137 * try also non-idle objects and finally if that's not sufficient either,
138 * add writeback. This avoids stalls and explicit writebacks with light or
139 * moderate memory pressure.
140 */
141static int xe_shrinker_walk(struct xe_shrinker *shrinker,
142 struct ttm_operation_ctx *ctx,
143 const struct xe_bo_shrink_flags flags,
144 unsigned long to_scan, unsigned long *scanned,
145 unsigned long *freed)
146{
147 bool no_wait_gpu = true;
148 struct xe_bo_shrink_flags save_flags = flags;
149 int ret;
150
151 swap(no_wait_gpu, ctx->no_wait_gpu);
152 save_flags.writeback = false;
153 ret = __xe_shrinker_walk(shrinker, ctx, save_flags, to_scan, scanned,
154 freed);
155 swap(no_wait_gpu, ctx->no_wait_gpu);
156 if (ret || *scanned >= to_scan)
157 return ret;
158
159 if (!ctx->no_wait_gpu) {
160 ret = __xe_shrinker_walk(shrinker, ctx, save_flags, to_scan, scanned,
161 freed);
162 if (ret || *scanned >= to_scan)
163 return ret;
164 }
165
166 if (flags.writeback)
167 ret = __xe_shrinker_walk(shrinker, ctx, flags, to_scan, scanned,
168 freed);
169
170 return ret;
171}
172
173static unsigned long
174xe_shrinker_count(struct shrinker *shrink, struct shrink_control *sc)
175{
176 struct xe_shrinker *shrinker = to_xe_shrinker(shrink);
177 unsigned long num_pages;
178 bool can_backup = !!(sc->gfp_mask & __GFP_FS);
179
180 num_pages = ttm_backup_bytes_avail() >> PAGE_SHIFT;
181 read_lock(&shrinker->lock);
182
183 if (can_backup)
184 num_pages = min_t(unsigned long, num_pages, shrinker->shrinkable_pages);
185 else
186 num_pages = 0;
187
188 num_pages += shrinker->purgeable_pages;
189 read_unlock(&shrinker->lock);
190
191 return num_pages ? num_pages : SHRINK_EMPTY;
192}
193
194/*
195 * Check if we need runtime pm, and if so try to grab a reference if
196 * already active. If grabbing a reference fails, queue a worker that
197 * does it for us outside of reclaim, but don't wait for it to complete.
198 * If bo shrinking needs an rpm reference and we don't have it (yet),
199 * that bo will be skipped anyway.
200 */
201static bool xe_shrinker_runtime_pm_get(struct xe_shrinker *shrinker, bool force,
202 unsigned long nr_to_scan, bool can_backup)
203{
204 struct xe_device *xe = shrinker->xe;
205
206 if (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe) ||
207 !ttm_backup_bytes_avail())
208 return false;
209
210 if (!force) {
211 read_lock(&shrinker->lock);
212 force = (nr_to_scan > shrinker->purgeable_pages && can_backup);
213 read_unlock(&shrinker->lock);
214 if (!force)
215 return false;
216 }
217
218 return __xe_shrinker_runtime_pm_get(shrinker);
219}
220
221static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc)
222{
223 struct xe_shrinker *shrinker = to_xe_shrinker(shrink);
224 struct ttm_operation_ctx ctx = {
225 .interruptible = false,
226 .no_wait_gpu = ttm_bo_shrink_avoid_wait(),
227 };
228 unsigned long nr_to_scan, nr_scanned = 0, freed = 0;
229 struct xe_bo_shrink_flags shrink_flags = {
230 .purge = true,
231 /* Don't request writeback without __GFP_IO. */
232 .writeback = !ctx.no_wait_gpu && (sc->gfp_mask & __GFP_IO),
233 };
234 bool runtime_pm;
235 bool purgeable;
236 bool can_backup = !!(sc->gfp_mask & __GFP_FS);
237
238 nr_to_scan = sc->nr_to_scan;
239
240 read_lock(&shrinker->lock);
241 purgeable = !!shrinker->purgeable_pages;
242 read_unlock(&shrinker->lock);
243
244 /* Might need runtime PM. Try to wake early if it looks like it. */
245 runtime_pm = xe_shrinker_runtime_pm_get(shrinker, false, nr_to_scan, can_backup);
246
247 if (purgeable && nr_scanned < nr_to_scan)
248 xe_shrinker_walk(shrinker, &ctx, shrink_flags,
249 nr_to_scan, &nr_scanned, &freed);
250
251 sc->nr_scanned = nr_scanned;
252 if (nr_scanned >= nr_to_scan || !can_backup)
253 goto out;
254
255 /* If we didn't wake before, try to do it now if needed. */
256 if (!runtime_pm)
257 runtime_pm = xe_shrinker_runtime_pm_get(shrinker, true, 0, can_backup);
258
259 shrink_flags.purge = false;
260
261 xe_shrinker_walk(shrinker, &ctx, shrink_flags,
262 nr_to_scan, &nr_scanned, &freed);
263
264 sc->nr_scanned = nr_scanned;
265out:
266 xe_shrinker_runtime_pm_put(shrinker, runtime_pm);
267 return nr_scanned ? freed : SHRINK_STOP;
268}
269
270/* Wake up the device for shrinking. */
271static void xe_shrinker_pm(struct work_struct *work)
272{
273 struct xe_shrinker *shrinker =
274 container_of(work, typeof(*shrinker), pm_worker);
275
276 xe_pm_runtime_get(shrinker->xe);
277 xe_pm_runtime_put(shrinker->xe);
278}
279
280static void xe_shrinker_fini(struct drm_device *drm, void *arg)
281{
282 struct xe_shrinker *shrinker = arg;
283
284 xe_assert(shrinker->xe, !shrinker->shrinkable_pages);
285 xe_assert(shrinker->xe, !shrinker->purgeable_pages);
286 shrinker_free(shrinker->shrink);
287 flush_work(&shrinker->pm_worker);
288 kfree(shrinker);
289}
290
291/**
292 * xe_shrinker_create() - Create an xe per-device shrinker
293 * @xe: Pointer to the xe device.
294 *
295 * Return: %0 on success. Negative error code on failure.
296 */
297int xe_shrinker_create(struct xe_device *xe)
298{
299 struct xe_shrinker *shrinker = kzalloc_obj(*shrinker);
300
301 if (!shrinker)
302 return -ENOMEM;
303
304 shrinker->shrink = shrinker_alloc(0, "drm-xe_gem:%s", xe->drm.unique);
305 if (!shrinker->shrink) {
306 kfree(shrinker);
307 return -ENOMEM;
308 }
309
310 INIT_WORK(&shrinker->pm_worker, xe_shrinker_pm);
311 shrinker->xe = xe;
312 rwlock_init(&shrinker->lock);
313 shrinker->shrink->count_objects = xe_shrinker_count;
314 shrinker->shrink->scan_objects = xe_shrinker_scan;
315 shrinker->shrink->private_data = shrinker;
316 shrinker_register(shrinker->shrink);
317 xe->mem.shrinker = shrinker;
318
319 return drmm_add_action_or_reset(&xe->drm, xe_shrinker_fini, shrinker);
320}