linux - drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm

2627 lines · 79.4 KiB

1	// SPDX-License-Identifier: MIT
2	/*
3	* Copyright 2015 Advanced Micro Devices, Inc.
4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the "Software"),
7	* to deal in the Software without restriction, including without limitation
8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
9	* and/or sell copies of the Software, and to permit persons to whom the
10	* Software is furnished to do so, subject to the following conditions:
11	*
12	* The above copyright notice and this permission notice shall be included in
13	* all copies or substantial portions of the Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21	* OTHER DEALINGS IN THE SOFTWARE.
22	*
23	* Authors: AMD
24	*
25	*/
26
27	#include "dm_services_types.h"
28	#include "dc.h"
29	#include "dc/dc_dmub_srv.h"
30	#include "dc/dc_stat.h"
31
32	#include "amdgpu.h"
33	#include "amdgpu_display.h"
34	#include "amdgpu_dm.h"
35	#include "amdgpu_dm_irq.h"
36	#include "dm_helpers.h"
37	#include "amdgpu_dm_crtc.h"
38	#include "amdgpu_dm_hdcp.h"
39	#include "amdgpu_dm_mst_types.h"
40	#include "amdgpu_dm_dmub.h"
41	#include "amdgpu_dm_trace.h"
42	#include "link/protocols/link_dpcd.h"
43	#include "link_service_types.h"
44	#include "ivsrcid/ivsrcid_vislands30.h"
45	#include "ivsrcid/dcn/irqsrcs_dcn_1_0.h"
46	#include "modules/inc/mod_freesync.h"
47	#include <drm/drm_vblank.h>
48
49	/**
50	* DOC: overview
51	*
52	* DM provides another layer of IRQ management on top of what the base driver
53	* already provides. This is something that could be cleaned up, and is a
54	* future TODO item.
55	*
56	* The base driver provides IRQ source registration with DRM, handler
57	* registration into the base driver's IRQ table, and a handler callback
58	* amdgpu_irq_handler(), with which DRM calls on interrupts. This generic
59	* handler looks up the IRQ table, and calls the respective
60	* &amdgpu_irq_src_funcs.process hookups.
61	*
62	* What DM provides on top are two IRQ tables specifically for top-half and
63	* bottom-half IRQ handling, with the bottom-half implementing workqueues:
64	*
65	* - &amdgpu_display_manager.irq_handler_list_high_tab
66	* - &amdgpu_display_manager.irq_handler_list_low_tab
67	*
68	* They override the base driver's IRQ table, and the effect can be seen
69	* in the hooks that DM provides for &amdgpu_irq_src_funcs.process. They
70	* are all set to the DM generic handler amdgpu_dm_irq_handler(), which looks up
71	* DM's IRQ tables. However, in order for base driver to recognize this hook, DM
72	* still needs to register the IRQ with the base driver. See
73	* amdgpu_dm_dce110_register_irq_handlers() and
74	* amdgpu_dm_dcn10_register_irq_handlers().
75	*
76	* To expose DC's hardware interrupt toggle to the base driver, DM implements
77	* &amdgpu_irq_src_funcs.set hooks. Base driver calls it through
78	* amdgpu_irq_update() to enable or disable the interrupt.
79	*/
80
81	/******************************************************************************
82	* Private declarations.
83	*****************************************************************************/
84
85	/**
86	* struct amdgpu_dm_irq_handler_data - Data for DM interrupt handlers.
87	*
88	* @list: Linked list entry referencing the next/previous handler
89	* @handler: Handler function
90	* @handler_arg: Argument passed to the handler when triggered
91	* @dm: DM which this handler belongs to
92	* @irq_source: DC interrupt source that this handler is registered for
93	* @work: work struct
94	*/
95	struct amdgpu_dm_irq_handler_data {
96	struct list_head list;
97	interrupt_handler handler;
98	void *handler_arg;
99
100	struct amdgpu_display_manager *dm;
101	/* DAL irq source which registered for this interrupt. */
102	enum dc_irq_source irq_source;
103	struct work_struct work;
104	};
105
106	#define DM_IRQ_TABLE_LOCK(adev, flags) \
107	spin_lock_irqsave(&adev->dm.irq_handler_list_table_lock, flags)
108
109	#define DM_IRQ_TABLE_UNLOCK(adev, flags) \
110	spin_unlock_irqrestore(&adev->dm.irq_handler_list_table_lock, flags)
111
112	/******************************************************************************
113	* Private functions.
114	*****************************************************************************/
115
116	static void init_handler_common_data(struct amdgpu_dm_irq_handler_data *hcd,
117	void (ih)(void ),
118	void *args,
119	struct amdgpu_display_manager *dm)
120	{
121	hcd->handler = ih;
122	hcd->handler_arg = args;
123	hcd->dm = dm;
124	}
125
126	/**
127	* dm_irq_work_func() - Handle an IRQ outside of the interrupt handler proper.
128	*
129	* @work: work struct
130	*/
131	static void dm_irq_work_func(struct work_struct *work)
132	{
133	struct amdgpu_dm_irq_handler_data *handler_data =
134	container_of(work, struct amdgpu_dm_irq_handler_data, work);
135
136	handler_data->handler(handler_data->handler_arg);
137
138	/* Call a DAL subcomponent which registered for interrupt notification
139	* at INTERRUPT_LOW_IRQ_CONTEXT.
140	* (The most common use is HPD interrupt)
141	*/
142	}
143
144	/*
145	* Remove a handler and return a pointer to handler list from which the
146	* handler was removed.
147	*/
148	static struct list_head remove_irq_handler(struct amdgpu_device adev,
149	void *ih,
150	const struct dc_interrupt_params *int_params)
151	{
152	struct list_head *hnd_list;
153	struct list_head entry, tmp;
154	struct amdgpu_dm_irq_handler_data *handler;
155	unsigned long irq_table_flags;
156	bool handler_removed = false;
157	enum dc_irq_source irq_source;
158
159	DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
160
161	irq_source = int_params->irq_source;
162
163	switch (int_params->int_context) {
164	case INTERRUPT_HIGH_IRQ_CONTEXT:
165	hnd_list = &adev->dm.irq_handler_list_high_tab[irq_source];
166	break;
167	case INTERRUPT_LOW_IRQ_CONTEXT:
168	default:
169	hnd_list = &adev->dm.irq_handler_list_low_tab[irq_source];
170	break;
171	}
172
173	list_for_each_safe(entry, tmp, hnd_list) {
174
175	handler = list_entry(entry, struct amdgpu_dm_irq_handler_data,
176	list);
177
178	if (handler == NULL)
179	continue;
180
181	if (ih == handler->handler) {
182	/* Found our handler. Remove it from the list. */
183	list_del(&handler->list);
184	handler_removed = true;
185	break;
186	}
187	}
188
189	DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
190
191	if (!handler_removed) {
192	/* Not necessarily an error - caller may not
193	* know the context.
194	*/
195	return NULL;
196	}
197
198	if (int_params->int_context == INTERRUPT_LOW_IRQ_CONTEXT)
199	cancel_work_sync(&handler->work);
200
201	kfree(handler);
202
203	DRM_DEBUG_KMS(
204	"DM_IRQ: removed irq handler: %p for: dal_src=%d, irq context=%d\n",
205	ih, int_params->irq_source, int_params->int_context);
206
207	return hnd_list;
208	}
209
210	static bool
211	validate_irq_registration_params(struct dc_interrupt_params *int_params,
212	void (ih)(void ))
213	{
214	if (NULL == int_params \|\| NULL == ih) {
215	DRM_ERROR("DM_IRQ: invalid input!\n");
216	return false;
217	}
218
219	if (int_params->int_context >= INTERRUPT_CONTEXT_NUMBER) {
220	DRM_ERROR("DM_IRQ: invalid context: %d!\n",
221	int_params->int_context);
222	return false;
223	}
224
225	if (!DAL_VALID_IRQ_SRC_NUM(int_params->irq_source)) {
226	DRM_ERROR("DM_IRQ: invalid irq_source: %d!\n",
227	int_params->irq_source);
228	return false;
229	}
230
231	return true;
232	}
233
234	static bool validate_irq_unregistration_params(enum dc_irq_source irq_source,
235	irq_handler_idx handler_idx)
236	{
237	if (handler_idx == DAL_INVALID_IRQ_HANDLER_IDX) {
238	DRM_ERROR("DM_IRQ: invalid handler_idx==NULL!\n");
239	return false;
240	}
241
242	if (!DAL_VALID_IRQ_SRC_NUM(irq_source)) {
243	DRM_ERROR("DM_IRQ: invalid irq_source:%d!\n", irq_source);
244	return false;
245	}
246
247	return true;
248	}
249	/******************************************************************************
250	* Public functions.
251	*
252	* Note: caller is responsible for input validation.
253	*****************************************************************************/
254
255	/**
256	* amdgpu_dm_irq_register_interrupt() - Register a handler within DM.
257	* @adev: The base driver device containing the DM device.
258	* @int_params: Interrupt parameters containing the source, and handler context
259	* @ih: Function pointer to the interrupt handler to register
260	* @handler_args: Arguments passed to the handler when the interrupt occurs
261	*
262	* Register an interrupt handler for the given IRQ source, under the given
263	* context. The context can either be high or low. High context handlers are
264	* executed directly within ISR context, while low context is executed within a
265	* workqueue, thereby allowing operations that sleep.
266	*
267	* Registered handlers are called in a FIFO manner, i.e. the most recently
268	* registered handler will be called first.
269	*
270	* Return: Handler data &struct amdgpu_dm_irq_handler_data containing the IRQ
271	* source, handler function, and args
272	*/
273	void amdgpu_dm_irq_register_interrupt(struct amdgpu_device adev,
274	struct dc_interrupt_params *int_params,
275	void (ih)(void ),
276	void *handler_args)
277	{
278	struct list_head *hnd_list;
279	struct amdgpu_dm_irq_handler_data *handler_data;
280	unsigned long irq_table_flags;
281	enum dc_irq_source irq_source;
282
283	if (!validate_irq_registration_params(int_params, ih))
284	return DAL_INVALID_IRQ_HANDLER_IDX;
285
286	handler_data = kzalloc_obj(*handler_data);
287	if (!handler_data) {
288	DRM_ERROR("DM_IRQ: failed to allocate irq handler!\n");
289	return DAL_INVALID_IRQ_HANDLER_IDX;
290	}
291
292	init_handler_common_data(handler_data, ih, handler_args, &adev->dm);
293
294	irq_source = int_params->irq_source;
295
296	handler_data->irq_source = irq_source;
297
298	/* Lock the list, add the handler. */
299	DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
300
301	switch (int_params->int_context) {
302	case INTERRUPT_HIGH_IRQ_CONTEXT:
303	hnd_list = &adev->dm.irq_handler_list_high_tab[irq_source];
304	break;
305	case INTERRUPT_LOW_IRQ_CONTEXT:
306	default:
307	hnd_list = &adev->dm.irq_handler_list_low_tab[irq_source];
308	INIT_WORK(&handler_data->work, dm_irq_work_func);
309	break;
310	}
311
312	list_add_tail(&handler_data->list, hnd_list);
313
314	DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
315
316	/* This pointer will be stored by code which requested interrupt
317	* registration.
318	* The same pointer will be needed in order to unregister the
319	* interrupt.
320	*/
321
322	DRM_DEBUG_KMS(
323	"DM_IRQ: added irq handler: %p for: dal_src=%d, irq context=%d\n",
324	handler_data,
325	irq_source,
326	int_params->int_context);
327
328	return handler_data;
329	}
330	EXPORT_IF_KUNIT(amdgpu_dm_irq_register_interrupt);
331
332	/**
333	* amdgpu_dm_irq_unregister_interrupt() - Remove a handler from the DM IRQ table
334	* @adev: The base driver device containing the DM device
335	* @irq_source: IRQ source to remove the given handler from
336	* @ih: Function pointer to the interrupt handler to unregister
337	*
338	* Go through both low and high context IRQ tables, and find the given handler
339	* for the given irq source. If found, remove it. Otherwise, do nothing.
340	*/
341	void amdgpu_dm_irq_unregister_interrupt(struct amdgpu_device *adev,
342	enum dc_irq_source irq_source,
343	void *ih)
344	{
345	struct list_head *handler_list;
346	struct dc_interrupt_params int_params;
347	int i;
348
349	if (!validate_irq_unregistration_params(irq_source, ih))
350	return;
351
352	memset(&int_params, 0, sizeof(int_params));
353
354	int_params.irq_source = irq_source;
355
356	for (i = 0; i < INTERRUPT_CONTEXT_NUMBER; i++) {
357
358	int_params.int_context = i;
359
360	handler_list = remove_irq_handler(adev, ih, &int_params);
361
362	if (handler_list != NULL)
363	break;
364	}
365
366	if (handler_list == NULL) {
367	/* If we got here, it means we searched all irq contexts
368	* for this irq source, but the handler was not found.
369	*/
370	DRM_ERROR(
371	"DM_IRQ: failed to find irq handler:%p for irq_source:%d!\n",
372	ih, irq_source);
373	}
374	}
375	EXPORT_IF_KUNIT(amdgpu_dm_irq_unregister_interrupt);
376
377	/**
378	* amdgpu_dm_irq_init() - Initialize DM IRQ management
379	* @adev: The base driver device containing the DM device
380	*
381	* Initialize DM's high and low context IRQ tables.
382	*
383	* The N by M table contains N IRQ sources, with M
384	* &struct amdgpu_dm_irq_handler_data hooked together in a linked list. The
385	* list_heads are initialized here. When an interrupt n is triggered, all m
386	* handlers are called in sequence, FIFO according to registration order.
387	*
388	* The low context table requires special steps to initialize, since handlers
389	* will be deferred to a workqueue. See &struct irq_list_head.
390	*/
391	int amdgpu_dm_irq_init(struct amdgpu_device *adev)
392	{
393	int src;
394	struct list_head *lh;
395
396	DRM_DEBUG_KMS("DM_IRQ\n");
397
398	spin_lock_init(&adev->dm.irq_handler_list_table_lock);
399	spin_lock_init(&adev->dm.irq_reg_lock);
400
401	adev->dm.irq_wq = alloc_workqueue("amdgpu_dm_irq",
402	WQ_UNBOUND \| WQ_HIGHPRI, 0);
403
404	if (!adev->dm.irq_wq)
405	return -ENOMEM;
406
407	adev->dm.vmin_vmax_wq = alloc_workqueue("amdgpu_dm_vmin_vmax",
408	WQ_UNBOUND, 0);
409
410	if (!adev->dm.vmin_vmax_wq) {
411	destroy_workqueue(adev->dm.irq_wq);
412	adev->dm.irq_wq = NULL;
413	return -ENOMEM;
414	}
415
416	for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++) {
417	/* low context handler list init */
418	lh = &adev->dm.irq_handler_list_low_tab[src];
419	INIT_LIST_HEAD(lh);
420	/* high context handler init */
421	INIT_LIST_HEAD(&adev->dm.irq_handler_list_high_tab[src]);
422	}
423
424	return 0;
425	}
426	EXPORT_IF_KUNIT(amdgpu_dm_irq_init);
427
428	/**
429	* amdgpu_dm_irq_fini() - Tear down DM IRQ management
430	* @adev: The base driver device containing the DM device
431	*
432	* Removes all handlers from the IRQ tables under the spinlock, cancels
433	* pending work items, and deallocates all handler data.
434	*/
435	void amdgpu_dm_irq_fini(struct amdgpu_device *adev)
436	{
437	int src;
438	LIST_HEAD(low_handlers);
439	LIST_HEAD(high_handlers);
440	struct list_head entry, tmp;
441	struct amdgpu_dm_irq_handler_data *handler;
442	unsigned long irq_table_flags;
443
444	DRM_DEBUG_KMS("DM_IRQ: releasing resources.\n");
445
446	for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++) {
447	DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
448
449	/*
450	* Move all handlers from the low and high context tables to
451	* temporary lists under the lock. This prevents the ISR from
452	* finding them while we process them outside the lock.
453	*/
454	list_splice_init(&adev->dm.irq_handler_list_low_tab[src],
455	&low_handlers);
456	list_splice_init(&adev->dm.irq_handler_list_high_tab[src],
457	&high_handlers);
458
459	DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
460
461	/*
462	* Cancel all pending work for the low-context handlers
463	* outside the lock. cancel_work_sync() may sleep and waits
464	* until any running work completes, preventing UAF.
465	*/
466	list_for_each_safe(entry, tmp, &low_handlers) {
467	handler = list_entry(entry,
468	struct amdgpu_dm_irq_handler_data,
469	list);
470	cancel_work_sync(&handler->work);
471	}
472
473	/*
474	* High-context handlers are executed synchronously within ISR
475	* context (see amdgpu_dm_irq_immediate_work()) and have no
476	* work_struct, so there is no pending work to cancel here.
477	* They will be freed along with low_handlers after the loop.
478	*/
479	}
480
481	/* Deallocate all handlers. */
482	list_for_each_safe(entry, tmp, &low_handlers) {
483	handler = list_entry(entry,
484	struct amdgpu_dm_irq_handler_data,
485	list);
486	list_del(&handler->list);
487	kfree(handler);
488	}
489
490	list_for_each_safe(entry, tmp, &high_handlers) {
491	handler = list_entry(entry,
492	struct amdgpu_dm_irq_handler_data,
493	list);
494	list_del(&handler->list);
495	kfree(handler);
496	}
497
498	if (adev->dm.vmin_vmax_wq) {
499	destroy_workqueue(adev->dm.vmin_vmax_wq);
500	adev->dm.vmin_vmax_wq = NULL;
501	}
502
503	if (adev->dm.irq_wq) {
504	destroy_workqueue(adev->dm.irq_wq);
505	adev->dm.irq_wq = NULL;
506	}
507	}
508	EXPORT_IF_KUNIT(amdgpu_dm_irq_fini);
509
510	void amdgpu_dm_irq_suspend(struct amdgpu_device *adev)
511	{
512	struct drm_device *dev = adev_to_drm(adev);
513	int src;
514	struct list_head *hnd_list_l;
515	unsigned long irq_table_flags;
516	struct list_head entry, tmp;
517	struct amdgpu_dm_irq_handler_data *handler;
518
519	DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
520
521	DRM_DEBUG_KMS("DM_IRQ: suspend\n");
522
523	/**
524	* Disable HW interrupt for HPD and HPDRX only since FLIP and VBLANK
525	* will be disabled from manage_dm_interrupts on disable CRTC.
526	*
527	* Disable the HW interrupt first, then flush any pending work. Since
528	* the HW interrupt is disabled under the lock, no new IRQ can be
529	* generated after the disable completes. Any work already queued by an
530	* in-flight ISR will be flushed below.
531	*/
532	for (src = DC_IRQ_SOURCE_HPD1; src <= DC_IRQ_SOURCE_HPD6RX; src++) {
533	hnd_list_l = &adev->dm.irq_handler_list_low_tab[src];
534	amdgpu_dm_irq_set(adev, src, false);
535
536	DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
537
538	if (!list_empty(hnd_list_l)) {
539	list_for_each_safe(entry, tmp, hnd_list_l) {
540	handler = list_entry(
541	entry,
542	struct amdgpu_dm_irq_handler_data,
543	list);
544	flush_work(&handler->work);
545	}
546	}
547	DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
548	}
549
550	DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
551
552	if (dev->mode_config.poll_enabled)
553	drm_kms_helper_poll_disable(dev);
554	}
555	EXPORT_IF_KUNIT(amdgpu_dm_irq_suspend);
556
557	void amdgpu_dm_irq_resume_early(struct amdgpu_device *adev)
558	{
559	int src;
560	struct list_head hnd_list_h, hnd_list_l;
561	unsigned long irq_table_flags;
562
563	DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
564
565	drm_dbg(adev_to_drm(adev), "DM_IRQ: early resume\n");
566
567	/* re-enable short pulse interrupts HW interrupt */
568	for (src = DC_IRQ_SOURCE_HPD1RX; src <= DC_IRQ_SOURCE_HPD6RX; src++) {
569	hnd_list_l = &adev->dm.irq_handler_list_low_tab[src];
570	hnd_list_h = &adev->dm.irq_handler_list_high_tab[src];
571	if (!list_empty(hnd_list_l) \|\| !list_empty(hnd_list_h))
572	amdgpu_dm_irq_set(adev, src, true);
573	}
574
575	DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
576	}
577	EXPORT_IF_KUNIT(amdgpu_dm_irq_resume_early);
578
579	void amdgpu_dm_irq_resume_late(struct amdgpu_device *adev)
580	{
581	struct drm_device *dev = adev_to_drm(adev);
582	int src;
583	struct list_head hnd_list_h, hnd_list_l;
584	unsigned long irq_table_flags;
585
586	DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
587
588	drm_dbg(adev_to_drm(adev), "DM_IRQ: resume\n");
589
590	/**
591	* Renable HW interrupt for HPD and only since FLIP and VBLANK
592	* will be enabled from manage_dm_interrupts on enable CRTC.
593	*/
594	for (src = DC_IRQ_SOURCE_HPD1; src <= DC_IRQ_SOURCE_HPD6; src++) {
595	hnd_list_l = &adev->dm.irq_handler_list_low_tab[src];
596	hnd_list_h = &adev->dm.irq_handler_list_high_tab[src];
597	if (!list_empty(hnd_list_l) \|\| !list_empty(hnd_list_h))
598	amdgpu_dm_irq_set(adev, src, true);
599	}
600
601	DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
602
603	if (dev->mode_config.poll_enabled)
604	drm_kms_helper_poll_enable(dev);
605	}
606	EXPORT_IF_KUNIT(amdgpu_dm_irq_resume_late);
607
608	/*
609	* amdgpu_dm_irq_schedule_work - schedule all work items registered for the
610	* "irq_source".
611	*/
612	STATIC_IFN_KUNIT void amdgpu_dm_irq_schedule_work(struct amdgpu_device *adev,
613	enum dc_irq_source irq_source)
614	{
615	struct list_head *handler_list = &adev->dm.irq_handler_list_low_tab[irq_source];
616	struct amdgpu_dm_irq_handler_data *handler_data;
617	bool work_queued = false;
618	unsigned long irq_table_flags;
619
620	DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
621
622	if (list_empty(handler_list))
623	goto out_unlock;
624
625	list_for_each_entry(handler_data, handler_list, list) {
626	if (queue_work(adev->dm.irq_wq, &handler_data->work)) {
627	work_queued = true;
628	break;
629	}
630	}
631
632	if (!work_queued) {
633	struct amdgpu_dm_irq_handler_data *handler_data_add;
634	/get the amdgpu_dm_irq_handler_data of first item pointed by handler_list/
635	handler_data = container_of(handler_list->next, struct amdgpu_dm_irq_handler_data, list);
636
637	/allocate a new amdgpu_dm_irq_handler_data/
638	handler_data_add = kzalloc_obj(*handler_data, GFP_ATOMIC);
639	if (!handler_data_add) {
640	DRM_ERROR("DM_IRQ: failed to allocate irq handler!\n");
641	goto out_unlock;
642	}
643
644	/copy new amdgpu_dm_irq_handler_data members from handler_data/
645	handler_data_add->handler = handler_data->handler;
646	handler_data_add->handler_arg = handler_data->handler_arg;
647	handler_data_add->dm = handler_data->dm;
648	handler_data_add->irq_source = irq_source;
649
650	list_add_tail(&handler_data_add->list, handler_list);
651
652	INIT_WORK(&handler_data_add->work, dm_irq_work_func);
653
654	if (queue_work(adev->dm.irq_wq, &handler_data_add->work))
655	DRM_DEBUG("Queued work for handling interrupt from "
656	"display for IRQ source %d\n",
657	irq_source);
658	else
659	DRM_ERROR("Failed to queue work for handling interrupt "
660	"from display for IRQ source %d\n",
661	irq_source);
662	}
663
664	out_unlock:
665	DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
666	}
667	EXPORT_IF_KUNIT(amdgpu_dm_irq_schedule_work);
668
669	/*
670	* amdgpu_dm_irq_immediate_work
671	* Callback high irq work immediately, don't send to work queue
672	*/
673	STATIC_IFN_KUNIT void amdgpu_dm_irq_immediate_work(struct amdgpu_device *adev,
674	enum dc_irq_source irq_source)
675	{
676	struct amdgpu_dm_irq_handler_data *handler_data;
677	unsigned long irq_table_flags;
678
679	DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
680
681	list_for_each_entry(handler_data,
682	&adev->dm.irq_handler_list_high_tab[irq_source],
683	list) {
684	/* Call a subcomponent which registered for immediate
685	* interrupt notification
686	*/
687	handler_data->handler(handler_data->handler_arg);
688	}
689
690	DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
691	}
692	EXPORT_IF_KUNIT(amdgpu_dm_irq_immediate_work);
693
694	bool amdgpu_dm_irq_set(struct amdgpu_device *adev, enum dc_irq_source src,
695	bool enable)
696	{
697	guard(spinlock_irqsave)(&adev->dm.irq_reg_lock);
698
699	return dc_interrupt_set(adev->dm.dc, src, enable);
700	}
701	EXPORT_IF_KUNIT(amdgpu_dm_irq_set);
702
703	void amdgpu_dm_irq_ack(struct amdgpu_device *adev, enum dc_irq_source src)
704	{
705	guard(spinlock_irqsave)(&adev->dm.irq_reg_lock);
706
707	dc_interrupt_ack(adev->dm.dc, src);
708	}
709	EXPORT_IF_KUNIT(amdgpu_dm_irq_ack);
710
711	/**
712	* amdgpu_dm_irq_handler - Generic DM IRQ handler
713	* @adev: amdgpu base driver device containing the DM device
714	* @source: Unused
715	* @entry: Data about the triggered interrupt
716	*
717	* Calls all registered high irq work immediately, and schedules work for low
718	* irq. The DM IRQ table is used to find the corresponding handlers.
719	*/
720	STATIC_IFN_KUNIT int amdgpu_dm_irq_handler(struct amdgpu_device *adev,
721	struct amdgpu_irq_src *source,
722	struct amdgpu_iv_entry *entry)
723	{
724
725	enum dc_irq_source src =
726	dc_interrupt_to_irq_source(
727	adev->dm.dc,
728	entry->src_id,
729	entry->src_data[0]);
730
731	amdgpu_dm_irq_ack(adev, src);
732
733	/* Call high irq work immediately */
734	amdgpu_dm_irq_immediate_work(adev, src);
735	/Schedule low_irq work /
736	amdgpu_dm_irq_schedule_work(adev, src);
737
738	return 0;
739	}
740	EXPORT_IF_KUNIT(amdgpu_dm_irq_handler);
741
742	STATIC_IFN_KUNIT enum dc_irq_source amdgpu_dm_hpd_to_dal_irq_source(unsigned int type)
743	{
744	switch (type) {
745	case AMDGPU_HPD_1:
746	return DC_IRQ_SOURCE_HPD1;
747	case AMDGPU_HPD_2:
748	return DC_IRQ_SOURCE_HPD2;
749	case AMDGPU_HPD_3:
750	return DC_IRQ_SOURCE_HPD3;
751	case AMDGPU_HPD_4:
752	return DC_IRQ_SOURCE_HPD4;
753	case AMDGPU_HPD_5:
754	return DC_IRQ_SOURCE_HPD5;
755	case AMDGPU_HPD_6:
756	return DC_IRQ_SOURCE_HPD6;
757	default:
758	return DC_IRQ_SOURCE_INVALID;
759	}
760	}
761	EXPORT_IF_KUNIT(amdgpu_dm_hpd_to_dal_irq_source);
762
763	STATIC_IFN_KUNIT int amdgpu_dm_set_hpd_irq_state(struct amdgpu_device *adev,
764	struct amdgpu_irq_src *source,
765	unsigned int type,
766	enum amdgpu_interrupt_state state)
767	{
768	enum dc_irq_source src = amdgpu_dm_hpd_to_dal_irq_source(type);
769	bool st = (state == AMDGPU_IRQ_STATE_ENABLE);
770
771	amdgpu_dm_irq_set(adev, src, st);
772	return 0;
773	}
774	EXPORT_IF_KUNIT(amdgpu_dm_set_hpd_irq_state);
775
776	static inline int dm_irq_state(struct amdgpu_device *adev,
777	struct amdgpu_irq_src *source,
778	unsigned int crtc_id,
779	enum amdgpu_interrupt_state state,
780	const enum irq_type dal_irq_type,
781	const char *func)
782	{
783	bool st;
784	enum dc_irq_source irq_source;
785	struct dc *dc = adev->dm.dc;
786	struct amdgpu_crtc *acrtc = adev->mode_info.crtcs[crtc_id];
787
788	if (!acrtc) {
789	DRM_ERROR(
790	"%s: crtc is NULL at id :%d\n",
791	func,
792	crtc_id);
793	return 0;
794	}
795
796	if (acrtc->otg_inst == -1)
797	return 0;
798
799	irq_source = dal_irq_type + acrtc->otg_inst;
800
801	st = (state == AMDGPU_IRQ_STATE_ENABLE);
802
803	if (dc && dc->caps.ips_support && dc->idle_optimizations_allowed)
804	dc_allow_idle_optimizations(dc, false);
805
806	amdgpu_dm_irq_set(adev, irq_source, st);
807	return 0;
808	}
809
810	STATIC_IFN_KUNIT int amdgpu_dm_set_pflip_irq_state(struct amdgpu_device *adev,
811	struct amdgpu_irq_src *source,
812	unsigned int crtc_id,
813	enum amdgpu_interrupt_state state)
814	{
815	return dm_irq_state(
816	adev,
817	source,
818	crtc_id,
819	state,
820	IRQ_TYPE_PFLIP,
821	__func__);
822	}
823	EXPORT_IF_KUNIT(amdgpu_dm_set_pflip_irq_state);
824
825	STATIC_IFN_KUNIT int amdgpu_dm_set_crtc_irq_state(struct amdgpu_device *adev,
826	struct amdgpu_irq_src *source,
827	unsigned int crtc_id,
828	enum amdgpu_interrupt_state state)
829	{
830	return dm_irq_state(
831	adev,
832	source,
833	crtc_id,
834	state,
835	IRQ_TYPE_VBLANK,
836	__func__);
837	}
838	EXPORT_IF_KUNIT(amdgpu_dm_set_crtc_irq_state);
839
840	STATIC_IFN_KUNIT int amdgpu_dm_set_vline0_irq_state(struct amdgpu_device *adev,
841	struct amdgpu_irq_src *source,
842	unsigned int crtc_id,
843	enum amdgpu_interrupt_state state)
844	{
845	return dm_irq_state(
846	adev,
847	source,
848	crtc_id,
849	state,
850	IRQ_TYPE_VLINE0,
851	__func__);
852	}
853	EXPORT_IF_KUNIT(amdgpu_dm_set_vline0_irq_state);
854
855	STATIC_IFN_KUNIT int amdgpu_dm_set_dmub_outbox_irq_state(struct amdgpu_device *adev,
856	struct amdgpu_irq_src *source,
857	unsigned int crtc_id,
858	enum amdgpu_interrupt_state state)
859	{
860	enum dc_irq_source irq_source = DC_IRQ_SOURCE_DMCUB_OUTBOX;
861	bool st = (state == AMDGPU_IRQ_STATE_ENABLE);
862
863	amdgpu_dm_irq_set(adev, irq_source, st);
864	return 0;
865	}
866	EXPORT_IF_KUNIT(amdgpu_dm_set_dmub_outbox_irq_state);
867
868	STATIC_IFN_KUNIT int amdgpu_dm_set_vupdate_irq_state(struct amdgpu_device *adev,
869	struct amdgpu_irq_src *source,
870	unsigned int crtc_id,
871	enum amdgpu_interrupt_state state)
872	{
873	return dm_irq_state(
874	adev,
875	source,
876	crtc_id,
877	state,
878	IRQ_TYPE_VUPDATE,
879	__func__);
880	}
881	EXPORT_IF_KUNIT(amdgpu_dm_set_vupdate_irq_state);
882
883	STATIC_IFN_KUNIT int amdgpu_dm_set_dmub_trace_irq_state(struct amdgpu_device *adev,
884	struct amdgpu_irq_src *source,
885	unsigned int type,
886	enum amdgpu_interrupt_state state)
887	{
888	enum dc_irq_source irq_source = DC_IRQ_SOURCE_DMCUB_OUTBOX0;
889	bool st = (state == AMDGPU_IRQ_STATE_ENABLE);
890
891	amdgpu_dm_irq_set(adev, irq_source, st);
892	return 0;
893	}
894	EXPORT_IF_KUNIT(amdgpu_dm_set_dmub_trace_irq_state);
895
896	static const struct amdgpu_irq_src_funcs dm_crtc_irq_funcs = {
897	.set = amdgpu_dm_set_crtc_irq_state,
898	.process = amdgpu_dm_irq_handler,
899	};
900
901	static const struct amdgpu_irq_src_funcs dm_vline0_irq_funcs = {
902	.set = amdgpu_dm_set_vline0_irq_state,
903	.process = amdgpu_dm_irq_handler,
904	};
905
906	static const struct amdgpu_irq_src_funcs dm_dmub_outbox_irq_funcs = {
907	.set = amdgpu_dm_set_dmub_outbox_irq_state,
908	.process = amdgpu_dm_irq_handler,
909	};
910
911	static const struct amdgpu_irq_src_funcs dm_vupdate_irq_funcs = {
912	.set = amdgpu_dm_set_vupdate_irq_state,
913	.process = amdgpu_dm_irq_handler,
914	};
915
916	static const struct amdgpu_irq_src_funcs dm_dmub_trace_irq_funcs = {
917	.set = amdgpu_dm_set_dmub_trace_irq_state,
918	.process = amdgpu_dm_irq_handler,
919	};
920
921	static const struct amdgpu_irq_src_funcs dm_pageflip_irq_funcs = {
922	.set = amdgpu_dm_set_pflip_irq_state,
923	.process = amdgpu_dm_irq_handler,
924	};
925
926	static const struct amdgpu_irq_src_funcs dm_hpd_irq_funcs = {
927	.set = amdgpu_dm_set_hpd_irq_state,
928	.process = amdgpu_dm_irq_handler,
929	};
930
931	void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev)
932	{
933	adev->crtc_irq.num_types = adev->mode_info.num_crtc;
934	adev->crtc_irq.funcs = &dm_crtc_irq_funcs;
935
936	adev->vline0_irq.num_types = adev->mode_info.num_crtc;
937	adev->vline0_irq.funcs = &dm_vline0_irq_funcs;
938
939	adev->dmub_outbox_irq.num_types = 1;
940	adev->dmub_outbox_irq.funcs = &dm_dmub_outbox_irq_funcs;
941
942	adev->vupdate_irq.num_types = adev->mode_info.num_crtc;
943	adev->vupdate_irq.funcs = &dm_vupdate_irq_funcs;
944
945	adev->dmub_trace_irq.num_types = 1;
946	adev->dmub_trace_irq.funcs = &dm_dmub_trace_irq_funcs;
947
948	adev->pageflip_irq.num_types = adev->mode_info.num_crtc;
949	adev->pageflip_irq.funcs = &dm_pageflip_irq_funcs;
950
951	adev->hpd_irq.num_types = adev->mode_info.num_hpd;
952	adev->hpd_irq.funcs = &dm_hpd_irq_funcs;
953	}
954	EXPORT_IF_KUNIT(amdgpu_dm_set_irq_funcs);
955
956	void amdgpu_dm_outbox_init(struct amdgpu_device *adev)
957	{
958	amdgpu_dm_irq_set(adev, DC_IRQ_SOURCE_DMCUB_OUTBOX, true);
959	}
960	EXPORT_IF_KUNIT(amdgpu_dm_outbox_init);
961
962	/**
963	* amdgpu_dm_hpd_init - hpd setup callback.
964	*
965	* @adev: amdgpu_device pointer
966	*
967	* Setup the hpd pins used by the card (evergreen+).
968	* Enable the pin, set the polarity, and enable the hpd interrupts.
969	*/
970	void amdgpu_dm_hpd_init(struct amdgpu_device *adev)
971	{
972	struct drm_device *dev = adev_to_drm(adev);
973	struct drm_connector *connector;
974	struct drm_connector_list_iter iter;
975	int irq_type;
976	int i;
977	bool use_polling = false;
978
979	/* First, clear all hpd and hpdrx interrupts */
980	for (i = DC_IRQ_SOURCE_HPD1; i <= DC_IRQ_SOURCE_HPD6RX; i++) {
981	if (!amdgpu_dm_irq_set(adev, i, false))
982	drm_err(dev, "Failed to clear hpd(rx) source=%d on init\n",
983	i);
984	}
985
986	drm_connector_list_iter_begin(dev, &iter);
987	drm_for_each_connector_iter(connector, &iter) {
988	struct amdgpu_dm_connector *amdgpu_dm_connector;
989	const struct dc_link *dc_link;
990
991	if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
992	continue;
993
994	amdgpu_dm_connector = to_amdgpu_dm_connector(connector);
995	dc_link = amdgpu_dm_connector->dc_link;
996	if (!dc_link)
997	continue;
998
999	/*
1000	* Analog connectors may be hot-plugged unlike other connector
1001	* types that don't support HPD. Only poll analog connectors.
1002	*/
1003	use_polling \|= dc_connector_supports_analog(dc_link->link_id.id);
1004
1005	/*
1006	* Get a base driver irq reference for hpd ints for the lifetime
1007	* of dm. Note that only hpd interrupt types are registered with
1008	* base driver; hpd_rx types aren't. IOW, amdgpu_irq_get/put on
1009	* hpd_rx isn't available. DM currently controls hpd_rx
1010	* explicitly with amdgpu_dm_irq_set()
1011	*/
1012	if (dc_link->irq_source_hpd != DC_IRQ_SOURCE_INVALID) {
1013	irq_type = dc_link->irq_source_hpd - DC_IRQ_SOURCE_HPD1;
1014	/*
1015	* TODO: There's a mismatch between mode_info.num_hpd
1016	* and what bios reports as the # of connectors with hpd
1017	* sources. Since the # of hpd source types registered
1018	* with base driver == mode_info.num_hpd, we have to
1019	* fallback to amdgpu_dm_irq_set for the remaining types.
1020	*/
1021	if (irq_type < adev->mode_info.num_hpd) {
1022	if (amdgpu_irq_get(adev, &adev->hpd_irq, irq_type))
1023	drm_err(dev, "DM_IRQ: Failed get HPD for source=%d)!\n",
1024	dc_link->irq_source_hpd);
1025	} else {
1026	amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd,
1027	true);
1028	}
1029	}
1030
1031	if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) {
1032	amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd_rx,
1033	true);
1034	}
1035	}
1036	drm_connector_list_iter_end(&iter);
1037
1038	if (use_polling)
1039	drm_kms_helper_poll_init(dev);
1040	}
1041	EXPORT_IF_KUNIT(amdgpu_dm_hpd_init);
1042
1043	/**
1044	* amdgpu_dm_hpd_fini - hpd tear down callback.
1045	*
1046	* @adev: amdgpu_device pointer
1047	*
1048	* Tear down the hpd pins used by the card (evergreen+).
1049	* Disable the hpd interrupts.
1050	*/
1051	void amdgpu_dm_hpd_fini(struct amdgpu_device *adev)
1052	{
1053	struct drm_device *dev = adev_to_drm(adev);
1054	struct drm_connector *connector;
1055	struct drm_connector_list_iter iter;
1056	int irq_type;
1057
1058	drm_connector_list_iter_begin(dev, &iter);
1059	drm_for_each_connector_iter(connector, &iter) {
1060	struct amdgpu_dm_connector *amdgpu_dm_connector;
1061	const struct dc_link *dc_link;
1062
1063	if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
1064	continue;
1065
1066	amdgpu_dm_connector = to_amdgpu_dm_connector(connector);
1067	dc_link = amdgpu_dm_connector->dc_link;
1068
1069	if (dc_link->irq_source_hpd != DC_IRQ_SOURCE_INVALID) {
1070	irq_type = dc_link->irq_source_hpd - DC_IRQ_SOURCE_HPD1;
1071
1072	/* TODO: See same TODO in amdgpu_dm_hpd_init() */
1073	if (irq_type < adev->mode_info.num_hpd) {
1074	if (amdgpu_irq_put(adev, &adev->hpd_irq, irq_type))
1075	drm_err(dev, "DM_IRQ: Failed put HPD for source=%d!\n",
1076	dc_link->irq_source_hpd);
1077	} else {
1078	amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd,
1079	false);
1080	}
1081	}
1082
1083	if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) {
1084	amdgpu_dm_irq_set(adev, dc_link->irq_source_hpd_rx,
1085	false);
1086	}
1087	}
1088	drm_connector_list_iter_end(&iter);
1089
1090	if (dev->mode_config.poll_enabled)
1091	drm_kms_helper_poll_fini(dev);
1092	}
1093	EXPORT_IF_KUNIT(amdgpu_dm_hpd_fini);
1094
1095	/* ========== HPD handling ========== */
1096	static void force_connector_state(
1097	struct amdgpu_dm_connector *aconnector,
1098	enum drm_connector_force force_state)
1099	{
1100	struct drm_connector *connector = &aconnector->base;
1101
1102	mutex_lock(&connector->dev->mode_config.mutex);
1103	aconnector->base.force = force_state;
1104	mutex_unlock(&connector->dev->mode_config.mutex);
1105
1106	mutex_lock(&aconnector->hpd_lock);
1107	drm_kms_helper_connector_hotplug_event(connector);
1108	mutex_unlock(&aconnector->hpd_lock);
1109	}
1110
1111	STATIC_IFN_KUNIT void dm_handle_hpd_rx_offload_work(struct work_struct *work)
1112	{
1113	struct hpd_rx_irq_offload_work *offload_work;
1114	struct amdgpu_dm_connector *aconnector;
1115	struct dc_link *dc_link;
1116	struct amdgpu_device *adev;
1117	enum dc_connection_type new_connection_type = dc_connection_none;
1118	unsigned long flags;
1119	union test_response test_response;
1120
1121	memset(&test_response, 0, sizeof(test_response));
1122
1123	offload_work = container_of(work, struct hpd_rx_irq_offload_work, work);
1124	aconnector = offload_work->offload_wq->aconnector;
1125	adev = offload_work->adev;
1126
1127	if (!aconnector) {
1128	drm_err(adev_to_drm(adev), "Can't retrieve aconnector in hpd_rx_irq_offload_work");
1129	goto skip;
1130	}
1131
1132	dc_link = aconnector->dc_link;
1133
1134	mutex_lock(&aconnector->hpd_lock);
1135	if (!dc_link_detect_connection_type(dc_link, &new_connection_type))
1136	drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n");
1137	mutex_unlock(&aconnector->hpd_lock);
1138
1139	if (new_connection_type == dc_connection_none)
1140	goto skip;
1141
1142	if (amdgpu_in_reset(adev))
1143	goto skip;
1144
1145	if (offload_work->data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY \|\|
1146	offload_work->data.bytes.device_service_irq.bits.DOWN_REP_MSG_RDY) {
1147	dm_handle_mst_sideband_msg_ready_event(&aconnector->mst_mgr, DOWN_OR_UP_MSG_RDY_EVENT);
1148	spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags);
1149	offload_work->offload_wq->is_handling_mst_msg_rdy_event = false;
1150	spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags);
1151	goto skip;
1152	}
1153
1154	mutex_lock(&adev->dm.dc_lock);
1155	if (offload_work->data.bytes.device_service_irq.bits.AUTOMATED_TEST) {
1156	dc_link_dp_handle_automated_test(dc_link);
1157
1158	if (aconnector->timing_changed) {
1159	/* force connector disconnect and reconnect */
1160	force_connector_state(aconnector, DRM_FORCE_OFF);
1161	msleep(100);
1162	force_connector_state(aconnector, DRM_FORCE_UNSPECIFIED);
1163	}
1164
1165	test_response.bits.ACK = 1;
1166
1167	core_link_write_dpcd(
1168	dc_link,
1169	DP_TEST_RESPONSE,
1170	&test_response.raw,
1171	sizeof(test_response));
1172	} else if ((dc_link->connector_signal != SIGNAL_TYPE_EDP) &&
1173	dc_link_check_link_loss_status(dc_link, &offload_work->data) &&
1174	dc_link_dp_allow_hpd_rx_irq(dc_link)) {
1175	/* offload_work->data is from handle_hpd_rx_irq->
1176	* schedule_hpd_rx_offload_work.this is defer handle
1177	* for hpd short pulse. upon here, link status may be
1178	* changed, need get latest link status from dpcd
1179	* registers. if link status is good, skip run link
1180	* training again.
1181	*/
1182	union hpd_irq_data irq_data;
1183
1184	memset(&irq_data, 0, sizeof(irq_data));
1185
1186	/* before dc_link_dp_handle_link_loss, allow new link lost handle
1187	* request be added to work queue if link lost at end of dc_link_
1188	* dp_handle_link_loss
1189	*/
1190	spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags);
1191	offload_work->offload_wq->is_handling_link_loss = false;
1192	spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags);
1193
1194	if ((dc_link_dp_read_hpd_rx_irq_data(dc_link, &irq_data) == DC_OK) &&
1195	dc_link_check_link_loss_status(dc_link, &irq_data))
1196	dc_link_dp_handle_link_loss(dc_link);
1197	}
1198	mutex_unlock(&adev->dm.dc_lock);
1199
1200	skip:
1201	kfree(offload_work);
1202
1203	}
1204	EXPORT_IF_KUNIT(dm_handle_hpd_rx_offload_work);
1205
1206	struct hpd_rx_irq_offload_work_queue amdgpu_dm_hpd_rx_irq_create_workqueue(struct amdgpu_device adev)
1207	{
1208	struct dc *dc = adev->dm.dc;
1209	int max_caps = dc->caps.max_links;
1210	int i = 0;
1211	struct hpd_rx_irq_offload_work_queue *hpd_rx_offload_wq = NULL;
1212
1213	hpd_rx_offload_wq = kzalloc_objs(*hpd_rx_offload_wq, max_caps);
1214
1215	if (!hpd_rx_offload_wq)
1216	return NULL;
1217
1218
1219	for (i = 0; i < max_caps; i++) {
1220	hpd_rx_offload_wq[i].wq =
1221	create_singlethread_workqueue("amdgpu_dm_hpd_rx_offload_wq");
1222
1223	if (hpd_rx_offload_wq[i].wq == NULL) {
1224	drm_err(adev_to_drm(adev), "create amdgpu_dm_hpd_rx_offload_wq fail!");
1225	goto out_err;
1226	}
1227
1228	spin_lock_init(&hpd_rx_offload_wq[i].offload_lock);
1229	}
1230
1231	return hpd_rx_offload_wq;
1232
1233	out_err:
1234	for (i = 0; i < max_caps; i++) {
1235	if (hpd_rx_offload_wq[i].wq)
1236	destroy_workqueue(hpd_rx_offload_wq[i].wq);
1237	}
1238	kfree(hpd_rx_offload_wq);
1239	return NULL;
1240	}
1241	EXPORT_IF_KUNIT(amdgpu_dm_hpd_rx_irq_create_workqueue);
1242
1243	void amdgpu_dm_hpd_rx_irq_work_suspend(struct amdgpu_display_manager *dm)
1244	{
1245	int i;
1246
1247	if (dm->hpd_rx_offload_wq) {
1248	for (i = 0; i < dm->dc->caps.max_links; i++)
1249	flush_workqueue(dm->hpd_rx_offload_wq[i].wq);
1250	}
1251	}
1252	EXPORT_IF_KUNIT(amdgpu_dm_hpd_rx_irq_work_suspend);
1253
1254	STATIC_IFN_KUNIT bool are_sinks_equal(const struct dc_sink sink1, const struct dc_sink sink2)
1255	{
1256	if (!sink1 \|\| !sink2)
1257	return false;
1258	if (sink1->sink_signal != sink2->sink_signal)
1259	return false;
1260
1261	if (sink1->dc_edid.length != sink2->dc_edid.length)
1262	return false;
1263
1264	if (memcmp(sink1->dc_edid.raw_edid, sink2->dc_edid.raw_edid,
1265	sink1->dc_edid.length) != 0)
1266	return false;
1267	return true;
1268	}
1269	EXPORT_IF_KUNIT(are_sinks_equal);
1270
1271
1272	/**
1273	* DOC: amdgpu_dm_hdmi_hpd_debounce_work
1274	*
1275	* HDMI HPD debounce delay in milliseconds. When an HDMI display toggles HPD
1276	* (such as during power save transitions), this delay determines how long to
1277	* wait before processing the HPD event. This allows distinguishing between a
1278	* physical unplug (>hdmi_hpd_debounce_delay)
1279	* and a spontaneous RX HPD toggle (<hdmi_hpd_debounce_delay).
1280	*
1281	* If the toggle is less than this delay, the driver compares sink capabilities
1282	* and permits a hotplug event if they changed.
1283	*
1284	* The default value of 1500ms was chosen based on experimental testing with
1285	* various monitors that exhibit spontaneous HPD toggling behavior.
1286	*/
1287	void amdgpu_dm_hdmi_hpd_debounce_work(struct work_struct *work)
1288	{
1289	struct amdgpu_dm_connector *aconnector =
1290	container_of(to_delayed_work(work), struct amdgpu_dm_connector,
1291	hdmi_hpd_debounce_work);
1292	struct drm_connector *connector = &aconnector->base;
1293	struct drm_device *dev = connector->dev;
1294	struct amdgpu_device *adev = drm_to_adev(dev);
1295	struct dc *dc = aconnector->dc_link->ctx->dc;
1296	bool fake_reconnect = false;
1297	bool reallow_idle = false;
1298	bool ret = false;
1299
1300	guard(mutex)(&aconnector->hpd_lock);
1301
1302	/* Re-detect the display */
1303	scoped_guard(mutex, &adev->dm.dc_lock) {
1304	if (dc->caps.ips_support && dc->ctx->dmub_srv->idle_allowed) {
1305	dc_allow_idle_optimizations(dc, false);
1306	reallow_idle = true;
1307	}
1308	ret = dc_link_detect(aconnector->dc_link, DETECT_REASON_HPD);
1309	}
1310
1311	if (ret) {
1312	/* Apply workaround delay for certain panels */
1313	amdgpu_dm_apply_delay_after_dpcd_poweroff(adev, aconnector->dc_sink);
1314	/* Compare sinks to determine if this was a spontaneous HPD toggle */
1315	if (are_sinks_equal(aconnector->dc_link->local_sink, aconnector->hdmi_prev_sink)) {
1316	/*
1317	* Sinks match - this was a spontaneous HDMI HPD toggle.
1318	*/
1319	drm_dbg_kms(dev, "HDMI HPD: Sink unchanged after debounce, internal re-enable\n");
1320	fake_reconnect = true;
1321	}
1322
1323	/* Update connector state */
1324	amdgpu_dm_update_connector_after_detect(aconnector);
1325
1326	drm_modeset_lock_all(dev);
1327	dm_restore_drm_connector_state(dev, connector);
1328	drm_modeset_unlock_all(dev);
1329
1330	/* Only notify OS if sink actually changed */
1331	if (!fake_reconnect && aconnector->base.force == DRM_FORCE_UNSPECIFIED)
1332	drm_kms_helper_hotplug_event(dev);
1333	}
1334
1335	/* Release the cached sink reference */
1336	if (aconnector->hdmi_prev_sink) {
1337	dc_sink_release(aconnector->hdmi_prev_sink);
1338	aconnector->hdmi_prev_sink = NULL;
1339	}
1340
1341	scoped_guard(mutex, &adev->dm.dc_lock) {
1342	if (reallow_idle && dc->caps.ips_support)
1343	dc_allow_idle_optimizations(dc, true);
1344	}
1345	}
1346	EXPORT_IF_KUNIT(amdgpu_dm_hdmi_hpd_debounce_work);
1347
1348	STATIC_IFN_KUNIT void handle_hpd_irq_helper(struct amdgpu_dm_connector *aconnector,
1349	enum dc_detect_reason reason)
1350	{
1351	struct drm_connector *connector = &aconnector->base;
1352	struct drm_device *dev = connector->dev;
1353	enum dc_connection_type new_connection_type = dc_connection_none;
1354	struct amdgpu_device *adev = drm_to_adev(dev);
1355	struct dm_connector_state *dm_con_state = to_dm_connector_state(connector->state);
1356	struct dc *dc = aconnector->dc_link->ctx->dc;
1357	bool ret = false;
1358	bool debounce_required = false;
1359
1360	if (adev->dm.disable_hpd_irq)
1361	return;
1362
1363	/*
1364	* In case of failure or MST no need to update connector status or notify the OS
1365	* since (for MST case) MST does this in its own context.
1366	*/
1367	guard(mutex)(&aconnector->hpd_lock);
1368
1369	if (adev->dm.hdcp_workqueue) {
1370	hdcp_reset_display(adev->dm.hdcp_workqueue, aconnector->dc_link->link_index);
1371	dm_con_state->update_hdcp = true;
1372	}
1373	if (aconnector->fake_enable)
1374	aconnector->fake_enable = false;
1375
1376	aconnector->timing_changed = false;
1377
1378	if (!dc_link_detect_connection_type(aconnector->dc_link, &new_connection_type))
1379	drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n");
1380
1381	/*
1382	* Check for HDMI disconnect with debounce enabled.
1383	*/
1384	debounce_required = (aconnector->hdmi_hpd_debounce_delay_ms > 0 &&
1385	dc_is_hdmi_signal(aconnector->dc_link->connector_signal) &&
1386	new_connection_type == dc_connection_none &&
1387	aconnector->dc_link->local_sink != NULL);
1388
1389	if (aconnector->base.force && new_connection_type == dc_connection_none) {
1390	amdgpu_dm_emulated_link_detect(aconnector->dc_link);
1391
1392	drm_modeset_lock_all(dev);
1393	dm_restore_drm_connector_state(dev, connector);
1394	drm_modeset_unlock_all(dev);
1395
1396	if (aconnector->base.force == DRM_FORCE_UNSPECIFIED \|\|
1397	reason == DETECT_REASON_HPDRX)
1398	drm_kms_helper_connector_hotplug_event(connector);
1399	} else if (debounce_required) {
1400	/*
1401	* HDMI disconnect detected - schedule delayed work instead of
1402	* processing immediately. This allows us to coalesce spurious
1403	* HDMI signals from physical unplugs.
1404	*/
1405	drm_dbg_kms(dev, "HDMI HPD: Disconnect detected, scheduling debounce work (%u ms)\n",
1406	aconnector->hdmi_hpd_debounce_delay_ms);
1407
1408	/* Cache the current sink for later comparison */
1409	if (aconnector->hdmi_prev_sink)
1410	dc_sink_release(aconnector->hdmi_prev_sink);
1411	aconnector->hdmi_prev_sink = aconnector->dc_link->local_sink;
1412	if (aconnector->hdmi_prev_sink)
1413	dc_sink_retain(aconnector->hdmi_prev_sink);
1414
1415	/* Schedule delayed detection. */
1416	if (mod_delayed_work(system_percpu_wq,
1417	&aconnector->hdmi_hpd_debounce_work,
1418	msecs_to_jiffies(aconnector->hdmi_hpd_debounce_delay_ms)))
1419	drm_dbg_kms(dev, "HDMI HPD: Re-scheduled debounce work\n");
1420
1421	} else {
1422
1423	/* If the aconnector->hdmi_hpd_debounce_work is scheduled, exit early */
1424	if (delayed_work_pending(&aconnector->hdmi_hpd_debounce_work))
1425	return;
1426
1427	scoped_guard(mutex, &adev->dm.dc_lock) {
1428	dc_exit_ips_for_hw_access(dc);
1429	ret = dc_link_detect(aconnector->dc_link, reason);
1430	}
1431	if (ret) {
1432	/* w/a delay for certain panels */
1433	amdgpu_dm_apply_delay_after_dpcd_poweroff(adev, aconnector->dc_sink);
1434	amdgpu_dm_update_connector_after_detect(aconnector);
1435
1436	drm_modeset_lock_all(dev);
1437	dm_restore_drm_connector_state(dev, connector);
1438	drm_modeset_unlock_all(dev);
1439
1440	if (aconnector->base.force == DRM_FORCE_UNSPECIFIED \|\|
1441	reason == DETECT_REASON_HPDRX)
1442	drm_kms_helper_connector_hotplug_event(connector);
1443	}
1444	}
1445	}
1446	EXPORT_IF_KUNIT(handle_hpd_irq_helper);
1447
1448	STATIC_IFN_KUNIT void handle_hpd_irq(void *param)
1449	{
1450	struct amdgpu_dm_connector aconnector = (struct amdgpu_dm_connector )param;
1451
1452	handle_hpd_irq_helper(aconnector, DETECT_REASON_HPD);
1453
1454	}
1455	EXPORT_IF_KUNIT(handle_hpd_irq);
1456
1457	STATIC_IFN_KUNIT void schedule_hpd_rx_offload_work(struct amdgpu_device adev, struct hpd_rx_irq_offload_work_queue offload_wq,
1458	union hpd_irq_data hpd_irq_data)
1459	{
1460	struct hpd_rx_irq_offload_work offload_work = kzalloc_obj(offload_work);
1461
1462	if (!offload_work) {
1463	drm_err(adev_to_drm(adev), "Failed to allocate hpd_rx_irq_offload_work.\n");
1464	return;
1465	}
1466
1467	INIT_WORK(&offload_work->work, dm_handle_hpd_rx_offload_work);
1468	offload_work->data = hpd_irq_data;
1469	offload_work->offload_wq = offload_wq;
1470	offload_work->adev = adev;
1471
1472	queue_work(offload_wq->wq, &offload_work->work);
1473	drm_dbg_kms(adev_to_drm(adev), "queue work to handle hpd_rx offload work");
1474	}
1475	EXPORT_IF_KUNIT(schedule_hpd_rx_offload_work);
1476
1477	STATIC_IFN_KUNIT void handle_hpd_rx_irq(void *param)
1478	{
1479	struct amdgpu_dm_connector aconnector = (struct amdgpu_dm_connector )param;
1480	struct drm_connector *connector = &aconnector->base;
1481	struct drm_device *dev = connector->dev;
1482	struct dc_link *dc_link = aconnector->dc_link;
1483	bool is_mst_root_connector = aconnector->mst_mgr.mst_state;
1484	bool result = false;
1485	enum dc_connection_type new_connection_type = dc_connection_none;
1486	struct amdgpu_device *adev = drm_to_adev(dev);
1487	union hpd_irq_data hpd_irq_data;
1488	bool link_loss = false;
1489	bool has_left_work = false;
1490	int idx = dc_link->link_index;
1491	struct hpd_rx_irq_offload_work_queue *offload_wq = &adev->dm.hpd_rx_offload_wq[idx];
1492	struct dc *dc = aconnector->dc_link->ctx->dc;
1493
1494	memset(&hpd_irq_data, 0, sizeof(hpd_irq_data));
1495
1496	if (adev->dm.disable_hpd_irq)
1497	return;
1498
1499	/*
1500	* TODO:Temporary add mutex to protect hpd interrupt not have a gpio
1501	* conflict, after implement i2c helper, this mutex should be
1502	* retired.
1503	*/
1504	mutex_lock(&aconnector->hpd_lock);
1505
1506	result = dc_link_handle_hpd_rx_irq(dc_link, &hpd_irq_data,
1507	&link_loss, true, &has_left_work);
1508
1509	if (!has_left_work)
1510	goto out;
1511
1512	if (hpd_irq_data.bytes.device_service_irq.bits.AUTOMATED_TEST) {
1513	schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data);
1514	goto out;
1515	}
1516
1517	if (dc_link_dp_allow_hpd_rx_irq(dc_link)) {
1518	if (hpd_irq_data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY \|\|
1519	hpd_irq_data.bytes.device_service_irq.bits.DOWN_REP_MSG_RDY) {
1520	bool skip = false;
1521
1522	/*
1523	* DOWN_REP_MSG_RDY is also handled by polling method
1524	* mgr->cbs->poll_hpd_irq()
1525	*/
1526	spin_lock(&offload_wq->offload_lock);
1527	skip = offload_wq->is_handling_mst_msg_rdy_event;
1528
1529	if (!skip)
1530	offload_wq->is_handling_mst_msg_rdy_event = true;
1531
1532	spin_unlock(&offload_wq->offload_lock);
1533
1534	if (!skip)
1535	schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data);
1536
1537	goto out;
1538	}
1539
1540	if (link_loss) {
1541	bool skip = false;
1542
1543	spin_lock(&offload_wq->offload_lock);
1544	skip = offload_wq->is_handling_link_loss;
1545
1546	if (!skip)
1547	offload_wq->is_handling_link_loss = true;
1548
1549	spin_unlock(&offload_wq->offload_lock);
1550
1551	if (!skip)
1552	schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data);
1553
1554	goto out;
1555	}
1556	}
1557
1558	out:
1559	if (result && !is_mst_root_connector) {
1560	/* Downstream Port status changed. */
1561	if (!dc_link_detect_connection_type(dc_link, &new_connection_type))
1562	drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n");
1563
1564	if (aconnector->base.force && new_connection_type == dc_connection_none) {
1565	amdgpu_dm_emulated_link_detect(dc_link);
1566
1567	if (aconnector->fake_enable)
1568	aconnector->fake_enable = false;
1569
1570	amdgpu_dm_update_connector_after_detect(aconnector);
1571
1572
1573	drm_modeset_lock_all(dev);
1574	dm_restore_drm_connector_state(dev, connector);
1575	drm_modeset_unlock_all(dev);
1576
1577	drm_kms_helper_connector_hotplug_event(connector);
1578	} else {
1579	bool ret = false;
1580
1581	mutex_lock(&adev->dm.dc_lock);
1582	dc_exit_ips_for_hw_access(dc);
1583	ret = dc_link_detect(dc_link, DETECT_REASON_HPDRX);
1584	mutex_unlock(&adev->dm.dc_lock);
1585
1586	if (ret) {
1587	if (aconnector->fake_enable)
1588	aconnector->fake_enable = false;
1589
1590	amdgpu_dm_update_connector_after_detect(aconnector);
1591
1592	drm_modeset_lock_all(dev);
1593	dm_restore_drm_connector_state(dev, connector);
1594	drm_modeset_unlock_all(dev);
1595
1596	drm_kms_helper_connector_hotplug_event(connector);
1597	}
1598	}
1599	}
1600	if (hpd_irq_data.bytes.device_service_irq.bits.CP_IRQ) {
1601	if (adev->dm.hdcp_workqueue)
1602	hdcp_handle_cpirq(adev->dm.hdcp_workqueue, aconnector->base.index);
1603	}
1604
1605	if (dc_link->type != dc_connection_mst_branch)
1606	drm_dp_cec_irq(&aconnector->dm_dp_aux.aux);
1607
1608	mutex_unlock(&aconnector->hpd_lock);
1609	}
1610	EXPORT_IF_KUNIT(handle_hpd_rx_irq);
1611
1612	/**
1613	* dmub_hpd_callback - DMUB HPD interrupt processing callback.
1614	* @adev: amdgpu_device pointer
1615	* @notify: dmub notification structure
1616	*
1617	* Dmub Hpd interrupt processing callback. Gets displayindex through the
1618	* ink index and calls helper to do the processing.
1619	*/
1620	STATIC_IFN_KUNIT void dmub_hpd_callback(struct amdgpu_device *adev,
1621	struct dmub_notification *notify)
1622	{
1623	struct amdgpu_dm_connector *aconnector;
1624	struct amdgpu_dm_connector *hpd_aconnector = NULL;
1625	struct drm_connector *connector;
1626	struct drm_connector_list_iter iter;
1627	struct dc_link *link;
1628	u8 link_index = 0;
1629	struct drm_device *dev;
1630
1631	if (adev == NULL)
1632	return;
1633
1634	if (notify == NULL) {
1635	drm_err(adev_to_drm(adev), "DMUB HPD callback notification was NULL");
1636	return;
1637	}
1638
1639	if (notify->link_index > adev->dm.dc->link_count) {
1640	drm_err(adev_to_drm(adev), "DMUB HPD index (%u)is abnormal", notify->link_index);
1641	return;
1642	}
1643
1644	/* Skip DMUB HPD IRQ in suspend/resume. We will probe them later. */
1645	if (notify->type == DMUB_NOTIFICATION_HPD && adev->in_suspend) {
1646	drm_info(adev_to_drm(adev), "Skip DMUB HPD IRQ callback in suspend/resume\n");
1647	return;
1648	}
1649
1650	link_index = notify->link_index;
1651	link = adev->dm.dc->links[link_index];
1652	dev = adev->dm.ddev;
1653
1654	drm_connector_list_iter_begin(dev, &iter);
1655	drm_for_each_connector_iter(connector, &iter) {
1656
1657	if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
1658	continue;
1659
1660	aconnector = to_amdgpu_dm_connector(connector);
1661	if (link && aconnector->dc_link == link) {
1662	if (notify->type == DMUB_NOTIFICATION_HPD)
1663	drm_info(adev_to_drm(adev), "DMUB HPD IRQ callback: link_index=%u\n", link_index);
1664	else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ)
1665	drm_info(adev_to_drm(adev), "DMUB HPD RX IRQ callback: link_index=%u\n", link_index);
1666	else
1667	drm_warn(adev_to_drm(adev), "DMUB Unknown HPD callback type %d, link_index=%u\n",
1668	notify->type, link_index);
1669
1670	hpd_aconnector = aconnector;
1671	break;
1672	}
1673	}
1674	drm_connector_list_iter_end(&iter);
1675
1676	if (hpd_aconnector) {
1677	if (notify->type == DMUB_NOTIFICATION_HPD) {
1678	if (hpd_aconnector->dc_link->hpd_status == (notify->hpd_status == DP_HPD_PLUG))
1679	drm_warn(adev_to_drm(adev), "DMUB reported hpd status unchanged. link_index=%u\n", link_index);
1680	handle_hpd_irq_helper(hpd_aconnector, DETECT_REASON_HPD);
1681	} else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ) {
1682	handle_hpd_rx_irq(hpd_aconnector);
1683	}
1684	}
1685	}
1686	EXPORT_IF_KUNIT(dmub_hpd_callback);
1687
1688	/**
1689	* dmub_hpd_sense_callback - DMUB HPD sense processing callback.
1690	* @adev: amdgpu_device pointer
1691	* @notify: dmub notification structure
1692	*
1693	* HPD sense changes can occur during low power states and need to be
1694	* notified from firmware to driver.
1695	*/
1696	STATIC_IFN_KUNIT void dmub_hpd_sense_callback(struct amdgpu_device *adev,
1697	struct dmub_notification *notify)
1698	{
1699	drm_dbg_driver(adev_to_drm(adev), "DMUB HPD SENSE callback.\n");
1700	}
1701	EXPORT_IF_KUNIT(dmub_hpd_sense_callback);
1702
1703	int amdgpu_dm_register_hpd_handlers(struct amdgpu_device *adev)
1704	{
1705	struct drm_device *dev = adev_to_drm(adev);
1706	struct drm_connector *connector;
1707	struct amdgpu_dm_connector *aconnector;
1708	const struct dc_link *dc_link;
1709	struct dc_interrupt_params int_params = {0};
1710
1711	int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
1712	int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
1713
1714	if (dc_is_dmub_outbox_supported(adev->dm.dc)) {
1715	if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD,
1716	dmub_hpd_callback, true)) {
1717	drm_err(adev_to_drm(adev), "fail to register dmub hpd callback");
1718	return -EINVAL;
1719	}
1720
1721	if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD_IRQ,
1722	dmub_hpd_callback, true)) {
1723	drm_err(adev_to_drm(adev), "fail to register dmub hpd callback");
1724	return -EINVAL;
1725	}
1726
1727	if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD_SENSE_NOTIFY,
1728	dmub_hpd_sense_callback, true)) {
1729	drm_err(adev_to_drm(adev), "fail to register dmub hpd sense callback");
1730	return -EINVAL;
1731	}
1732	}
1733
1734	list_for_each_entry(connector,
1735	&dev->mode_config.connector_list, head) {
1736
1737	if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
1738	continue;
1739
1740	aconnector = to_amdgpu_dm_connector(connector);
1741	dc_link = aconnector->dc_link;
1742
1743	if (dc_link->irq_source_hpd != DC_IRQ_SOURCE_INVALID) {
1744	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
1745	int_params.irq_source = dc_link->irq_source_hpd;
1746
1747	if (int_params.irq_source == DC_IRQ_SOURCE_INVALID \|\|
1748	int_params.irq_source < DC_IRQ_SOURCE_HPD1 \|\|
1749	int_params.irq_source > DC_IRQ_SOURCE_HPD6) {
1750	drm_err(adev_to_drm(adev),
1751	"Failed to register hpd irq %u for %s!\n",
1752	int_params.irq_source, connector->name);
1753	return -EINVAL;
1754	}
1755
1756	if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
1757	handle_hpd_irq, (void *) aconnector))
1758	return -ENOMEM;
1759	}
1760
1761	if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) {
1762
1763	/* Also register for DP short pulse (hpd_rx). */
1764	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
1765	int_params.irq_source = dc_link->irq_source_hpd_rx;
1766
1767	if (int_params.irq_source == DC_IRQ_SOURCE_INVALID \|\|
1768	int_params.irq_source < DC_IRQ_SOURCE_HPD1RX \|\|
1769	int_params.irq_source > DC_IRQ_SOURCE_HPD6RX) {
1770	drm_err(adev_to_drm(adev),
1771	"Failed to register hpd rx irq %u for %s!\n",
1772	int_params.irq_source, connector->name);
1773	return -EINVAL;
1774	}
1775
1776	if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
1777	handle_hpd_rx_irq, (void *) aconnector))
1778	return -ENOMEM;
1779	}
1780	}
1781	return 0;
1782	}
1783	EXPORT_IF_KUNIT(amdgpu_dm_register_hpd_handlers);
1784
1785	/* ========== IRQ handlers ========== */
1786	struct amdgpu_crtc *
1787	amdgpu_dm_get_crtc_by_otg_inst(struct amdgpu_device *adev,
1788	int otg_inst)
1789	{
1790	struct drm_device *dev = adev_to_drm(adev);
1791	struct drm_crtc *crtc;
1792	struct amdgpu_crtc *amdgpu_crtc;
1793
1794	if (WARN_ON(otg_inst == -1))
1795	return adev->mode_info.crtcs[0];
1796
1797	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
1798	amdgpu_crtc = to_amdgpu_crtc(crtc);
1799
1800	if (amdgpu_crtc->otg_inst == otg_inst)
1801	return amdgpu_crtc;
1802	}
1803
1804	return NULL;
1805	}
1806	EXPORT_IF_KUNIT(amdgpu_dm_get_crtc_by_otg_inst);
1807
1808	/**
1809	* dm_pflip_high_irq() - Handle pageflip interrupt
1810	* @interrupt_params: ignored
1811	*
1812	* Handles the pageflip interrupt by notifying all interested parties
1813	* that the pageflip has been completed.
1814	*/
1815	STATIC_IFN_KUNIT void dm_pflip_high_irq(void *interrupt_params)
1816	{
1817	struct amdgpu_crtc *amdgpu_crtc;
1818	struct common_irq_params *irq_params = interrupt_params;
1819	struct amdgpu_device *adev = irq_params->adev;
1820	struct drm_device *dev = adev_to_drm(adev);
1821	unsigned long flags;
1822	struct drm_pending_vblank_event *e;
1823	u32 vpos, hpos, v_blank_start, v_blank_end;
1824	bool vrr_active;
1825
1826	amdgpu_crtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_PFLIP);
1827
1828	/* IRQ could occur when in initial stage */
1829	/* TODO work and BO cleanup */
1830	if (amdgpu_crtc == NULL) {
1831	drm_dbg_state(dev, "CRTC is null, returning.\n");
1832	return;
1833	}
1834
1835	spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
1836
1837	if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED) {
1838	drm_dbg_state(dev,
1839	"amdgpu_crtc->pflip_status = %d != AMDGPU_FLIP_SUBMITTED(%d) on crtc:%d[%p]\n",
1840	amdgpu_crtc->pflip_status, AMDGPU_FLIP_SUBMITTED,
1841	amdgpu_crtc->crtc_id, amdgpu_crtc);
1842	spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
1843	return;
1844	}
1845
1846	/* page flip completed. */
1847	e = amdgpu_crtc->event;
1848	amdgpu_crtc->event = NULL;
1849
1850	WARN_ON(!e);
1851
1852	vrr_active = amdgpu_dm_crtc_vrr_active_irq(amdgpu_crtc);
1853
1854	/* Fixed refresh rate, or VRR scanout position outside front-porch? */
1855	if (!vrr_active \|\|
1856	!dc_stream_get_scanoutpos(amdgpu_crtc->dm_irq_params.stream, &v_blank_start,
1857	&v_blank_end, &hpos, &vpos) \|\|
1858	(vpos < v_blank_start)) {
1859	/* Update to correct count and vblank timestamp if racing with
1860	* vblank irq. This also updates to the correct vblank timestamp
1861	* even in VRR mode, as scanout is past the front-porch atm.
1862	*/
1863	drm_crtc_accurate_vblank_count(&amdgpu_crtc->base);
1864
1865	/* Wake up userspace by sending the pageflip event with proper
1866	* count and timestamp of vblank of flip completion.
1867	*/
1868	if (e) {
1869	drm_crtc_send_vblank_event(&amdgpu_crtc->base, e);
1870
1871	/* Event sent, so done with vblank for this flip */
1872	drm_crtc_vblank_put(&amdgpu_crtc->base);
1873	}
1874	} else if (e) {
1875	/* VRR active and inside front-porch: vblank count and
1876	* timestamp for pageflip event will only be up to date after
1877	* drm_crtc_handle_vblank() has been executed from late vblank
1878	* irq handler after start of back-porch (vline 0). We queue the
1879	* pageflip event for send-out by drm_crtc_handle_vblank() with
1880	* updated timestamp and count, once it runs after us.
1881	*
1882	* We need to open-code this instead of using the helper
1883	* drm_crtc_arm_vblank_event(), as that helper would
1884	* call drm_crtc_accurate_vblank_count(), which we must
1885	* not call in VRR mode while we are in front-porch!
1886	*/
1887
1888	/* sequence will be replaced by real count during send-out. */
1889	e->sequence = drm_crtc_vblank_count(&amdgpu_crtc->base);
1890	e->pipe = amdgpu_crtc->crtc_id;
1891
1892	list_add_tail(&e->base.link, &adev_to_drm(adev)->vblank_event_list);
1893	e = NULL;
1894	}
1895
1896	/* Keep track of vblank of this flip for flip throttling. We use the
1897	* cooked hw counter, as that one incremented at start of this vblank
1898	* of pageflip completion, so last_flip_vblank is the forbidden count
1899	* for queueing new pageflips if vsync + VRR is enabled.
1900	*/
1901	amdgpu_crtc->dm_irq_params.last_flip_vblank =
1902	amdgpu_get_vblank_counter_kms(&amdgpu_crtc->base);
1903
1904	amdgpu_crtc->pflip_status = AMDGPU_FLIP_NONE;
1905	spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
1906
1907	drm_dbg_state(dev,
1908	"crtc:%d[%p], pflip_stat:AMDGPU_FLIP_NONE, vrr[%d]-fp %d\n",
1909	amdgpu_crtc->crtc_id, amdgpu_crtc, vrr_active, (int)!e);
1910	}
1911	EXPORT_IF_KUNIT(dm_pflip_high_irq);
1912
1913	STATIC_IFN_KUNIT void dm_handle_vmin_vmax_update(struct work_struct *offload_work)
1914	{
1915	struct vupdate_offload_work *work = container_of(offload_work, struct vupdate_offload_work, work);
1916	struct amdgpu_device *adev = work->adev;
1917	struct dc_stream_state *stream = work->stream;
1918	struct dc_crtc_timing_adjust *adjust = work->adjust;
1919
1920	mutex_lock(&adev->dm.dc_lock);
1921	dc_stream_adjust_vmin_vmax(adev->dm.dc, stream, adjust);
1922	mutex_unlock(&adev->dm.dc_lock);
1923
1924	dc_stream_release(stream);
1925	kfree(work->adjust);
1926	kfree(work);
1927	}
1928	EXPORT_IF_KUNIT(dm_handle_vmin_vmax_update);
1929
1930	static void schedule_dc_vmin_vmax(struct amdgpu_device *adev,
1931	struct dc_stream_state *stream,
1932	struct dc_crtc_timing_adjust *adjust)
1933	{
1934	struct vupdate_offload_work offload_work = kzalloc_obj(offload_work,
1935	GFP_NOWAIT);
1936	if (!offload_work) {
1937	drm_dbg_driver(adev_to_drm(adev), "Failed to allocate vupdate_offload_work\n");
1938	return;
1939	}
1940
1941	struct dc_crtc_timing_adjust adjust_copy = kzalloc_obj(adjust_copy,
1942	GFP_NOWAIT);
1943	if (!adjust_copy) {
1944	drm_dbg_driver(adev_to_drm(adev), "Failed to allocate adjust_copy\n");
1945	kfree(offload_work);
1946	return;
1947	}
1948
1949	dc_stream_retain(stream);
1950	memcpy(adjust_copy, adjust, sizeof(*adjust_copy));
1951
1952	INIT_WORK(&offload_work->work, dm_handle_vmin_vmax_update);
1953	offload_work->adev = adev;
1954	offload_work->stream = stream;
1955	offload_work->adjust = adjust_copy;
1956
1957	queue_work(adev->dm.vmin_vmax_wq, &offload_work->work);
1958	}
1959
1960	/**
1961	* dm_crtc_high_irq_handler() - Common OTG vblank/flip event handling
1962	* @adev: amdgpu device
1963	* @acrtc: the CRTC to service
1964	* Performs writeback completion, vblank event handling, CRC processing, VRR BTR
1965	* updates and pageflip completion delivery.
1966	*
1967	* On DCN this is driven by VUPDATE_NO_LOCK (the register latch point) from
1968	* dm_vupdate_high_irq(); on DCE it is driven by VLINE0 at the start of vblank
1969	* from dm_crtc_high_irq().
1970	*/
1971	static void dm_crtc_high_irq_handler(struct amdgpu_device *adev,
1972	struct amdgpu_crtc *acrtc)
1973	{
1974	unsigned long flags;
1975	int vrr_active;
1976	bool is_dcn = amdgpu_ip_version(adev, DCE_HWIP, 0) != 0;
1977
1978	if (acrtc->wb_conn && acrtc->wb_pending) {
1979	if (acrtc->wb_frame_done) {
1980	/*
1981	* Second vblank: the DMA for the captured frame has
1982	* had a full frame period to flush to memory. Signal
1983	* the out fence now.
1984	*/
1985	amdgpu_dm_crtc_complete_writeback(acrtc);
1986	} else {
1987	/*
1988	* First vblank after arming: the frame has been
1989	* scanned out and the DMA is finishing. Disable
1990	* writeback immediately to prevent the hardware from
1991	* starting a new capture that would overwrite the
1992	* buffer. Signal completion on the next vblank to
1993	* ensure the DMA is fully flushed to memory.
1994	*/
1995	dc_stream_fc_disable_writeback(adev->dm.dc,
1996	acrtc->dm_irq_params.stream, 0);
1997	acrtc->wb_frame_done = true;
1998	}
1999	}
2000
2001	vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc);
2002
2003	drm_dbg_vbl(adev_to_drm(adev),
2004	"crtc:%d, vupdate-vrr:%d, planes:%d\n", acrtc->crtc_id,
2005	vrr_active, acrtc->dm_irq_params.active_planes);
2006
2007	/**
2008	* Core vblank handling.
2009	*
2010	* On DCN this handler runs at VUPDATE_NO_LOCK, the register latch
2011	* point, which is the correct place to timestamp both VRR and non-VRR
2012	* vblanks.
2013	*
2014	* On DCE this handler runs at the start of front-porch, where only
2015	* non-VRR timestamping is valid; VRR vblank is deferred to
2016	* dm_vupdate_high_irq() after end of front-porch.
2017	*/
2018	if (is_dcn \|\| !vrr_active)
2019	amdgpu_dm_crtc_handle_vblank(acrtc);
2020
2021	/**
2022	* Following stuff must happen at start of vblank, for crc
2023	* computation and below-the-range btr support in vrr mode.
2024	*/
2025	amdgpu_dm_crtc_handle_crc_irq(&acrtc->base);
2026
2027	/* BTR updates need to happen before VUPDATE on Vega and above. */
2028	if (adev->family < AMDGPU_FAMILY_AI)
2029	return;
2030
2031	spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
2032
2033	if (acrtc->dm_irq_params.stream &&
2034	acrtc->dm_irq_params.vrr_params.supported) {
2035	bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled;
2036	bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled;
2037	bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state == VRR_STATE_ACTIVE_VARIABLE;
2038
2039	mod_freesync_handle_v_update(adev->dm.freesync_module,
2040	acrtc->dm_irq_params.stream,
2041	&acrtc->dm_irq_params.vrr_params);
2042
2043	/* update vmin_vmax only if freesync is enabled, or only if PSR and REPLAY are disabled */
2044	if (fs_active_var_en \|\| (!fs_active_var_en && !replay_en && !psr_en)) {
2045	schedule_dc_vmin_vmax(adev, acrtc->dm_irq_params.stream,
2046	&acrtc->dm_irq_params.vrr_params.adjust);
2047	}
2048	}
2049
2050	/*
2051	* Deliver pageflip completion events (DCN only).
2052	*
2053	* Since GRPH_PFLIP is not used, VUPDATE_NO_LOCK is the flip latch
2054	* point. Deliver any pending pageflip completion event from here,
2055	* once HW has consumed the new address (the OTG no longer reports a
2056	* pending flip).
2057	*
2058	* Also handle the case here where there aren't any active planes and
2059	* DCN HUBP may be clock-gated, so the flip-pending status may be
2060	* undefined.
2061	*/
2062	if (is_dcn && acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED &&
2063	acrtc->event) {
2064
2065	if (!dc_get_flip_pending_on_otg(adev->dm.dc, acrtc->otg_inst)) {
2066	drm_crtc_send_vblank_event(&acrtc->base, acrtc->event);
2067	acrtc->event = NULL;
2068	drm_crtc_vblank_put(&acrtc->base);
2069	acrtc->pflip_status = AMDGPU_FLIP_NONE;
2070	}
2071	/*
2072	* If the flip is still pending, leave it armed and
2073	* retry on the next vupdate.
2074	*/
2075	} else if (is_dcn && acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED &&
2076	acrtc->dm_irq_params.active_planes == 0) {
2077
2078	if (acrtc->event) {
2079	drm_crtc_send_vblank_event(&acrtc->base, acrtc->event);
2080	acrtc->event = NULL;
2081	drm_crtc_vblank_put(&acrtc->base);
2082	}
2083	acrtc->pflip_status = AMDGPU_FLIP_NONE;
2084	}
2085
2086	spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
2087	}
2088
2089	STATIC_IFN_KUNIT void dm_vupdate_high_irq(void *interrupt_params)
2090	{
2091	struct common_irq_params *irq_params = interrupt_params;
2092	struct amdgpu_device *adev = irq_params->adev;
2093	struct amdgpu_crtc *acrtc;
2094	struct drm_device *drm_dev;
2095	struct drm_vblank_crtc *vblank;
2096	ktime_t frame_duration_ns, previous_timestamp;
2097	unsigned long flags;
2098	int vrr_active;
2099
2100	acrtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VUPDATE);
2101	if (!acrtc)
2102	return;
2103
2104	vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc);
2105	drm_dev = acrtc->base.dev;
2106	vblank = drm_crtc_vblank_crtc(&acrtc->base);
2107	previous_timestamp = atomic64_read(&irq_params->previous_timestamp);
2108	frame_duration_ns = vblank->time - previous_timestamp;
2109
2110	if (frame_duration_ns > 0) {
2111	trace_amdgpu_refresh_rate_track(acrtc->base.index,
2112	frame_duration_ns,
2113	ktime_divns(NSEC_PER_SEC, frame_duration_ns));
2114	atomic64_set(&irq_params->previous_timestamp, vblank->time);
2115	}
2116
2117	drm_dbg_vbl(drm_dev,
2118	"crtc:%d, vupdate-vrr:%d\n", acrtc->crtc_id,
2119	vrr_active);
2120
2121	/*
2122	* On DCN, VUPDATE_NO_LOCK is the single OTG interrupt used to deliver
2123	* vblank and pageflip completion events; VSTARTUP and GRPH_PFLIP are
2124	* not used. Run the full handler here.
2125	*/
2126	if (amdgpu_ip_version(adev, DCE_HWIP, 0) != 0) {
2127	dm_crtc_high_irq_handler(adev, acrtc);
2128	return;
2129	}
2130
2131	/* DCE only below. */
2132
2133	/* Core vblank handling is done here after end of front-porch in
2134	* vrr mode, as vblank timestamping will give valid results
2135	* while now done after front-porch. This will also deliver
2136	* page-flip completion events that have been queued to us
2137	* if a pageflip happened inside front-porch.
2138	*/
2139	if (vrr_active && acrtc->dm_irq_params.stream) {
2140	bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled;
2141	bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled;
2142	bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state
2143	== VRR_STATE_ACTIVE_VARIABLE;
2144
2145	amdgpu_dm_crtc_handle_vblank(acrtc);
2146
2147	/* BTR processing for pre-DCE12 ASICs */
2148	if (adev->family < AMDGPU_FAMILY_AI) {
2149	spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
2150	mod_freesync_handle_v_update(
2151	adev->dm.freesync_module,
2152	acrtc->dm_irq_params.stream,
2153	&acrtc->dm_irq_params.vrr_params);
2154
2155	if (fs_active_var_en \|\| (!fs_active_var_en && !replay_en && !psr_en)) {
2156	schedule_dc_vmin_vmax(adev,
2157	acrtc->dm_irq_params.stream,
2158	&acrtc->dm_irq_params.vrr_params.adjust);
2159	}
2160	spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
2161	}
2162	}
2163	}
2164	EXPORT_IF_KUNIT(dm_vupdate_high_irq);
2165
2166	/**
2167	* dm_crtc_high_irq() - Handles CRTC interrupt
2168	* @interrupt_params: used for determining the CRTC instance
2169	*
2170	* Handles the CRTC/VSYNC interrupt by notifying DRM's VBLANK event handler.
2171	*
2172	* Used on DCE (VLINE0, set to vblank start). On DCN the equivalent handling is
2173	* driven by VUPDATE_NO_LOCK in dm_vupdate_high_irq().
2174	*/
2175	STATIC_IFN_KUNIT void dm_crtc_high_irq(void *interrupt_params)
2176	{
2177	struct common_irq_params *irq_params = interrupt_params;
2178	struct amdgpu_device *adev = irq_params->adev;
2179	struct amdgpu_crtc *acrtc;
2180
2181	acrtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VBLANK);
2182	if (!acrtc)
2183	return;
2184
2185	dm_crtc_high_irq_handler(adev, acrtc);
2186	}
2187	EXPORT_IF_KUNIT(dm_crtc_high_irq);
2188
2189	#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
2190	/**
2191	* dm_dcn_vertical_interrupt0_high_irq() - Handles OTG Vertical interrupt0 for
2192	* DCN generation ASICs
2193	* @interrupt_params: interrupt parameters
2194	*
2195	* Used to set crc window/read out crc value at vertical line 0 position
2196	*/
2197	static void dm_dcn_vertical_interrupt0_high_irq(void *interrupt_params)
2198	{
2199	struct common_irq_params *irq_params = interrupt_params;
2200	struct amdgpu_device *adev = irq_params->adev;
2201	struct amdgpu_crtc *acrtc;
2202
2203	acrtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VLINE0);
2204
2205	if (!acrtc)
2206	return;
2207
2208	amdgpu_dm_crtc_handle_crc_window_irq(&acrtc->base);
2209	}
2210	#endif /* CONFIG_DRM_AMD_SECURE_DISPLAY */
2211
2212	STATIC_IFN_KUNIT void dm_handle_hpd_work(struct work_struct *work)
2213	{
2214	struct dmub_hpd_work *dmub_hpd_wrk;
2215
2216	dmub_hpd_wrk = container_of(work, struct dmub_hpd_work, handle_hpd_work);
2217
2218	if (!dmub_hpd_wrk->dmub_notify) {
2219	drm_err(adev_to_drm(dmub_hpd_wrk->adev), "dmub_hpd_wrk dmub_notify is NULL");
2220	return;
2221	}
2222
2223	if (dmub_hpd_wrk->dmub_notify->type < ARRAY_SIZE(dmub_hpd_wrk->adev->dm.dmub_callback)) {
2224	dmub_hpd_wrk->adev->dm.dmub_callback[dmub_hpd_wrk->dmub_notify->type](dmub_hpd_wrk->adev,
2225	dmub_hpd_wrk->dmub_notify);
2226	}
2227
2228	kfree(dmub_hpd_wrk->dmub_notify);
2229	kfree(dmub_hpd_wrk);
2230
2231	}
2232	EXPORT_IF_KUNIT(dm_handle_hpd_work);
2233
2234	STATIC_IFN_KUNIT const char *dmub_notification_type_str(enum dmub_notification_type e)
2235	{
2236	switch (e) {
2237	case DMUB_NOTIFICATION_NO_DATA:
2238	return "NO_DATA";
2239	case DMUB_NOTIFICATION_AUX_REPLY:
2240	return "AUX_REPLY";
2241	case DMUB_NOTIFICATION_HPD:
2242	return "HPD";
2243	case DMUB_NOTIFICATION_HPD_IRQ:
2244	return "HPD_IRQ";
2245	case DMUB_NOTIFICATION_SET_CONFIG_REPLY:
2246	return "SET_CONFIG_REPLY";
2247	case DMUB_NOTIFICATION_DPIA_NOTIFICATION:
2248	return "DPIA_NOTIFICATION";
2249	case DMUB_NOTIFICATION_HPD_SENSE_NOTIFY:
2250	return "HPD_SENSE_NOTIFY";
2251	case DMUB_NOTIFICATION_FUSED_IO:
2252	return "FUSED_IO";
2253	default:
2254	return "<unknown>";
2255	}
2256	}
2257	EXPORT_IF_KUNIT(dmub_notification_type_str);
2258
2259	#define DMUB_TRACE_MAX_READ 64
2260	/**
2261	* dm_dmub_outbox1_low_irq() - Handles Outbox interrupt
2262	* @interrupt_params: used for determining the Outbox instance
2263	*
2264	* Handles the Outbox Interrupt
2265	* event handler.
2266	*/
2267	STATIC_IFN_KUNIT void dm_dmub_outbox1_low_irq(void *interrupt_params)
2268	{
2269	struct dmub_notification notify = {0};
2270	struct common_irq_params *irq_params = interrupt_params;
2271	struct amdgpu_device *adev = irq_params->adev;
2272	struct amdgpu_display_manager *dm = &adev->dm;
2273	struct dmcub_trace_buf_entry entry = { 0 };
2274	u32 count = 0;
2275	struct dmub_hpd_work *dmub_hpd_wrk;
2276
2277	do {
2278	if (dc_dmub_srv_get_dmub_outbox0_msg(dm->dc, &entry)) {
2279	trace_amdgpu_dmub_trace_high_irq(entry.trace_code, entry.tick_count,
2280	entry.param0, entry.param1);
2281
2282	drm_dbg_driver(adev_to_drm(adev), "trace_code:%u, tick_count:%u, param0:%u, param1:%u\n",
2283	entry.trace_code, entry.tick_count, entry.param0, entry.param1);
2284	} else
2285	break;
2286
2287	count++;
2288
2289	} while (count <= DMUB_TRACE_MAX_READ);
2290
2291	if (count > DMUB_TRACE_MAX_READ)
2292	drm_dbg_driver(adev_to_drm(adev), "Warning : count > DMUB_TRACE_MAX_READ");
2293
2294	if (dc_enable_dmub_notifications(adev->dm.dc) &&
2295	irq_params->irq_src == DC_IRQ_SOURCE_DMCUB_OUTBOX) {
2296
2297	do {
2298	dc_stat_get_dmub_notification(adev->dm.dc, &notify);
2299	if (notify.type >= ARRAY_SIZE(dm->dmub_thread_offload)) {
2300	drm_err(adev_to_drm(adev), "DM: notify type %d invalid!", notify.type);
2301	continue;
2302	}
2303	if (!dm->dmub_callback[notify.type]) {
2304	drm_warn(adev_to_drm(adev), "DMUB notification skipped due to no handler: type=%s\n",
2305	dmub_notification_type_str(notify.type));
2306	continue;
2307	}
2308	if (dm->dmub_thread_offload[notify.type]) {
2309	dmub_hpd_wrk = kzalloc_obj(*dmub_hpd_wrk,
2310	GFP_ATOMIC);
2311	if (!dmub_hpd_wrk) {
2312	drm_err(adev_to_drm(adev), "Failed to allocate dmub_hpd_wrk");
2313	return;
2314	}
2315	dmub_hpd_wrk->dmub_notify = kmemdup(&notify, sizeof(struct dmub_notification),
2316	GFP_ATOMIC);
2317	if (!dmub_hpd_wrk->dmub_notify) {
2318	kfree(dmub_hpd_wrk);
2319	drm_err(adev_to_drm(adev), "Failed to allocate dmub_hpd_wrk->dmub_notify");
2320	return;
2321	}
2322	INIT_WORK(&dmub_hpd_wrk->handle_hpd_work, dm_handle_hpd_work);
2323	dmub_hpd_wrk->adev = adev;
2324	queue_work(adev->dm.delayed_hpd_wq, &dmub_hpd_wrk->handle_hpd_work);
2325	} else {
2326	dm->dmub_callback[notify.type](adev, &notify);
2327	}
2328	} while (notify.pending_notification);
2329	}
2330	}
2331	EXPORT_IF_KUNIT(dm_dmub_outbox1_low_irq);
2332
2333	/* Register IRQ sources and initialize IRQ callbacks */
2334	int amdgpu_dm_dce110_register_irq_handlers(struct amdgpu_device *adev)
2335	{
2336	struct dc *dc = adev->dm.dc;
2337	struct common_irq_params *c_irq_params;
2338	struct dc_interrupt_params int_params = {0};
2339	int r;
2340	int i;
2341	unsigned int src_id;
2342	unsigned int client_id = AMDGPU_IRQ_CLIENTID_LEGACY;
2343	/* Use different interrupts for VBLANK on DCE 6 vs. newer. */
2344	const unsigned int vblank_d1 =
2345	adev->dm.dc->ctx->dce_version >= DCE_VERSION_8_0
2346	? VISLANDS30_IV_SRCID_D1_VERTICAL_INTERRUPT0 : 1;
2347
2348	if (adev->family >= AMDGPU_FAMILY_AI)
2349	client_id = SOC15_IH_CLIENTID_DCE;
2350
2351	int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
2352	int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
2353
2354	/*
2355	* Actions of amdgpu_irq_add_id():
2356	* 1. Register a set() function with base driver.
2357	* Base driver will call set() function to enable/disable an
2358	* interrupt in DC hardware.
2359	* 2. Register amdgpu_dm_irq_handler().
2360	* Base driver will call amdgpu_dm_irq_handler() for ALL interrupts
2361	* coming from DC hardware.
2362	* amdgpu_dm_irq_handler() will re-direct the interrupt to DC
2363	* for acknowledging and handling.
2364	*/
2365
2366	/* Use VBLANK interrupt */
2367	for (i = 0; i < adev->mode_info.num_crtc; i++) {
2368	src_id = vblank_d1 + i;
2369	r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->crtc_irq);
2370	if (r) {
2371	drm_err(adev_to_drm(adev), "Failed to add crtc irq id!\n");
2372	return r;
2373	}
2374
2375	int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
2376	int_params.irq_source =
2377	dc_interrupt_to_irq_source(dc, src_id, 0);
2378
2379	if (int_params.irq_source == DC_IRQ_SOURCE_INVALID \|\|
2380	int_params.irq_source < DC_IRQ_SOURCE_VBLANK1 \|\|
2381	int_params.irq_source > DC_IRQ_SOURCE_VBLANK6) {
2382	drm_err(adev_to_drm(adev), "Failed to register vblank irq!\n");
2383	return -EINVAL;
2384	}
2385
2386	c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1];
2387
2388	c_irq_params->adev = adev;
2389	c_irq_params->irq_src = int_params.irq_source;
2390
2391	if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
2392	dm_crtc_high_irq, c_irq_params))
2393	return -ENOMEM;
2394	}
2395
2396	if (dc_supports_vrr(adev->dm.dc->ctx->dce_version)) {
2397	/* Use VUPDATE interrupt */
2398	for (i = 0; i < adev->mode_info.num_crtc; i++) {
2399	src_id = VISLANDS30_IV_SRCID_D1_V_UPDATE_INT + i * 2;
2400	r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->vupdate_irq);
2401	if (r) {
2402	drm_err(adev_to_drm(adev), "Failed to add vupdate irq id!\n");
2403	return r;
2404	}
2405
2406	int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
2407	int_params.irq_source =
2408	dc_interrupt_to_irq_source(dc, src_id, 0);
2409
2410	if (int_params.irq_source == DC_IRQ_SOURCE_INVALID \|\|
2411	int_params.irq_source < DC_IRQ_SOURCE_VUPDATE1 \|\|
2412	int_params.irq_source > DC_IRQ_SOURCE_VUPDATE6) {
2413	drm_err(adev_to_drm(adev), "Failed to register vupdate irq!\n");
2414	return -EINVAL;
2415	}
2416
2417	c_irq_params = &adev->dm.vupdate_params[
2418	int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1];
2419	c_irq_params->adev = adev;
2420	c_irq_params->irq_src = int_params.irq_source;
2421
2422	if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
2423	dm_vupdate_high_irq, c_irq_params))
2424	return -ENOMEM;
2425	}
2426	}
2427
2428	/* Use GRPH_PFLIP interrupt */
2429	for (i = VISLANDS30_IV_SRCID_D1_GRPH_PFLIP;
2430	i <= VISLANDS30_IV_SRCID_D6_GRPH_PFLIP; i += 2) {
2431	r = amdgpu_irq_add_id(adev, client_id, i, &adev->pageflip_irq);
2432	if (r) {
2433	drm_err(adev_to_drm(adev), "Failed to add page flip irq id!\n");
2434	return r;
2435	}
2436
2437	int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
2438	int_params.irq_source =
2439	dc_interrupt_to_irq_source(dc, i, 0);
2440
2441	if (int_params.irq_source == DC_IRQ_SOURCE_INVALID \|\|
2442	int_params.irq_source < DC_IRQ_SOURCE_PFLIP_FIRST \|\|
2443	int_params.irq_source > DC_IRQ_SOURCE_PFLIP_LAST) {
2444	drm_err(adev_to_drm(adev), "Failed to register pflip irq!\n");
2445	return -EINVAL;
2446	}
2447
2448	c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST];
2449
2450	c_irq_params->adev = adev;
2451	c_irq_params->irq_src = int_params.irq_source;
2452
2453	if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
2454	dm_pflip_high_irq, c_irq_params))
2455	return -ENOMEM;
2456	}
2457
2458	/* HPD */
2459	r = amdgpu_irq_add_id(adev, client_id,
2460	VISLANDS30_IV_SRCID_HOTPLUG_DETECT_A, &adev->hpd_irq);
2461	if (r) {
2462	drm_err(adev_to_drm(adev), "Failed to add hpd irq id!\n");
2463	return r;
2464	}
2465
2466	r = amdgpu_dm_register_hpd_handlers(adev);
2467
2468	return r;
2469	}
2470	EXPORT_IF_KUNIT(amdgpu_dm_dce110_register_irq_handlers);
2471
2472	/* Register IRQ sources and initialize IRQ callbacks */
2473	int amdgpu_dm_dcn10_register_irq_handlers(struct amdgpu_device *adev)
2474	{
2475	struct dc *dc = adev->dm.dc;
2476	struct common_irq_params *c_irq_params;
2477	struct dc_interrupt_params int_params = {0};
2478	int r;
2479	int i;
2480	#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
2481	static const unsigned int vrtl_int_srcid[] = {
2482	DCN_1_0__SRCID__OTG1_VERTICAL_INTERRUPT0_CONTROL,
2483	DCN_1_0__SRCID__OTG2_VERTICAL_INTERRUPT0_CONTROL,
2484	DCN_1_0__SRCID__OTG3_VERTICAL_INTERRUPT0_CONTROL,
2485	DCN_1_0__SRCID__OTG4_VERTICAL_INTERRUPT0_CONTROL,
2486	DCN_1_0__SRCID__OTG5_VERTICAL_INTERRUPT0_CONTROL,
2487	DCN_1_0__SRCID__OTG6_VERTICAL_INTERRUPT0_CONTROL
2488	};
2489	#endif
2490
2491	int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
2492	int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
2493
2494	/*
2495	* Actions of amdgpu_irq_add_id():
2496	* 1. Register a set() function with base driver.
2497	* Base driver will call set() function to enable/disable an
2498	* interrupt in DC hardware.
2499	* 2. Register amdgpu_dm_irq_handler().
2500	* Base driver will call amdgpu_dm_irq_handler() for ALL interrupts
2501	* coming from DC hardware.
2502	* amdgpu_dm_irq_handler() will re-direct the interrupt to DC
2503	* for acknowledging and handling.
2504	*/
2505
2506	/* Use otg vertical line interrupt */
2507	#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
2508	for (i = 0; i <= adev->mode_info.num_crtc - 1; i++) {
2509	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE,
2510	vrtl_int_srcid[i], &adev->vline0_irq);
2511
2512	if (r) {
2513	drm_err(adev_to_drm(adev), "Failed to add vline0 irq id!\n");
2514	return r;
2515	}
2516
2517	int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
2518	int_params.irq_source =
2519	dc_interrupt_to_irq_source(dc, vrtl_int_srcid[i], 0);
2520
2521	if (int_params.irq_source == DC_IRQ_SOURCE_INVALID \|\|
2522	int_params.irq_source < DC_IRQ_SOURCE_DC1_VLINE0 \|\|
2523	int_params.irq_source > DC_IRQ_SOURCE_DC6_VLINE0) {
2524	drm_err(adev_to_drm(adev), "Failed to register vline0 irq!\n");
2525	return -EINVAL;
2526	}
2527
2528	c_irq_params = &adev->dm.vline0_params[int_params.irq_source
2529	- DC_IRQ_SOURCE_DC1_VLINE0];
2530
2531	c_irq_params->adev = adev;
2532	c_irq_params->irq_src = int_params.irq_source;
2533
2534	if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
2535	dm_dcn_vertical_interrupt0_high_irq,
2536	c_irq_params))
2537	return -ENOMEM;
2538	}
2539	#endif
2540
2541	/* Use VUPDATE_NO_LOCK interrupt on DCN, which seems to correspond to
2542	* the regular VUPDATE interrupt on DCE. We want DC_IRQ_SOURCE_VUPDATEx
2543	* to trigger at end of each vblank, regardless of state of the lock,
2544	* matching DCE behaviour.
2545	*/
2546	for (i = DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT;
2547	i <= DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT + adev->mode_info.num_crtc - 1;
2548	i++) {
2549	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->vupdate_irq);
2550
2551	if (r) {
2552	drm_err(adev_to_drm(adev), "Failed to add vupdate irq id!\n");
2553	return r;
2554	}
2555
2556	int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
2557	int_params.irq_source =
2558	dc_interrupt_to_irq_source(dc, i, 0);
2559
2560	if (int_params.irq_source == DC_IRQ_SOURCE_INVALID \|\|
2561	int_params.irq_source < DC_IRQ_SOURCE_VUPDATE1 \|\|
2562	int_params.irq_source > DC_IRQ_SOURCE_VUPDATE6) {
2563	drm_err(adev_to_drm(adev), "Failed to register vupdate irq!\n");
2564	return -EINVAL;
2565	}
2566
2567	c_irq_params = &adev->dm.vupdate_params[int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1];
2568
2569	c_irq_params->adev = adev;
2570	c_irq_params->irq_src = int_params.irq_source;
2571
2572	if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
2573	dm_vupdate_high_irq, c_irq_params))
2574	return -ENOMEM;
2575	}
2576
2577	/* HPD */
2578	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DC_HPD1_INT,
2579	&adev->hpd_irq);
2580	if (r) {
2581	drm_err(adev_to_drm(adev), "Failed to add hpd irq id!\n");
2582	return r;
2583	}
2584
2585	r = amdgpu_dm_register_hpd_handlers(adev);
2586
2587	return r;
2588	}
2589	EXPORT_IF_KUNIT(amdgpu_dm_dcn10_register_irq_handlers);
2590
2591	/* Register Outbox IRQ sources and initialize IRQ callbacks */
2592	int amdgpu_dm_register_outbox_irq_handlers(struct amdgpu_device *adev)
2593	{
2594	struct dc *dc = adev->dm.dc;
2595	struct common_irq_params *c_irq_params;
2596	struct dc_interrupt_params int_params = {0};
2597	int r, i;
2598
2599	int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
2600	int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
2601
2602	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT,
2603	&adev->dmub_outbox_irq);
2604	if (r) {
2605	drm_err(adev_to_drm(adev), "Failed to add outbox irq id!\n");
2606	return r;
2607	}
2608
2609	if (dc->ctx->dmub_srv) {
2610	i = DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT;
2611	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
2612	int_params.irq_source =
2613	dc_interrupt_to_irq_source(dc, i, 0);
2614
2615	c_irq_params = &adev->dm.dmub_outbox_params[0];
2616
2617	c_irq_params->adev = adev;
2618	c_irq_params->irq_src = int_params.irq_source;
2619
2620	if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
2621	dm_dmub_outbox1_low_irq, c_irq_params))
2622	return -ENOMEM;
2623	}
2624
2625	return 0;
2626	}
2627	EXPORT_IF_KUNIT(amdgpu_dm_register_outbox_irq_handlers);