projects/linux·blob master · refs

416 lines · 12.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3use core::ops::Range;
4
5use kernel::{
6 device,
7 dma::Device,
8 fmt,
9 io::Io,
10 num::Bounded,
11 pci,
12 prelude::*,
13 sizes::SizeConstants, //
14};
15
16use crate::{
17 bounded_enum,
18 driver::Bar0,
19 falcon::{
20 gsp::Gsp as GspFalcon,
21 sec2::Sec2 as Sec2Falcon,
22 Falcon, //
23 },
24 fb::SysmemFlush,
25 fsp::Fsp,
26 gsp::{
27 self,
28 commands::GetGspStaticInfoReply,
29 Gsp,
30 GspBootContext, //
31 },
32 regs,
33 vgpu::VgpuManager, //
34};
35
36mod hal;
37
38macro_rules! define_chipset {
39 ({ $($variant:ident = $value:expr),* $(,)* }) =>
40 {
41 /// Enum representation of the GPU chipset.
42 #[derive(fmt::Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq)]
43 pub(crate) enum Chipset {
44 $($variant = $value),*,
45 }
46
47 impl Chipset {
48 pub(crate) const ALL: &'static [Chipset] = &[
49 $( Chipset::$variant, )*
50 ];
51
52 ::kernel::macros::paste!(
53 /// Returns the name of this chipset, in lowercase.
54 ///
55 /// # Examples
56 ///
57 /// ```
58 /// let chipset = Chipset::GA102;
59 /// assert_eq!(chipset.name(), "ga102");
60 /// ```
61 pub(crate) const fn name(&self) -> &'static str {
62 match *self {
63 $(
64 Chipset::$variant => stringify!([<$variant:lower>]),
65 )*
66 }
67 }
68 );
69 }
70
71 // TODO[FPRI]: replace with something like derive(FromPrimitive)
72 impl TryFrom<u32> for Chipset {
73 type Error = kernel::error::Error;
74
75 fn try_from(value: u32) -> Result<Self, Self::Error> {
76 match value {
77 $( $value => Ok(Chipset::$variant), )*
78 _ => Err(ENODEV),
79 }
80 }
81 }
82 }
83}
84
85define_chipset!({
86 // Turing
87 TU102 = 0x162,
88 TU104 = 0x164,
89 TU106 = 0x166,
90 TU117 = 0x167,
91 TU116 = 0x168,
92 // Ampere
93 GA100 = 0x170,
94 GA102 = 0x172,
95 GA103 = 0x173,
96 GA104 = 0x174,
97 GA106 = 0x176,
98 GA107 = 0x177,
99 // Hopper
100 GH100 = 0x180,
101 // Ada
102 AD102 = 0x192,
103 AD103 = 0x193,
104 AD104 = 0x194,
105 AD106 = 0x196,
106 AD107 = 0x197,
107 // Blackwell GB10x
108 GB100 = 0x1a0,
109 GB102 = 0x1a2,
110 // Blackwell GB20x
111 GB202 = 0x1b2,
112 GB203 = 0x1b3,
113 GB205 = 0x1b5,
114 GB206 = 0x1b6,
115 GB207 = 0x1b7,
116});
117
118impl Chipset {
119 pub(crate) const fn arch(self) -> Architecture {
120 match self {
121 Self::TU102 | Self::TU104 | Self::TU106 | Self::TU117 | Self::TU116 => {
122 Architecture::Turing
123 }
124 Self::GA100 | Self::GA102 | Self::GA103 | Self::GA104 | Self::GA106 | Self::GA107 => {
125 Architecture::Ampere
126 }
127 Self::GH100 => Architecture::Hopper,
128 Self::AD102 | Self::AD103 | Self::AD104 | Self::AD106 | Self::AD107 => {
129 Architecture::Ada
130 }
131 Self::GB100 | Self::GB102 => Architecture::BlackwellGB10x,
132 Self::GB202 | Self::GB203 | Self::GB205 | Self::GB206 | Self::GB207 => {
133 Architecture::BlackwellGB20x
134 }
135 }
136 }
137
138 /// Returns the address range of the PCI config mirror space.
139 pub(crate) fn pci_config_mirror_range(self) -> Range<u32> {
140 hal::gpu_hal(self).pci_config_mirror_range()
141 }
142}
143
144// TODO
145//
146// The resulting strings are used to generate firmware paths, hence the
147// generated strings have to be stable.
148//
149// Hence, replace with something like strum_macros derive(Display).
150//
151// For now, redirect to fmt::Debug for convenience.
152impl fmt::Display for Chipset {
153 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
154 write!(f, "{self:?}")
155 }
156}
157
158bounded_enum! {
159 /// Enum representation of the GPU generation.
160 #[derive(fmt::Debug, Copy, Clone)]
161 pub(crate) enum Architecture with TryFrom<Bounded<u32, 6>> {
162 Turing = 0x16,
163 Ampere = 0x17,
164 Hopper = 0x18,
165 Ada = 0x19,
166 BlackwellGB10x = 0x1a,
167 BlackwellGB20x = 0x1b,
168 }
169}
170
171#[derive(Clone, Copy)]
172pub(crate) struct Revision {
173 major: Bounded<u8, 4>,
174 minor: Bounded<u8, 4>,
175}
176
177impl From<regs::NV_PMC_BOOT_42> for Revision {
178 fn from(boot0: regs::NV_PMC_BOOT_42) -> Self {
179 Self {
180 major: boot0.major_revision().cast(),
181 minor: boot0.minor_revision().cast(),
182 }
183 }
184}
185
186impl fmt::Display for Revision {
187 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
188 write!(f, "{:x}.{:x}", self.major, self.minor)
189 }
190}
191
192/// Structure holding a basic description of the GPU: `Chipset` and `Revision`.
193#[derive(Clone, Copy)]
194pub(crate) struct Spec {
195 chipset: Chipset,
196 revision: Revision,
197}
198
199impl Spec {
200 fn new(dev: &device::Device, bar: Bar0<'_>) -> Result<Spec> {
201 // Some brief notes about boot0 and boot42, in chronological order:
202 //
203 // NV04 through NV50:
204 //
205 // Not supported by Nova. boot0 is necessary and sufficient to identify these GPUs.
206 // boot42 may not even exist on some of these GPUs.
207 //
208 // Fermi through Volta:
209 //
210 // Not supported by Nova. boot0 is still sufficient to identify these GPUs, but boot42
211 // is also guaranteed to be both present and accurate.
212 //
213 // Turing and later:
214 //
215 // Supported by Nova. Identified by first checking boot0 to ensure that the GPU is not
216 // from an earlier (pre-Fermi) era, and then using boot42 to precisely identify the GPU.
217 // Somewhere in the Rubin timeframe, boot0 will no longer have space to add new GPU IDs.
218
219 let boot0 = bar.read(regs::NV_PMC_BOOT_0);
220
221 if boot0.is_older_than_fermi() {
222 return Err(ENODEV);
223 }
224
225 let boot42 = bar.read(regs::NV_PMC_BOOT_42);
226 Spec::try_from(boot42).inspect_err(|_| {
227 dev_err!(dev, "Unsupported chipset: {}\n", boot42);
228 })
229 }
230}
231
232impl TryFrom<regs::NV_PMC_BOOT_42> for Spec {
233 type Error = Error;
234
235 fn try_from(boot42: regs::NV_PMC_BOOT_42) -> Result<Self> {
236 Ok(Self {
237 chipset: boot42.chipset()?,
238 revision: boot42.into(),
239 })
240 }
241}
242
243impl fmt::Display for Spec {
244 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
245 f.write_fmt(fmt!(
246 "Chipset: {}, Architecture: {:?}, Revision: {}",
247 self.chipset,
248 self.chipset.arch(),
249 self.revision
250 ))
251 }
252}
253
254/// Self-contained resources to operate and drop the GSP.
255#[pin_data(PinnedDrop)]
256struct GspResources<'gpu> {
257 /// Device owning the GPU.
258 device: &'gpu pci::Device<device::Bound>,
259 /// Details about the chipset.
260 spec: Spec,
261 /// MMIO mapping of PCI BAR 0.
262 bar: Bar0<'gpu>,
263 /// GSP falcon instance, used for GSP boot up and cleanup.
264 gsp_falcon: Falcon<'gpu, GspFalcon>,
265 /// SEC2 falcon instance, used for GSP boot up and cleanup.
266 sec2_falcon: Falcon<'gpu, Sec2Falcon>,
267 /// FSP instance, if on an arch that supports it.
268 // TODO: use different resource types for each boot method, and make the relevant Gsp methods
269 // generic against them.
270 fsp: Option<Fsp<'gpu>>,
271 /// vGPU state detected before GSP boot.
272 vgpu: VgpuManager,
273 /// GSP runtime data.
274 #[pin]
275 gsp: Gsp,
276 /// GSP unload firmware bundle, if any.
277 unload_bundle: Option<gsp::UnloadBundle>,
278}
279
280/// Structure holding the resources required to operate the GPU.
281#[pin_data]
282pub(crate) struct Gpu<'gpu> {
283 spec: Spec,
284 /// Static GPU information as provided by the GSP.
285 gsp_static_info: GetGspStaticInfoReply,
286 /// GSP and its resources.
287 #[pin]
288 gsp_resources: GspResources<'gpu>,
289 /// System memory page required for flushing all pending GPU-side memory writes done through
290 /// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation).
291 ///
292 /// Must be kept declared *after* `gsp_resources`, as the latter's `PinnedDrop` implementation
293 /// requires the sysmem flush page to be in place.
294 sysmem_flush: SysmemFlush<'gpu>,
295}
296
297#[pinned_drop]
298impl PinnedDrop for GspResources<'_> {
299 fn drop(self: Pin<&mut Self>) {
300 let this = self.project();
301 let device = *this.device;
302 let bar = *this.bar;
303 let bundle = this.unload_bundle.take();
304
305 let _ = this
306 .gsp
307 .as_ref()
308 .get_ref()
309 .unload(
310 GspBootContext {
311 pdev: device,
312 bar,
313 chipset: this.spec.chipset,
314 gsp_falcon: &*this.gsp_falcon,
315 sec2_falcon: &*this.sec2_falcon,
316 fsp: this.fsp.as_mut(),
317 vgpu: &*this.vgpu,
318 },
319 bundle,
320 )
321 .inspect_err(|e| dev_err!(device, "failed to unload GSP: {:?}\n", e));
322 }
323}
324
325impl<'gpu> Gpu<'gpu> {
326 pub(crate) fn new<'a>(
327 pdev: &'gpu pci::Device<device::Core<'a>>,
328 bar: Bar0<'gpu>,
329 ) -> impl PinInit<Self, Error> + use<'gpu, 'a> {
330 let dev = pdev.as_ref();
331
332 try_pin_init!(Self {
333 spec: Spec::new(dev, bar).inspect(|spec| {
334 dev_info!(dev,"NVIDIA ({})\n", spec);
335 })?,
336
337 // We must wait for GFW_BOOT completion before doing any significant setup on the GPU.
338 _: {
339 let hal = hal::gpu_hal(spec.chipset);
340 let dma_mask = hal.dma_mask();
341
342 // SAFETY: `Gpu` owns all DMA allocations for this device, and we are
343 // still constructing it, so no concurrent DMA allocations can exist.
344 unsafe { pdev.dma_set_mask_and_coherent(dma_mask)? };
345
346 hal.wait_gfw_boot_completion(bar)
347 .inspect_err(|_| dev_err!(dev, "GFW boot did not complete\n"))?;
348 },
349
350 // Initialize this early because `gsp_resources` depends on it.
351 sysmem_flush: SysmemFlush::register(dev, bar, spec.chipset)?,
352
353 gsp_resources <- try_pin_init!(GspResources {
354 device: pdev,
355
356 spec: *spec,
357
358 bar,
359
360 gsp_falcon: Falcon::new(
361 dev,
362 spec.chipset,
363 bar
364 )
365 .inspect(|falcon| falcon.clear_swgen0_intr())?,
366
367 sec2_falcon: Falcon::new(dev, spec.chipset, bar)?,
368
369 fsp: Fsp::try_new(dev, bar, spec.chipset)?,
370
371 vgpu: VgpuManager::new(pdev, spec.chipset, fsp.as_mut()),
372
373 gsp <- Gsp::new(pdev),
374
375 // This member must be initialized last, so the `UnloadBundle` can never be dropped
376 // from outside of the constructed `GspResources`, ensuring that the unload sequence
377 // is properly run in case of failure.
378 unload_bundle: gsp.boot(GspBootContext {
379 pdev,
380 bar,
381 chipset: spec.chipset,
382 gsp_falcon,
383 sec2_falcon,
384 fsp: fsp.as_mut(),
385 vgpu,
386 })?,
387 }),
388
389 gsp_static_info: {
390 // Obtain and display basic GPU information.
391 let info = gsp_resources.gsp.get_static_info(bar)?;
392 match info.gpu_name() {
393 Ok(name) => dev_info!(dev, "GPU name: {}\n", name),
394 Err(e) => dev_warn!(dev, "GPU name unavailable: {:?}\n", e),
395 }
396
397 if !info.usable_fb_regions.is_empty() {
398 dev_dbg!(dev, "Usable FB regions:\n");
399 for region in &info.usable_fb_regions {
400 dev_dbg!(dev, " - {:#x?}\n", region);
401 }
402
403 dev_dbg!(
404 dev,
405 "Total usable VRAM: {} MiB\n",
406 info.usable_fb_regions.iter().fold(0u64, |res, region| res
407 .saturating_add(region.end - region.start))
408 / u64::SZ_1M
409 );
410 }
411
412 info
413 }
414 })
415 }
416}