projects/linux·blob master · refs

858 lines · 31.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3mod continuation;
4
5use core::{
6 mem,
7 sync::atomic::{
8 fence,
9 Ordering, //
10 },
11};
12
13use kernel::{
14 device,
15 dma::{
16 Coherent,
17 CoherentBox,
18 DmaAddress, //
19 },
20 io::{
21 io_project,
22 poll::read_poll_timeout,
23 Io, //
24 },
25 new_mutex,
26 prelude::*,
27 ptr,
28 sync::{
29 aref::ARef,
30 Mutex, //
31 },
32 time::Delta,
33 transmute::{
34 AsBytes,
35 FromBytes, //
36 },
37};
38
39use continuation::{
40 ContinuationRecord,
41 SplitState, //
42};
43
44use pin_init::pin_init_scope;
45
46use crate::{
47 driver::Bar0,
48 gsp::{
49 fw::{
50 GspMsgElement,
51 MsgFunction,
52 MsgqRxHeader,
53 MsgqTxHeader,
54 GSP_MSG_QUEUE_ELEMENT_SIZE_MAX, //
55 },
56 PteArray,
57 GSP_PAGE_SHIFT,
58 GSP_PAGE_SIZE, //
59 },
60 num,
61 sbuffer::SBufferIter, //
62};
63
64use super::regs;
65
66/// Marker type representing the absence of a reply for a command. Commands using this as their
67/// reply type are sent using [`Cmdq::send_command_no_wait`].
68pub(crate) struct NoReply;
69
70/// Trait implemented by types representing a command to send to the GSP.
71///
72/// The main purpose of this trait is to provide [`Cmdq`] with the information it needs to send
73/// a given command.
74///
75/// [`CommandToGsp::init`] in particular is responsible for initializing the command directly
76/// into the space reserved for it in the command queue buffer.
77///
78/// Some commands may be followed by a variable-length payload. For these, the
79/// [`CommandToGsp::variable_payload_len`] and [`CommandToGsp::init_variable_payload`] need to be
80/// defined as well.
81pub(crate) trait CommandToGsp {
82 /// Function identifying this command to the GSP.
83 const FUNCTION: MsgFunction;
84
85 /// Type generated by [`CommandToGsp::init`], to be written into the command queue buffer.
86 type Command: FromBytes + AsBytes;
87
88 /// Type of the reply expected from the GSP, or [`NoReply`] for commands that don't
89 /// have a reply.
90 type Reply;
91
92 /// Error type returned by [`CommandToGsp::init`].
93 type InitError;
94
95 /// In-place command initializer responsible for filling the command in the command queue
96 /// buffer.
97 fn init(&self) -> impl Init<Self::Command, Self::InitError>;
98
99 /// Size of the variable-length payload following the command structure generated by
100 /// [`CommandToGsp::init`].
101 ///
102 /// Most commands don't have a variable-length payload, so this is zero by default.
103 fn variable_payload_len(&self) -> usize {
104 0
105 }
106
107 /// Method initializing the variable-length payload.
108 ///
109 /// The command buffer is circular, which means that we may need to jump back to its beginning
110 /// while in the middle of a command. For this reason, the variable-length payload is
111 /// initialized using a [`SBufferIter`].
112 ///
113 /// This method will receive a buffer of the length returned by
114 /// [`CommandToGsp::variable_payload_len`], and must write every single byte of it. Leaving
115 /// unwritten space will lead to an error.
116 ///
117 /// Most commands don't have a variable-length payload, so this does nothing by default.
118 fn init_variable_payload(
119 &self,
120 _dst: &mut SBufferIter<core::array::IntoIter<&mut [u8], 2>>,
121 ) -> Result {
122 Ok(())
123 }
124
125 /// Total size of the command (including its variable-length payload) without the
126 /// [`GspMsgElement`] header.
127 fn size(&self) -> usize {
128 size_of::<Self::Command>() + self.variable_payload_len()
129 }
130}
131
132/// Trait representing messages received from the GSP.
133///
134/// This trait tells [`Cmdq::receive_msg`] how it can receive a given type of message.
135pub(crate) trait MessageFromGsp: Sized {
136 /// Function identifying this message from the GSP.
137 const FUNCTION: MsgFunction;
138
139 /// Error type returned by [`MessageFromGsp::read`].
140 type InitError;
141
142 /// Type containing the raw message to be read from the message queue.
143 type Message: FromBytes;
144
145 /// Method reading the message from the message queue and returning it.
146 ///
147 /// From a `Self::Message` and a [`SBufferIter`], constructs an instance of `Self` and returns
148 /// it.
149 fn read(
150 msg: &Self::Message,
151 sbuffer: &mut SBufferIter<core::array::IntoIter<&[u8], 2>>,
152 ) -> Result<Self, Self::InitError>;
153}
154
155/// Number of GSP pages making the [`Msgq`].
156pub(crate) const MSGQ_NUM_PAGES: u32 = 0x3f;
157
158/// Circular buffer of a [`Msgq`].
159///
160/// This area of memory is to be shared between the driver and the GSP to exchange commands or
161/// messages.
162#[repr(C, align(0x1000))]
163#[derive(Debug)]
164struct MsgqData {
165 data: [[u8; GSP_PAGE_SIZE]; num::u32_as_usize(MSGQ_NUM_PAGES)],
166}
167
168// Annoyingly we are forced to use a literal to specify the alignment of
169// `MsgqData`, so check that it corresponds to the actual GSP page size here.
170static_assert!(align_of::<MsgqData>() == GSP_PAGE_SIZE);
171
172/// Unidirectional message queue.
173///
174/// Contains the data for a message queue, that either the driver or GSP writes to.
175///
176/// Note that while the write pointer of `tx` corresponds to the `msgq` of the same instance, the
177/// read pointer of `rx` actually refers to the `Msgq` owned by the other side.
178/// This design ensures that only the driver or GSP ever writes to a given instance of this struct.
179#[repr(C)]
180// There is no struct defined for this in the open-gpu-kernel-source headers.
181// Instead it is defined by code in `GspMsgQueuesInit()`.
182struct Msgq {
183 /// Header for sending messages, including the write pointer.
184 tx: MsgqTxHeader,
185 /// Header for receiving messages, including the read pointer.
186 rx: MsgqRxHeader,
187 /// The message queue proper.
188 msgq: MsgqData,
189}
190
191/// Structure shared between the driver and the GSP and containing the command and message queues.
192#[repr(C)]
193struct GspMem {
194 /// Self-mapping page table entries.
195 ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>,
196 /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the
197 /// write and read pointers that the CPU updates. This means that the read pointer here is an
198 /// index into the GSP queue.
199 ///
200 /// This member is read-only for the GSP.
201 cpuq: Msgq,
202 /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the
203 /// write and read pointers that the GSP updates. This means that the read pointer here is an
204 /// index into the CPU queue.
205 ///
206 /// This member is read-only for the driver.
207 gspq: Msgq,
208}
209
210impl GspMem {
211 const PTE_ARRAY_SIZE: usize = GSP_PAGE_SIZE / size_of::<u64>();
212}
213
214// SAFETY: These structs don't meet the no-padding requirements of AsBytes but
215// that is not a problem because they are not used outside the kernel.
216unsafe impl AsBytes for GspMem {}
217
218// SAFETY: These structs don't meet the no-padding requirements of FromBytes but
219// that is not a problem because they are not used outside the kernel.
220unsafe impl FromBytes for GspMem {}
221
222/// Wrapper around [`GspMem`] to share it with the GPU using a [`Coherent`].
223///
224/// This provides the low-level functionality to communicate with the GSP, including allocation of
225/// queue space to write messages to and management of read/write pointers.
226///
227/// This is shared with the GSP, with clear ownership rules regarding the command queues:
228///
229/// * The driver owns (i.e. can write to) the part of the CPU message queue between the CPU write
230/// pointer and the GSP read pointer. This region is returned by [`Self::driver_write_area`].
231/// * The driver owns (i.e. can read from) the part of the GSP message queue between the CPU read
232/// pointer and the GSP write pointer. This region is returned by [`Self::driver_read_area`].
233struct DmaGspMem(Coherent<GspMem>);
234
235impl DmaGspMem {
236 /// Allocate a new instance and map it for `dev`.
237 fn new(dev: &device::Device<device::Bound>) -> Result<Self> {
238 const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::<Msgq>() }>();
239 const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>();
240
241 let mut gsp_mem = CoherentBox::<GspMem>::zeroed(dev, GFP_KERNEL)?;
242 gsp_mem.cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES);
243 gsp_mem.cpuq.rx = MsgqRxHeader::new();
244
245 let gsp_mem: Coherent<_> = gsp_mem.into();
246 PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_address())?;
247
248 Ok(Self(gsp_mem))
249 }
250
251 /// Returns the region of the CPU message queue that the driver is currently allowed to write
252 /// to.
253 ///
254 /// As the message queue is a circular buffer, the region may be discontiguous in memory. In
255 /// that case the second slice will have a non-zero length.
256 fn driver_write_area(&mut self) -> (&mut [[u8; GSP_PAGE_SIZE]], &mut [[u8; GSP_PAGE_SIZE]]) {
257 let tx = self.cpu_write_ptr();
258 let rx = self.gsp_read_ptr();
259
260 // Pointer to the first entry of the CPU message queue.
261 let data = ptr::project!(mut self.0.as_mut_ptr(), .cpuq.msgq.data[build: 0]);
262
263 let (tail_end, wrap_end) = if rx == 0 {
264 // The write area is non-wrapping, and stops at the second-to-last entry of the command
265 // queue (to leave the last one empty).
266 (MSGQ_NUM_PAGES - 1, 0)
267 } else if rx <= tx {
268 // The write area wraps and continues until `rx - 1`.
269 (MSGQ_NUM_PAGES, rx - 1)
270 } else {
271 // The write area doesn't wrap and stops at `rx - 1`.
272 (rx - 1, 0)
273 };
274
275 // SAFETY:
276 // - `data` was created from a valid pointer, and `rx` and `tx` are in the
277 // `0..MSGQ_NUM_PAGES` range per the invariants of `cpu_write_ptr` and `gsp_read_ptr`,
278 // thus the created slices are valid.
279 // - The area starting at `tx` and ending at `rx - 2` modulo `MSGQ_NUM_PAGES`,
280 // inclusive, belongs to the driver for writing and is not accessed concurrently by
281 // the GSP.
282 // - The caller holds a reference to `self` for as long as the returned slices are live,
283 // meaning the CPU write pointer cannot be advanced and thus that the returned area
284 // remains exclusive to the CPU for the duration of the slices.
285 // - The created slices point to non-overlapping sub-ranges of `data` in all
286 // branches (in the `rx <= tx` case, the second slice ends at `rx - 1` which is strictly
287 // less than `tx` where the first slice starts; in the other cases the second slice is
288 // empty), so creating two `&mut` references from them does not violate aliasing rules.
289 unsafe {
290 (
291 core::slice::from_raw_parts_mut(
292 data.add(num::u32_as_usize(tx)),
293 num::u32_as_usize(tail_end - tx),
294 ),
295 core::slice::from_raw_parts_mut(data, num::u32_as_usize(wrap_end)),
296 )
297 }
298 }
299
300 /// Returns the size of the region of the CPU message queue that the driver is currently allowed
301 /// to write to, in bytes.
302 fn driver_write_area_size(&self) -> usize {
303 let tx = self.cpu_write_ptr();
304 let rx = self.gsp_read_ptr();
305
306 // `rx` and `tx` are both in `0..MSGQ_NUM_PAGES` per the invariants of `gsp_read_ptr` and
307 // `cpu_write_ptr`. The minimum value case is where `rx == 0` and `tx == MSGQ_NUM_PAGES -
308 // 1`, which gives `0 + MSGQ_NUM_PAGES - (MSGQ_NUM_PAGES - 1) - 1 == 0`.
309 let slots = (rx + MSGQ_NUM_PAGES - tx - 1) % MSGQ_NUM_PAGES;
310 num::u32_as_usize(slots) * GSP_PAGE_SIZE
311 }
312
313 /// Returns the region of the GSP message queue that the driver is currently allowed to read
314 /// from.
315 ///
316 /// As the message queue is a circular buffer, the region may be discontiguous in memory. In
317 /// that case the second slice will have a non-zero length.
318 fn driver_read_area(&self) -> (&[[u8; GSP_PAGE_SIZE]], &[[u8; GSP_PAGE_SIZE]]) {
319 let tx = self.gsp_write_ptr();
320 let rx = self.cpu_read_ptr();
321
322 // Pointer to the first entry of the GSP message queue.
323 let data = ptr::project!(self.0.as_ptr(), .gspq.msgq.data[build: 0]);
324
325 let (tail_end, wrap_end) = if rx <= tx {
326 // Read area is non-wrapping and stops right before `tx`.
327 (tx, 0)
328 } else {
329 // Read area is wrapping and stops right before `tx`.
330 (MSGQ_NUM_PAGES, tx)
331 };
332
333 // SAFETY:
334 // - `data` was created from a valid pointer, and `rx` and `tx` are in the
335 // `0..MSGQ_NUM_PAGES` range per the invariants of `gsp_write_ptr` and `cpu_read_ptr`,
336 // thus the created slices are valid.
337 // - The area starting at `rx` and ending at `tx - 1` modulo `MSGQ_NUM_PAGES`,
338 // inclusive, belongs to the driver for reading and is not accessed concurrently by
339 // the GSP.
340 // - The caller holds a reference to `self` for as long as the returned slices are live,
341 // meaning the CPU read pointer cannot be advanced and thus that the returned area
342 // remains exclusive to the CPU for the duration of the slices.
343 unsafe {
344 (
345 core::slice::from_raw_parts(
346 data.add(num::u32_as_usize(rx)),
347 num::u32_as_usize(tail_end - rx),
348 ),
349 core::slice::from_raw_parts(data, num::u32_as_usize(wrap_end)),
350 )
351 }
352 }
353
354 /// Allocates a region on the command queue that is large enough to send a command of `size`
355 /// bytes, waiting for space to become available based on the provided timeout.
356 ///
357 /// This returns a [`GspCommand`] ready to be written to by the caller.
358 ///
359 /// # Errors
360 ///
361 /// - `EMSGSIZE` if the command is larger than [`GSP_MSG_QUEUE_ELEMENT_SIZE_MAX`].
362 /// - `ETIMEDOUT` if space does not become available within the timeout.
363 /// - `EIO` if the command header is not properly aligned.
364 fn allocate_command(&mut self, size: usize, timeout: Delta) -> Result<GspCommand<'_>> {
365 if size_of::<GspMsgElement>() + size > GSP_MSG_QUEUE_ELEMENT_SIZE_MAX {
366 return Err(EMSGSIZE);
367 }
368 read_poll_timeout(
369 || Ok(self.driver_write_area_size()),
370 |available_bytes| *available_bytes >= size_of::<GspMsgElement>() + size,
371 Delta::from_micros(1),
372 timeout,
373 )?;
374
375 // Get the current writable area as an array of bytes.
376 let (slice_1, slice_2) = {
377 let (slice_1, slice_2) = self.driver_write_area();
378
379 (slice_1.as_flattened_mut(), slice_2.as_flattened_mut())
380 };
381
382 // Extract area for the `GspMsgElement`.
383 let (header, slice_1) = GspMsgElement::from_bytes_mut_prefix(slice_1).ok_or(EIO)?;
384
385 // Create the contents area.
386 let (slice_1, slice_2) = if slice_1.len() > size {
387 // Contents fits entirely in `slice_1`.
388 (&mut slice_1[..size], &mut slice_2[0..0])
389 } else {
390 // Need all of `slice_1` and some of `slice_2`.
391 let slice_2_len = size - slice_1.len();
392 (slice_1, &mut slice_2[..slice_2_len])
393 };
394
395 Ok(GspCommand {
396 header,
397 contents: (slice_1, slice_2),
398 })
399 }
400
401 // Returns the index of the memory page the GSP will write the next message to.
402 //
403 // # Invariants
404 //
405 // - The returned value is within `0..MSGQ_NUM_PAGES`.
406 fn gsp_write_ptr(&self) -> u32 {
407 MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES
408 }
409
410 // Returns the index of the memory page the GSP will read the next command from.
411 //
412 // # Invariants
413 //
414 // - The returned value is within `0..MSGQ_NUM_PAGES`.
415 fn gsp_read_ptr(&self) -> u32 {
416 MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES
417 }
418
419 // Returns the index of the memory page the CPU can read the next message from.
420 //
421 // # Invariants
422 //
423 // - The returned value is within `0..MSGQ_NUM_PAGES`.
424 fn cpu_read_ptr(&self) -> u32 {
425 MsgqRxHeader::read_ptr(io_project!(self.0, .cpuq.rx)) % MSGQ_NUM_PAGES
426 }
427
428 // Informs the GSP that it can send `elem_count` new pages into the message queue.
429 fn advance_cpu_read_ptr(&mut self, elem_count: u32) {
430 let rx = io_project!(self.0, .cpuq.rx);
431 let rptr = MsgqRxHeader::read_ptr(rx).wrapping_add(elem_count) % MSGQ_NUM_PAGES;
432
433 // Ensure read pointer is properly ordered.
434 fence(Ordering::SeqCst);
435
436 MsgqRxHeader::set_read_ptr(rx, rptr)
437 }
438
439 // Returns the index of the memory page the CPU can write the next command to.
440 //
441 // # Invariants
442 //
443 // - The returned value is within `0..MSGQ_NUM_PAGES`.
444 fn cpu_write_ptr(&self) -> u32 {
445 MsgqTxHeader::write_ptr(io_project!(self.0, .cpuq.tx)) % MSGQ_NUM_PAGES
446 }
447
448 // Informs the GSP that it can process `elem_count` new pages from the command queue.
449 fn advance_cpu_write_ptr(&mut self, elem_count: u32) {
450 let tx = io_project!(self.0, .cpuq.tx);
451 let wptr = MsgqTxHeader::write_ptr(tx).wrapping_add(elem_count) % MSGQ_NUM_PAGES;
452 MsgqTxHeader::set_write_ptr(tx, wptr);
453
454 // Ensure all command data is visible before triggering the GSP read.
455 fence(Ordering::SeqCst);
456 }
457}
458
459/// A command ready to be sent on the command queue.
460///
461/// This is the type returned by [`DmaGspMem::allocate_command`].
462struct GspCommand<'a> {
463 // Writable reference to the header of the command.
464 header: &'a mut GspMsgElement,
465 // Writable slices to the contents of the command. The second slice is zero unless the command
466 // loops over the command queue.
467 contents: (&'a mut [u8], &'a mut [u8]),
468}
469
470/// A message ready to be processed from the message queue.
471///
472/// This is the type returned by [`Cmdq::wait_for_msg`].
473struct GspMessage<'a> {
474 // Reference to the header of the message.
475 header: &'a GspMsgElement,
476 // Slices to the contents of the message. The second slice is zero unless the message loops
477 // over the message queue.
478 contents: (&'a [u8], &'a [u8]),
479}
480
481/// GSP command queue.
482///
483/// Provides the ability to send commands and receive messages from the GSP using a shared memory
484/// area.
485#[pin_data]
486pub(crate) struct Cmdq {
487 /// Inner mutex-protected state.
488 #[pin]
489 inner: Mutex<CmdqInner>,
490 /// DMA address of the command queue's shared memory region.
491 pub(super) dma_addr: DmaAddress,
492}
493
494impl Cmdq {
495 /// Offset of the data after the PTEs.
496 const POST_PTE_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq);
497
498 /// Offset of command queue ring buffer.
499 pub(crate) const CMDQ_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq)
500 + core::mem::offset_of!(Msgq, msgq)
501 - Self::POST_PTE_OFFSET;
502
503 /// Offset of message queue ring buffer.
504 pub(crate) const STATQ_OFFSET: usize = core::mem::offset_of!(GspMem, gspq)
505 + core::mem::offset_of!(Msgq, msgq)
506 - Self::POST_PTE_OFFSET;
507
508 /// Number of page table entries for the GSP shared region.
509 pub(crate) const NUM_PTES: usize = size_of::<GspMem>() >> GSP_PAGE_SHIFT;
510
511 /// Default timeout for receiving a message from the GSP.
512 pub(super) const RECEIVE_TIMEOUT: Delta = Delta::from_secs(5);
513
514 /// Creates a new command queue for `dev`.
515 pub(crate) fn new(dev: &device::Device<device::Bound>) -> impl PinInit<Self, Error> + '_ {
516 pin_init_scope(move || {
517 let gsp_mem = DmaGspMem::new(dev)?;
518
519 Ok(try_pin_init!(Self {
520 dma_addr: gsp_mem.0.dma_address(),
521 inner <- new_mutex!(CmdqInner {
522 dev: dev.into(),
523 gsp_mem,
524 seq: 0,
525 }),
526 }))
527 })
528 }
529
530 /// Computes the checksum for the message pointed to by `it`.
531 ///
532 /// A message is made of several parts, so `it` is an iterator over byte slices representing
533 /// these parts.
534 fn calculate_checksum<T: Iterator<Item = u8>>(it: T) -> u32 {
535 let sum64 = it
536 .enumerate()
537 .map(|(idx, byte)| (((idx % 8) * 8) as u32, byte))
538 .fold(0, |acc, (rol, byte)| acc ^ u64::from(byte).rotate_left(rol));
539
540 ((sum64 >> 32) as u32) ^ (sum64 as u32)
541 }
542
543 /// Notifies the GSP that we have updated the command queue pointers.
544 fn notify_gsp(bar: Bar0<'_>) {
545 bar.write_reg(regs::NV_PGSP_QUEUE_HEAD::zeroed().with_address(0u32));
546 }
547
548 /// Sends `command` to the GSP and waits for the reply.
549 ///
550 /// Messages with non-matching function codes are silently consumed until the expected reply
551 /// arrives.
552 ///
553 /// The queue is locked for the entire send+receive cycle to ensure that no other command can
554 /// be interleaved.
555 ///
556 /// # Errors
557 ///
558 /// - `ETIMEDOUT` if space does not become available to send the command, or if the reply is
559 /// not received within the timeout.
560 /// - `EIO` if the variable payload requested by the command has not been entirely
561 /// written to by its [`CommandToGsp::init_variable_payload`] method.
562 ///
563 /// Error codes returned by the command and reply initializers are propagated as-is.
564 pub(crate) fn send_command<M>(&self, bar: Bar0<'_>, command: M) -> Result<M::Reply>
565 where
566 M: CommandToGsp,
567 M::Reply: MessageFromGsp,
568 Error: From<M::InitError>,
569 Error: From<<M::Reply as MessageFromGsp>::InitError>,
570 {
571 let mut inner = self.inner.lock();
572 inner.send_command(bar, command)?;
573
574 loop {
575 match inner.receive_msg::<M::Reply>(Self::RECEIVE_TIMEOUT) {
576 Ok(reply) => break Ok(reply),
577 Err(ERANGE) => continue,
578 Err(e) => break Err(e),
579 }
580 }
581 }
582
583 /// Sends `command` to the GSP without waiting for a reply.
584 ///
585 /// # Errors
586 ///
587 /// - `ETIMEDOUT` if space does not become available within the timeout.
588 /// - `EIO` if the variable payload requested by the command has not been entirely
589 /// written to by its [`CommandToGsp::init_variable_payload`] method.
590 ///
591 /// Error codes returned by the command initializers are propagated as-is.
592 pub(crate) fn send_command_no_wait<M>(&self, bar: Bar0<'_>, command: M) -> Result
593 where
594 M: CommandToGsp<Reply = NoReply>,
595 Error: From<M::InitError>,
596 {
597 self.inner.lock().send_command(bar, command)
598 }
599
600 /// Receive a message from the GSP.
601 ///
602 /// See [`CmdqInner::receive_msg`] for details.
603 pub(crate) fn receive_msg<M: MessageFromGsp>(&self, timeout: Delta) -> Result<M>
604 where
605 // This allows all error types, including `Infallible`, to be used for `M::InitError`.
606 Error: From<M::InitError>,
607 {
608 self.inner.lock().receive_msg(timeout)
609 }
610}
611
612/// Inner mutex protected state of [`Cmdq`].
613struct CmdqInner {
614 /// Device this command queue belongs to.
615 dev: ARef<device::Device>,
616 /// Current command sequence number.
617 seq: u32,
618 /// Memory area shared with the GSP for communicating commands and messages.
619 gsp_mem: DmaGspMem,
620}
621
622impl CmdqInner {
623 /// Timeout for waiting for space on the command queue.
624 const ALLOCATE_TIMEOUT: Delta = Delta::from_secs(1);
625
626 /// Sends `command` to the GSP, without splitting it.
627 ///
628 /// # Errors
629 ///
630 /// - `EMSGSIZE` if the command exceeds the maximum queue element size.
631 /// - `ETIMEDOUT` if space does not become available within the timeout.
632 /// - `EIO` if the variable payload requested by the command has not been entirely
633 /// written to by its [`CommandToGsp::init_variable_payload`] method.
634 ///
635 /// Error codes returned by the command initializers are propagated as-is.
636 fn send_single_command<M>(&mut self, bar: Bar0<'_>, command: M) -> Result
637 where
638 M: CommandToGsp,
639 // This allows all error types, including `Infallible`, to be used for `M::InitError`.
640 Error: From<M::InitError>,
641 {
642 let size_in_bytes = command.size();
643 let dst = self
644 .gsp_mem
645 .allocate_command(size_in_bytes, Self::ALLOCATE_TIMEOUT)?;
646
647 // Extract area for the command itself. The GSP message header and the command header
648 // together are guaranteed to fit entirely into a single page, so it's ok to only look
649 // at `dst.contents.0` here.
650 let (cmd, payload_1) = M::Command::from_bytes_mut_prefix(dst.contents.0).ok_or(EIO)?;
651
652 // Fill the header and command in-place.
653 let msg_element = GspMsgElement::init(self.seq, size_in_bytes, M::FUNCTION);
654 // SAFETY: `msg_header` and `cmd` are valid references, and not touched if the initializer
655 // fails.
656 unsafe {
657 pin_init::raw_try_init(core::ptr::from_mut(dst.header), msg_element)?;
658 pin_init::raw_try_init(core::ptr::from_mut(cmd), command.init())?;
659 }
660
661 // Fill the variable-length payload, which may be empty.
662 let mut sbuffer = SBufferIter::new_writer([&mut payload_1[..], &mut dst.contents.1[..]]);
663 command.init_variable_payload(&mut sbuffer)?;
664
665 if !sbuffer.is_empty() {
666 return Err(EIO);
667 }
668 drop(sbuffer);
669
670 // Compute checksum now that the whole message is ready.
671 dst.header
672 .set_checksum(Cmdq::calculate_checksum(SBufferIter::new_reader([
673 dst.header.as_bytes(),
674 dst.contents.0,
675 dst.contents.1,
676 ])));
677
678 dev_dbg!(
679 &self.dev,
680 "GSP RPC: send: seq# {}, function={:?}, length=0x{:x}\n",
681 self.seq,
682 M::FUNCTION,
683 dst.header.length(),
684 );
685
686 // All set - update the write pointer and inform the GSP of the new command.
687 let elem_count = dst.header.element_count();
688 self.seq += 1;
689 self.gsp_mem.advance_cpu_write_ptr(elem_count);
690 Cmdq::notify_gsp(bar);
691
692 Ok(())
693 }
694
695 /// Sends `command` to the GSP.
696 ///
697 /// The command may be split into multiple messages if it is large.
698 ///
699 /// # Errors
700 ///
701 /// - `ETIMEDOUT` if space does not become available within the timeout.
702 /// - `EIO` if the variable payload requested by the command has not been entirely
703 /// written to by its [`CommandToGsp::init_variable_payload`] method.
704 ///
705 /// Error codes returned by the command initializers are propagated as-is.
706 fn send_command<M>(&mut self, bar: Bar0<'_>, command: M) -> Result
707 where
708 M: CommandToGsp,
709 Error: From<M::InitError>,
710 {
711 match SplitState::new(command)? {
712 SplitState::Single(command) => self.send_single_command(bar, command),
713 SplitState::Split(command, mut continuations) => {
714 self.send_single_command(bar, command)?;
715
716 while let Some(continuation) = continuations.next() {
717 // Turbofish needed because the compiler cannot infer M here.
718 self.send_single_command::<ContinuationRecord<'_>>(bar, continuation)?;
719 }
720
721 Ok(())
722 }
723 }
724 }
725
726 /// Wait for a message to become available on the message queue.
727 ///
728 /// This works purely at the transport layer and does not interpret or validate the message
729 /// beyond the advertised length in its [`GspMsgElement`].
730 ///
731 /// This method returns:
732 ///
733 /// - A reference to the [`GspMsgElement`] of the message,
734 /// - Two byte slices with the contents of the message. The second slice is empty unless the
735 /// message loops across the message queue.
736 ///
737 /// # Errors
738 ///
739 /// - `ETIMEDOUT` if `timeout` has elapsed before any message becomes available.
740 /// - `EIO` if there was some inconsistency (e.g. message shorter than advertised) on the
741 /// message queue.
742 ///
743 /// Error codes returned by the message constructor are propagated as-is.
744 fn wait_for_msg(&self, timeout: Delta) -> Result<GspMessage<'_>> {
745 // Wait for a message to arrive from the GSP.
746 let (slice_1, slice_2) = read_poll_timeout(
747 || Ok(self.gsp_mem.driver_read_area()),
748 |driver_area| !driver_area.0.is_empty(),
749 Delta::from_millis(1),
750 timeout,
751 )
752 .map(|(slice_1, slice_2)| (slice_1.as_flattened(), slice_2.as_flattened()))?;
753
754 // Extract the `GspMsgElement`.
755 let (header, slice_1) = GspMsgElement::from_bytes_prefix(slice_1).ok_or(EIO)?;
756
757 dev_dbg!(
758 &self.dev,
759 "GSP RPC: receive: seq# {}, function={:?}, length=0x{:x}\n",
760 header.sequence(),
761 header.function(),
762 header.length(),
763 );
764
765 let payload_length = header.payload_length();
766
767 // Check that the driver read area is large enough for the message.
768 if slice_1.len() + slice_2.len() < payload_length {
769 return Err(EIO);
770 }
771
772 // Cut the message slices down to the actual length of the message.
773 let (slice_1, slice_2) = if slice_1.len() > payload_length {
774 // PANIC: we checked above that `slice_1` is at least as long as `payload_length`.
775 (slice_1.split_at(payload_length).0, &slice_2[0..0])
776 } else {
777 (
778 slice_1,
779 // PANIC: we checked above that `slice_1.len() + slice_2.len()` is at least as
780 // large as `payload_length`.
781 slice_2.split_at(payload_length - slice_1.len()).0,
782 )
783 };
784
785 // Validate checksum.
786 if Cmdq::calculate_checksum(SBufferIter::new_reader([
787 header.as_bytes(),
788 slice_1,
789 slice_2,
790 ])) != 0
791 {
792 dev_err!(
793 &self.dev,
794 "GSP RPC: receive: Call {} - bad checksum\n",
795 header.sequence()
796 );
797 return Err(EIO);
798 }
799
800 Ok(GspMessage {
801 header,
802 contents: (slice_1, slice_2),
803 })
804 }
805
806 /// Receive a message from the GSP.
807 ///
808 /// The expected message type is specified using the `M` generic parameter. If the pending
809 /// message has a different function code, `ERANGE` is returned and the message is consumed.
810 ///
811 /// The read pointer is always advanced past the message, regardless of whether it matched.
812 ///
813 /// # Errors
814 ///
815 /// - `ETIMEDOUT` if `timeout` has elapsed before any message becomes available.
816 /// - `EIO` if there was some inconsistency (e.g. message shorter than advertised) on the
817 /// message queue.
818 /// - `EINVAL` if the function code of the message was not recognized.
819 /// - `ERANGE` if the message had a recognized but non-matching function code.
820 ///
821 /// Error codes returned by [`MessageFromGsp::read`] are propagated as-is.
822 fn receive_msg<M: MessageFromGsp>(&mut self, timeout: Delta) -> Result<M>
823 where
824 // This allows all error types, including `Infallible`, to be used for `M::InitError`.
825 Error: From<M::InitError>,
826 {
827 let message = self.wait_for_msg(timeout)?;
828 let function = message.header.function().map_err(|_| EINVAL)?;
829
830 // Extract the message. Store the result as we want to advance the read pointer even in
831 // case of failure.
832 let result = if function == M::FUNCTION {
833 let (cmd, contents_1) = M::Message::from_bytes_prefix(message.contents.0).ok_or(EIO)?;
834 let mut sbuffer = SBufferIter::new_reader([contents_1, message.contents.1]);
835
836 M::read(cmd, &mut sbuffer)
837 .map_err(|e| e.into())
838 .inspect(|_| {
839 if !sbuffer.is_empty() {
840 dev_warn!(
841 &self.dev,
842 "GSP message {:?} has unprocessed data\n",
843 function
844 );
845 }
846 })
847 } else {
848 Err(ERANGE)
849 };
850
851 // Advance the read pointer past this message.
852 self.gsp_mem.advance_cpu_read_ptr(u32::try_from(
853 message.header.length().div_ceil(GSP_PAGE_SIZE),
854 )?);
855
856 result
857 }
858}