projects/linux·blob master · refs

286 lines · 11.3 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3//! This module provides an interface for blk-mq drivers to implement.
4//!
5//! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h)
6
7use crate::{
8 bindings,
9 block::mq::{request::RequestDataWrapper, Request},
10 error::{from_result, Result},
11 prelude::*,
12 sync::{aref::ARef, Refcount},
13 types::ForeignOwnable,
14};
15use core::marker::PhantomData;
16
17type ForeignBorrowed<'a, T> = <T as ForeignOwnable>::Borrowed<'a>;
18
19/// Implement this trait to interface blk-mq as block devices.
20///
21/// To implement a block device driver, implement this trait as described in the
22/// [module level documentation]. The kernel will use the implementation of the
23/// functions defined in this trait to interface a block device driver. Note:
24/// There is no need for an exit_request() implementation, because the `drop`
25/// implementation of the [`Request`] type will be invoked by automatically by
26/// the C/Rust glue logic.
27///
28/// [module level documentation]: kernel::block::mq
29#[macros::vtable]
30pub trait Operations: Sized {
31 /// Data associated with the `struct request_queue` that is allocated for
32 /// the `GenDisk` associated with this `Operations` implementation.
33 type QueueData: ForeignOwnable;
34
35 /// Called by the kernel to queue a request with the driver. If `is_last` is
36 /// `false`, the driver is allowed to defer committing the request.
37 fn queue_rq(
38 queue_data: ForeignBorrowed<'_, Self::QueueData>,
39 rq: ARef<Request<Self>>,
40 is_last: bool,
41 ) -> Result;
42
43 /// Called by the kernel to indicate that queued requests should be submitted.
44 fn commit_rqs(queue_data: ForeignBorrowed<'_, Self::QueueData>);
45
46 /// Called by the kernel when the request is completed.
47 fn complete(rq: ARef<Request<Self>>);
48
49 /// Called by the kernel to poll the device for completed requests. Only
50 /// used for poll queues.
51 fn poll() -> bool {
52 build_error!(crate::error::VTABLE_DEFAULT_ERROR)
53 }
54}
55
56/// A vtable for blk-mq to interact with a block device driver.
57///
58/// A `bindings::blk_mq_ops` vtable is constructed from pointers to the `extern
59/// "C"` functions of this struct, exposed through the `OperationsVTable::VTABLE`.
60///
61/// For general documentation of these methods, see the kernel source
62/// documentation related to `struct blk_mq_operations` in
63/// [`include/linux/blk-mq.h`].
64///
65/// [`include/linux/blk-mq.h`]: srctree/include/linux/blk-mq.h
66pub(crate) struct OperationsVTable<T: Operations>(PhantomData<T>);
67
68impl<T: Operations> OperationsVTable<T> {
69 /// This function is called by the C kernel. A pointer to this function is
70 /// installed in the `blk_mq_ops` vtable for the driver.
71 ///
72 /// # Safety
73 ///
74 /// - The caller of this function must ensure that the pointee of `bd` is
75 /// valid for reads for the duration of this function.
76 /// - This function must be called for an initialized and live `hctx`. That
77 /// is, `Self::init_hctx_callback` was called and
78 /// `Self::exit_hctx_callback()` was not yet called.
79 /// - `(*bd).rq` must point to an initialized and live `bindings:request`.
80 /// That is, `Self::init_request_callback` was called but
81 /// `Self::exit_request_callback` was not yet called for the request.
82 /// - `(*bd).rq` must be owned by the driver. That is, the block layer must
83 /// promise to not access the request until the driver calls
84 /// `bindings::blk_mq_end_request` for the request.
85 unsafe extern "C" fn queue_rq_callback(
86 hctx: *mut bindings::blk_mq_hw_ctx,
87 bd: *const bindings::blk_mq_queue_data,
88 ) -> bindings::blk_status_t {
89 // SAFETY: `bd.rq` is valid as required by the safety requirement for
90 // this function.
91 let request = unsafe { &*(*bd).rq.cast::<Request<T>>() };
92
93 // One refcount for the ARef, one for being in flight
94 request.wrapper_ref().refcount().set(2);
95
96 // SAFETY:
97 // - We own a refcount that we took above. We pass that to `ARef`.
98 // - By the safety requirements of this function, `request` is a valid
99 // `struct request` and the private data is properly initialized.
100 // - `rq` will be alive until `blk_mq_end_request` is called and is
101 // reference counted by `ARef` until then.
102 let rq = unsafe { Request::aref_from_raw((*bd).rq) };
103
104 // SAFETY: `hctx` is valid as required by this function.
105 let queue_data = unsafe { (*(*hctx).queue).queuedata };
106
107 // SAFETY: `queue.queuedata` was created by `GenDiskBuilder::build` with
108 // a call to `ForeignOwnable::into_foreign` to create `queuedata`.
109 // `ForeignOwnable::from_foreign` is only called when the tagset is
110 // dropped, which happens after we are dropped.
111 let queue_data = unsafe { T::QueueData::borrow(queue_data) };
112
113 // SAFETY: We have exclusive access and we just set the refcount above.
114 unsafe { Request::start_unchecked(&rq) };
115
116 let ret = T::queue_rq(
117 queue_data,
118 rq,
119 // SAFETY: `bd` is valid as required by the safety requirement for
120 // this function.
121 unsafe { (*bd).last },
122 );
123
124 if let Err(e) = ret {
125 e.to_blk_status()
126 } else {
127 bindings::BLK_STS_OK as bindings::blk_status_t
128 }
129 }
130
131 /// This function is called by the C kernel. A pointer to this function is
132 /// installed in the `blk_mq_ops` vtable for the driver.
133 ///
134 /// # Safety
135 ///
136 /// This function may only be called by blk-mq C infrastructure. The caller
137 /// must ensure that `hctx` is valid.
138 unsafe extern "C" fn commit_rqs_callback(hctx: *mut bindings::blk_mq_hw_ctx) {
139 // SAFETY: `hctx` is valid as required by this function.
140 let queue_data = unsafe { (*(*hctx).queue).queuedata };
141
142 // SAFETY: `queue.queuedata` was created by `GenDisk::try_new()` with a
143 // call to `ForeignOwnable::into_foreign()` to create `queuedata`.
144 // `ForeignOwnable::from_foreign()` is only called when the tagset is
145 // dropped, which happens after we are dropped.
146 let queue_data = unsafe { T::QueueData::borrow(queue_data) };
147 T::commit_rqs(queue_data)
148 }
149
150 /// This function is called by the C kernel. A pointer to this function is
151 /// installed in the `blk_mq_ops` vtable for the driver.
152 ///
153 /// # Safety
154 ///
155 /// This function may only be called by blk-mq C infrastructure. `rq` must
156 /// point to a valid request that has been marked as completed. The pointee
157 /// of `rq` must be valid for write for the duration of this function.
158 unsafe extern "C" fn complete_callback(rq: *mut bindings::request) {
159 // SAFETY: This function can only be dispatched through
160 // `Request::complete`. We leaked a refcount then which we pick back up
161 // now.
162 let aref = unsafe { Request::aref_from_raw(rq) };
163 T::complete(aref);
164 }
165
166 /// This function is called by the C kernel. A pointer to this function is
167 /// installed in the `blk_mq_ops` vtable for the driver.
168 ///
169 /// # Safety
170 ///
171 /// This function may only be called by blk-mq C infrastructure.
172 unsafe extern "C" fn poll_callback(
173 _hctx: *mut bindings::blk_mq_hw_ctx,
174 _iob: *mut bindings::io_comp_batch,
175 ) -> crate::ffi::c_int {
176 T::poll().into()
177 }
178
179 /// This function is called by the C kernel. A pointer to this function is
180 /// installed in the `blk_mq_ops` vtable for the driver.
181 ///
182 /// # Safety
183 ///
184 /// This function may only be called by blk-mq C infrastructure. This
185 /// function may only be called once before `exit_hctx_callback` is called
186 /// for the same context.
187 unsafe extern "C" fn init_hctx_callback(
188 _hctx: *mut bindings::blk_mq_hw_ctx,
189 _tagset_data: *mut crate::ffi::c_void,
190 _hctx_idx: crate::ffi::c_uint,
191 ) -> crate::ffi::c_int {
192 from_result(|| Ok(0))
193 }
194
195 /// This function is called by the C kernel. A pointer to this function is
196 /// installed in the `blk_mq_ops` vtable for the driver.
197 ///
198 /// # Safety
199 ///
200 /// This function may only be called by blk-mq C infrastructure.
201 unsafe extern "C" fn exit_hctx_callback(
202 _hctx: *mut bindings::blk_mq_hw_ctx,
203 _hctx_idx: crate::ffi::c_uint,
204 ) {
205 }
206
207 /// This function is called by the C kernel. A pointer to this function is
208 /// installed in the `blk_mq_ops` vtable for the driver.
209 ///
210 /// # Safety
211 ///
212 /// - This function may only be called by blk-mq C infrastructure.
213 /// - `_set` must point to an initialized `TagSet<T>`.
214 /// - `rq` must point to an initialized `bindings::request`.
215 /// - The allocation pointed to by `rq` must be at the size of `Request`
216 /// plus the size of `RequestDataWrapper`.
217 unsafe extern "C" fn init_request_callback(
218 _set: *mut bindings::blk_mq_tag_set,
219 rq: *mut bindings::request,
220 _hctx_idx: crate::ffi::c_uint,
221 _numa_node: crate::ffi::c_int,
222 ) -> crate::ffi::c_int {
223 from_result(|| {
224 // SAFETY: By the safety requirements of this function, `rq` points
225 // to a valid allocation.
226 let pdu = unsafe { Request::wrapper_ptr(rq.cast::<Request<T>>()) };
227
228 // SAFETY: The refcount field is allocated but not initialized, so
229 // it is valid for writes.
230 unsafe { RequestDataWrapper::refcount_ptr(pdu.as_ptr()).write(Refcount::new(0)) };
231
232 Ok(0)
233 })
234 }
235
236 /// This function is called by the C kernel. A pointer to this function is
237 /// installed in the `blk_mq_ops` vtable for the driver.
238 ///
239 /// # Safety
240 ///
241 /// - This function may only be called by blk-mq C infrastructure.
242 /// - `_set` must point to an initialized `TagSet<T>`.
243 /// - `rq` must point to an initialized and valid `Request`.
244 unsafe extern "C" fn exit_request_callback(
245 _set: *mut bindings::blk_mq_tag_set,
246 rq: *mut bindings::request,
247 _hctx_idx: crate::ffi::c_uint,
248 ) {
249 // SAFETY: The tagset invariants guarantee that all requests are allocated with extra memory
250 // for the request data.
251 let pdu = unsafe { bindings::blk_mq_rq_to_pdu(rq) }.cast::<RequestDataWrapper>();
252
253 // SAFETY: `pdu` is valid for read and write and is properly initialised.
254 unsafe { core::ptr::drop_in_place(pdu) };
255 }
256
257 const VTABLE: bindings::blk_mq_ops = bindings::blk_mq_ops {
258 queue_rq: Some(Self::queue_rq_callback),
259 queue_rqs: None,
260 commit_rqs: Some(Self::commit_rqs_callback),
261 get_budget: None,
262 put_budget: None,
263 set_rq_budget_token: None,
264 get_rq_budget_token: None,
265 timeout: None,
266 poll: if T::HAS_POLL {
267 Some(Self::poll_callback)
268 } else {
269 None
270 },
271 complete: Some(Self::complete_callback),
272 init_hctx: Some(Self::init_hctx_callback),
273 exit_hctx: Some(Self::exit_hctx_callback),
274 init_request: Some(Self::init_request_callback),
275 exit_request: Some(Self::exit_request_callback),
276 cleanup_rq: None,
277 busy: None,
278 map_queues: None,
279 #[cfg(CONFIG_BLK_DEBUG_FS)]
280 show_rq: None,
281 };
282
283 pub(crate) const fn build() -> &'static bindings::blk_mq_ops {
284 &Self::VTABLE
285 }
286}