nuclei/config.rs
1/// Nuclei's proactor configuration: the io_uring settings of every shard, and
2/// the backend of every runtime that names none. It is set once, before any
3/// IO, with [`Proactor::with_config`](crate::Proactor::with_config); a
4/// runtime with io_uring settings of its own takes them from
5/// [`Builder::io_uring`](crate::Builder::io_uring) instead.
6#[derive(Clone, Debug, Default)]
7pub struct NucleiConfig {
8 /// The settings every shard's io_uring ring is made with.
9 pub iouring: IoUringConfiguration,
10 /// The kernel interface every runtime that names none runs its IO on
11 /// ([`Builder::backend`](crate::Builder::backend) names one for one
12 /// runtime). `None` (the default) picks io_uring where the kernel lets
13 /// this process have it, and the platform's poller otherwise. Having it
14 /// means every kernel feature a shard's ring needs with this
15 /// configuration: the buffer ring for multishot receives while
16 /// [`recv_buffers`](IoUringConfiguration::recv_buffers) is not 0, and the
17 /// table of fixed files. A backend named here that this process cannot
18 /// have makes the first shard fail with the reason. A non-empty
19 /// `NUCLEI_BACKEND` overrides it.
20 pub backend: Option<crate::IoBackend>,
21}
22
23/// The settings of every shard's io_uring ring (Linux 6.1 or later).
24///
25/// The default is [`interrupt_driven`](Self::interrupt_driven) with 2048
26/// queue entries and the io-wq worker limits `Some(256)` and `Some(512)`.
27#[derive(Clone, Debug)]
28pub struct IoUringConfiguration {
29 /// Entries in each ring's submission queue (default 2048); the completion
30 /// queue holds twice as many.
31 ///
32 /// At least 64, the
33 /// floor a ring is shrunk to when the system has no memory for more; a
34 /// smaller value is refused as the configuration's mistake when the first
35 /// shard starts. The kernel rounds the count up to a power of two and
36 /// clamps it to its own maximum (32768 on current kernels), so a larger
37 /// value is accepted and gets what the kernel allows.
38 pub queue_len: u32,
39 /// With `Some(ms)`, a kernel thread polls the submission queue and sleeps
40 /// after that many idle milliseconds: fewer system calls for more CPU
41 /// (default `None`: no polling thread).
42 ///
43 /// Before version 5.11 of the Linux kernel, to successfully use this feature, the application
44 /// must register a set of files to be used for IO through io_uring_register(2) using the
45 /// `IORING_REGISTER_FILES` opcode. Failure to do so will result in submitted IO being errored
46 /// with EBADF. The presence of this feature can be detected by the `IORING_FEAT_SQPOLL_NONFIXED`
47 /// feature flag. In version 5.11 and later, it is no longer necessary to register files to use
48 /// this feature. 5.11 also allows using this as non-root, if the user has the CAP_SYS_NICE
49 /// capability. In 5.13 this requirement was also relaxed, and no special privileges are needed
50 /// for SQPOLL in newer kernels. Certain stable kernels older than 5.13 may also support
51 /// unprivileged SQPOLL.
52 ///
53 /// A shorter interval keeps the kernel thread awake more, for more CPU; a
54 /// longer one lets it sleep, and a submission after it slept wakes it.
55 pub sqpoll_wake_interval: Option<u32>,
56
57 /// The most io_uring worker threads per NUMA node for IO bound in time,
58 /// on regular files and block devices (default `Some(256)`).
59 ///
60 /// `Some(0)` leaves the kernel's limit as it is; with `None` the kernel
61 /// limits them to the ring's submission entries or four times the online
62 /// CPUs, whichever is smaller.
63 pub per_numa_bounded_worker_count: Option<u32>,
64
65 /// The most io_uring worker threads per NUMA node for IO that may never
66 /// complete, such as on sockets (default `Some(512)`).
67 ///
68 /// `Some(0)` leaves the kernel's limit as it is; with `None` the
69 /// process's task limit (`RLIMIT_NPROC`) bounds them.
70 pub per_numa_unbounded_worker_count: Option<u32>,
71
72 /// Has no effect, and is kept so code written against earlier versions
73 /// builds (default `false`). Every shard already looks at its ring's
74 /// completions while it spins before it sleeps, and a kernel thread that
75 /// polls the submission queue is
76 /// [`sqpoll_wake_interval`](Self::sqpoll_wake_interval); no third kind of
77 /// polling is left for this flag to turn on.
78 pub aggressive_poll: bool,
79
80 /// Busy-waits for file completions instead of taking interrupts (default
81 /// `false`): lower latency for more CPU, and only for files opened with
82 /// `O_DIRECT` on file systems that support polling.
83 pub iopoll_enabled: bool,
84
85 /// Buffers in each shard's group for multishot receives (default 256, or
86 /// [`queue_len`](Self::queue_len) rounded up to a power of two when that is
87 /// fewer): the kernel fills one as bytes arrive, so a stream's receive
88 /// stays armed for every arrival and a receive that waits holds no buffer
89 /// of its own.
90 ///
91 /// The buffers, with their descriptors, count against the shard's
92 /// [`io_buffer_cap`](crate::Builder::io_buffer_cap) for as long as the
93 /// shard lives, and the shard's own pool gets the rest; a group that
94 /// leaves the pool less than 512 bytes is refused as the configuration's
95 /// mistake when the shard starts, and one the system has no memory for
96 /// fails the shard's start with `OutOfMemory`.
97 ///
98 /// 0 turns multishot receives off, and each read then takes a buffer of
99 /// the shard's own pool for its receive. Otherwise a power of two from 1 to
100 /// 32768 and below twice `queue_len`, so the group never takes more than
101 /// half the completion queue (twice `queue_len`, rounded up to a power of
102 /// two); any other value is refused as the configuration's mistake when
103 /// the first shard starts. A ring the system could only make with fewer
104 /// entries than `queue_len` (it had no memory for more) gets a group cut to
105 /// the same rule.
106 ///
107 /// A kernel that refuses provided buffer rings makes the default backend
108 /// choice pick the poller; 0 keeps io_uring there. Ubuntu's 6.8 kernels
109 /// from 6.8.0-136 on refuse every valid registration: a backport inverted
110 /// the check of its reserved words
111 /// (<https://bugs.launchpad.net/ubuntu/+source/linux/+bug/2162843>).
112 pub recv_buffers: u16,
113
114 /// The size in bytes of each of those buffers (default 16384); for UDP,
115 /// one that holds the largest datagram expected.
116 ///
117 /// From 512 (smaller buffers split a stream into chunks of a few packets
118 /// each) to 1 GiB; a
119 /// value outside is refused as the configuration's mistake when the first
120 /// shard starts. Not checked while `recv_buffers` is 0. `recv_buffers`
121 /// buffers of this size count against the shard's
122 /// [`io_buffer_cap`](crate::Builder::io_buffer_cap), as said there.
123 ///
124 /// A receive is stopped once the bytes it handed out and its reader has
125 /// not taken pass an eighth of the group (at least one buffer), and asked
126 /// for again when its reader catches up. What the kernel took for it
127 /// before the stop stays with it, so a reader with a long backlog may
128 /// hold more until it reads; other receives take buffers of the shard's
129 /// own pool meanwhile.
130 pub recv_buffer_size: u32,
131}
132
133/// The fewest entries a ring may be asked for: the floor a ring made short of
134/// memory is shrunk to.
135pub(crate) const MIN_QUEUE_LEN: u32 = 64;
136/// The smallest receive buffer.
137pub(crate) const MIN_RECV_BUFFER_SIZE: u32 = 512;
138/// The largest receive buffer.
139pub(crate) const MAX_RECV_BUFFER_SIZE: u32 = 1 << 30;
140
141/// The error for a setting outside what the runtime can work with: the
142/// configuration's mistake, naming the setting, its value and what is allowed.
143/// The one place every refused setting is worded.
144pub(crate) fn refuse(
145 setting: &str,
146 value: impl core::fmt::Display,
147 allowed: impl core::fmt::Display,
148) -> std::io::Error {
149 std::io::Error::new(
150 std::io::ErrorKind::InvalidInput,
151 format!("{setting} is {value}; it must be {allowed}"),
152 )
153}
154
155/// The default of [`IoUringConfiguration::recv_buffers`].
156const DEFAULT_RECV_BUFFERS: u16 = 256;
157
158/// The default receive buffers of a ring of `queue_len` entries: as many as
159/// half its completion queue holds, up to [`DEFAULT_RECV_BUFFERS`].
160fn default_recv_buffers(queue_len: u32) -> u16 {
161 u16::try_from(
162 queue_len
163 .min(u32::from(DEFAULT_RECV_BUFFERS))
164 .next_power_of_two(),
165 )
166 .unwrap_or(DEFAULT_RECV_BUFFERS)
167}
168/// The default of [`IoUringConfiguration::recv_buffer_size`].
169const DEFAULT_RECV_BUFFER_SIZE: u32 = 16 * 1024;
170
171impl IoUringConfiguration {
172 /// Checks the sizes against what a ring can work with.
173 ///
174 /// # Errors
175 ///
176 /// `InvalidInput` naming the setting, its value and the allowed range.
177 pub(crate) fn validate(&self) -> std::io::Result<()> {
178 if self.queue_len < MIN_QUEUE_LEN {
179 return Err(refuse(
180 "io_uring queue_len",
181 self.queue_len,
182 format_args!("at least {MIN_QUEUE_LEN}"),
183 ));
184 }
185 if self.recv_buffers == 0 {
186 return Ok(());
187 }
188 let twice = self.queue_len.saturating_mul(2);
189 if !self.recv_buffers.is_power_of_two() || u32::from(self.recv_buffers) >= twice {
190 return Err(refuse(
191 "io_uring recv_buffers",
192 self.recv_buffers,
193 format_args!(
194 "0, or a power of two from 1 to 32768 and below twice queue_len ({twice})"
195 ),
196 ));
197 }
198 if !(MIN_RECV_BUFFER_SIZE..=MAX_RECV_BUFFER_SIZE).contains(&self.recv_buffer_size) {
199 return Err(refuse(
200 "io_uring recv_buffer_size",
201 self.recv_buffer_size,
202 format_args!("from {MIN_RECV_BUFFER_SIZE} to {MAX_RECV_BUFFER_SIZE}"),
203 ));
204 }
205 Ok(())
206 }
207
208 /// No polling: a completion wakes its shard by interrupt. The one to pick
209 /// when unsure.
210 pub fn interrupt_driven(queue_len: u32) -> Self {
211 Self {
212 queue_len,
213 sqpoll_wake_interval: None,
214 per_numa_bounded_worker_count: None,
215 per_numa_unbounded_worker_count: None,
216 aggressive_poll: false,
217 iopoll_enabled: false,
218 recv_buffers: default_recv_buffers(queue_len),
219 recv_buffer_size: DEFAULT_RECV_BUFFER_SIZE,
220 }
221 }
222
223 /// A kernel thread polls the submission queue and sleeps after 2 idle
224 /// milliseconds: lower latency under steady traffic, for more CPU.
225 pub fn low_latency_driven(queue_len: u32) -> Self {
226 Self {
227 queue_len,
228 sqpoll_wake_interval: Some(2),
229 aggressive_poll: true,
230 recv_buffers: default_recv_buffers(queue_len),
231 ..Self::default()
232 }
233 }
234
235 /// A kernel thread polls the submission queue and sleeps after 2 idle
236 /// milliseconds, as [`low_latency_driven`](Self::low_latency_driven).
237 pub fn kernel_poll_only(queue_len: u32) -> Self {
238 Self {
239 queue_len,
240 sqpoll_wake_interval: Some(2),
241 aggressive_poll: false,
242 recv_buffers: default_recv_buffers(queue_len),
243 ..Self::default()
244 }
245 }
246
247 /// Busy-waits for file completions ([`iopoll_enabled`](Self::iopoll_enabled)):
248 /// low-latency IO on files opened with `O_DIRECT`.
249 pub fn io_poll(queue_len: u32) -> Self {
250 Self {
251 queue_len,
252 iopoll_enabled: true,
253 recv_buffers: default_recv_buffers(queue_len),
254 ..Self::default()
255 }
256 }
257}
258
259/// The interrupt-driven profile, with the io-wq worker limits nuclei sets on
260/// every ring.
261impl Default for IoUringConfiguration {
262 fn default() -> Self {
263 Self {
264 per_numa_bounded_worker_count: Some(1 << 8),
265 per_numa_unbounded_worker_count: Some(1 << 9),
266 ..Self::interrupt_driven(1 << 11)
267 }
268 }
269}
270
271#[cfg(all(test, ring))]
272mod tests;