Chris Morgan  ›  Git  ›  sanitise-file-name  ›  blob
a54ab33f4917a5144a6110699fa075123e8858d9
[sanitise-file-name] / src / lib.rs
1 //! Sanitise names for use in file systems and the likes.
2 //!
3 //! The output string is guaranteed to be shorter than or equal to the input string in length,
4 //! except for file names that are reserved on Windows (see [`Options::windows_safe`]), in which
5 //! case an underscore is appended to the base name (e.g. NUL → NUL_, aux.h → aux_.h).
6 //!
7 //! The key parts of the API:
8 //!
9 #![cfg_attr(feature = "alloc", doc = "\
10 - <code>[sanitise][](input: &str) -> String</code>: the simplest thing to call;
11
12 - <code>[sanitise_with_options][](input: &str, options: &Options&lt;_>) -> String</code>:
13 when you want to tweak the nature of the sanitisation; and")]
14 #![cfg_attr(not(feature = "alloc"), doc = "\
15 - <s><code>[sanitise][](input: &str) -> String</code>: the simplest thing to call</s>
16 *(disabled in this build due to compiling without the `alloc` feature)*;
17
18 - <s><code>[sanitise_with_options][](input: &str, options: &Options&lt;_>) -> String</code>:
19 when you want to tweak the nature of the sanitisation</s> *(disabled in this build due to
20 compiling without the `alloc` feature)*; and")]
21 //!
22 //! - [`Options`], with detailed descriptions of each option.
23 //!
24 //! And for advanced users that want to control allocations or other similar things:
25 //!
26 //! - <code>[sanitise_to][](input: &str, options: &Options&lt;_>, out: &mut <em>String</em>)</code>,
27 //! sanitising into
28 #![cfg_attr(feature = "alloc", doc = " a `String`")]
29 #![cfg_attr(not(feature = "alloc"), doc = " a string")]
30 #![cfg_attr(feature = "tinyvec_string", doc = " or [`tinyvec_string::ArrayString`]")]
31 #![cfg_attr(all(docsrs, feature = "tinyvec_string"), doc = " (when enabled)")]
32 //! that you provide,
33 //! for which the following methods may help:
34 //!
35 //! - <code>[max_alloc_size][](options: &Options&lt;_>)</code> or
36 //! <code>[max_alloc_size_const][](options: &Options&lt;Option&lt;char>>)</code>,
37 //! to suggest a size for scratch buffer
38 #![cfg_attr(feature = "tinyvec_string", doc = " or `ArrayString`")]
39 //! applications; and
40 //!
41 //! - <code>[sufficient_alloc_size][](input: &str, options: &Options&lt;_>) -> usize</code>, to
42 //! suggest a size that will definitely be sufficient for one given input (mainly useful when you
43 //! are crafting a path with stuff before and after it).
44 //!
45 //! … but that’s dangerous territory, deep rabbit holes; ask if you actually *need* them—don’t be
46 //! like me. (When I am laid in earth, may my wrongs create no trouble in thy breast. Remember me,
47 //! but ah! forget my fate.)
48 //!
49 //! ### Conditional compilation/Cargo features
50 //!
51 //! This crate has several features:
52 //!
53 //! - **std**, enabled by default. Implies *alloc*. Disable it to get `#![no_std]` operation.
54 //!
55 //! - **alloc**, enabled by default via *std*. Provides the ability to sanitise to a `String` in
56 //! `sanitise_to`, and the `sanitise` and `sanitise_with_options` functions.
57 //!
58 //! - **tinyvec_string**, disabled by default. Provides the ability to sanitise to
59 //! `tinyvec_string::ArrayString`, which works without *alloc*.
60 //!
61 //! - **const-fn-trait-bound**, disabled by default, requires rustc nightly at the time of writing.
62 //! Makes [`max_alloc_size`] const.
63 //!
64 //! These docs were built with these features enabled:
65 #![cfg_attr(feature = "std", doc = " <span class='stab portability'><code>std</code></span>")]
66 #![cfg_attr(feature = "alloc", doc = " <span class='stab portability'><code>alloc</code></span>")]
67 #![cfg_attr(feature = "tinyvec_string", doc = " <span class='stab portability'><code>tinyvec_string</code></span>")]
68 #![cfg_attr(feature = "const-fn-trait-bound", doc = " <span class='stab portability'><code>const-fn-trait-bound</code></span>")]
69 #![cfg_attr(
70 all(
71 not(feature = "std"),
72 not(feature = "alloc"),
73 not(feature = "tinyvec_string"),
74 not(feature = "const-fn-trait-bound"),
75 ),
76 doc = " *(none of them)*")]
77 //!
78 //! … and these features disabled:
79 #![cfg_attr(not(feature = "std"), doc = " <span class='stab portability'><code>std</code></span>")]
80 #![cfg_attr(not(feature = "alloc"), doc = " <span class='stab portability'><code>alloc</code></span>")]
81 #![cfg_attr(not(feature = "tinyvec_string"), doc = " <span class='stab portability'><code>tinyvec_string</code></span>")]
82 #![cfg_attr(not(feature = "const-fn-trait-bound"), doc = " <span class='stab portability'><code>const-fn-trait-bound</code></span>")]
83 #![cfg_attr(
84 all(
85 feature = "std",
86 feature = "alloc",
87 feature = "tinyvec_string",
88 feature = "const-fn-trait-bound",
89 ),
90 doc = " *(none of them)*")]
91
92 // End docs.
93
94 #![cfg_attr(not(feature = "std"), no_std)]
95 #![cfg_attr(feature = "const-fn-trait-bound", feature(const_fn_trait_bound))]
96 #![cfg_attr(docsrs, feature(doc_cfg))]
97 #![cfg_attr(not(feature = "alloc"), allow(rustdoc::broken_intra_doc_links))] // I’m lazy.
98
99 #[cfg(feature = "alloc")]
100 extern crate alloc;
101 #[cfg(feature = "alloc")]
102 use alloc::string::String;
103
104 use core::ops::{Deref, Index, Range, RangeFrom, RangeBounds};
105
106 /// Sanitisation options. Defaults marked on each field.
107 ///
108 /// Take a look around, but I think everything’s pretty sane by default; the ones I think you’re
109 /// most likely to want to change are `url_safe` and `windows_safe`, though `replace_with`,
110 /// `collapse_replacements` and `six_measures_of_barley` can be interesting too for yielding
111 /// prettier results.
112 ///
113 /// If you set `length_limit` to `usize::MAX`, all the bool fields to `false`, and
114 /// `six_measures_of_barley` to an empty string, `sanitise` will not alter the input string in any
115 /// way. But that would be a rather expensive alternative to `.clone()`. In practice, I doubt you
116 /// ever want to disable `most_fs_safe`, which is a good baseline.
117 #[derive(Debug)]
118 pub struct Options<R: Replace> {
119 /// Limit the complete file name to this many UTF-8 code units. The default is **255**, which
120 /// is suitable for all practical platforms.
121 ///
122 /// (Some file systems limit lengths in UTF-8 code units and some in UTF-16 code units, but
123 /// UTF-16 never takes more code units than UTF-8 to encode a given Unicode string, so we can
124 /// ignore it.)
125 ///
126 /// Reasons you might want to reduce it:
127 ///
128 /// 1. You haven’t appended the extension yet, and so want to subtract the extension’s length.
129 /// (In that case I suggest writing `Options::DEFAULT.length_limit` instead of hard coding
130 /// 255—that’ll work in const context.)
131 ///
132 /// 2. You want smoother Windows support, for on Windows some things start falling over if the
133 /// total path length is greater than 260 characters; so measuring or estimating the path
134 /// length could potentially be useful—but unless you know, probably don’t worry too much,
135 /// someone’ll probably drop it deep in a node_modules tree at some point and then you’ll be
136 /// in trouble anyway. 😀 <!-- Okay, okay, so node_modules trees aren’t typically flattened
137 /// almost entirely these days; but let me have my joke, please? -->
138 ///
139 /// One other mildly significant note here: if you care about Apple’s pre-2017 HFS+ file
140 /// system, you should perform Unicode normalisation to NFD (most likely via the
141 /// `unicode-normalization` crate) before performing sanitisation, because the decomposed form
142 /// may be longer; if you don’t, then the path will be normalised to NFD by the file system
143 /// when you try to write it, which could take it over 255 and make it fail. I don’t think
144 /// there are any popular file systems that normalise any more, though APFS kinda prefers NFC,
145 /// so you might want to normalise to NFC. I do not know if normalising to NFC will ever
146 /// lengthen a UTF-8 string, but the spec allows it to (UAX #15, goal 3.2).
147 ///
148 /// The minimum permitted value is 10, for reasons of implementation convenience and because I
149 /// don’t think there’s any legitimate use case for a smaller value. If you provide a value
150 /// less than ten, you’ll get an empty string back every time.
151 ///
152 /// Truncations are performed at `char` granularity (Unicode scalar value), which means that
153 /// extended grapheme clusters could be broken. This could change in the future (it’ll be an
154 /// optional dependency on `unicode-segmentation`), but for now it was just too much thought.
155 /// If I ever implement this, I’ll probably ditch the minimum value of 10 too.
156 // (Most significantly, it doesn’t play terribly nicely with extension cleverness: six would no
157 // longer be sufficient to guarantee a base name, so more involved calculations and overflow
158 // tracking would need to be done. It’s perfectly achievable, but painful.)
159 pub length_limit: usize,
160
161 /// When allocating the string (since it allocates as small a string as possible), reserve at
162 /// least this many extra bytes. This is good for efficiency when you append the extension
163 /// after sanitisation (in which case, also disable `extension_cleverness`). Default **0**.
164 pub reserve_extra: usize,
165
166 /// Make other options try to be clever about a file extension in the input. Default `true`.
167 ///
168 /// Specifically, if a file extension is detected (done by looking for the last full stop in
169 /// the name, and splitting at that point into base name and extension):
170 ///
171 /// 1. `length_limit` will try to keep the extension intact, truncating the base name rather
172 /// than the extension. “Try”, because if the extension is longer than six code units less
173 /// than the length limit, it will be deemed unsalvageable. (Why six? The base name must
174 /// retain at least one character, so for convenience that’s four UTF-8 code units, plus one
175 /// more for the dot, and if `windows_safe` is on, the longest reserved name causes a five
176 /// code unit base name like `LPT1_`, and ridiculously long extensions are a corner case
177 /// anyway so I decided to just call it a day at six. If I subsequently implement
178 /// grapheme-cluster-aware truncation, this six will increase if the first grapheme cluster
179 /// in the base name is more than five code units long.) An unsalvageable extension is the
180 /// only case where sanitisation may take two steps to quiesce, rather than one: if the
181 /// extension is entirely truncated and the base name contains a dot which in a subsequent
182 /// run will be interpreted as the extension separator, trimming will happen around it on
183 /// that subsequent run but not the first.
184 ///
185 /// 2. `windows_safe` will detect reserved names with extensions.
186 ///
187 /// 3. `trim_spaces_and_full_stops` and `trim_more_punctuation` will trim those characters from
188 /// the end of the base name and the start of the extension, in addition to the start and
189 /// end of the full name. (Expressed otherwise, the base name and extension will be trimmed
190 /// independently.)
191 ///
192 /// If you’re appending the extension after sanitisation, you should turn this to false.
193 pub extension_cleverness: bool,
194
195 /// Remove characters that are not safe on just about any file system. Default `true`, and if
196 /// you actually want to disable it you’re probably using the wrong crate.
197 ///
198 /// This plus `length_limit` is enough to satisfy most platforms other than Windows, though
199 /// cleaning somewhat more is probably a good idea.
200 ///
201 /// Characters removed:
202 ///
203 /// - `/` (slash)
204 /// - ␀ (null, character zero)
205 ///
206 /// Also disallows names comprising exclusively dots (`"."`, `".."`, `"..."`, *&c.*), NOT using
207 /// `replace_with` on them but yielding an empty string.
208 ///
209 /// This is a tiny subset of `windows_safe`.
210 pub most_fs_safe: bool,
211
212 /// Ensure the file name is safe on Windows. Default `true`.
213 ///
214 /// [These are the rules applied:](https://docs.microsoft.com/en-au/windows/win32/fileio/naming-a-file#naming-conventions)
215 ///
216 /// - These characters are removed (and `replace_with` employed):
217 ///
218 /// - `<` (less than)
219 /// - `>` (greater than)
220 /// - `:` (colon)
221 /// - `"` (double quote)
222 /// - `/` (forward slash)
223 /// - `\` (backslash)
224 /// - `|` (vertical bar/pipe)
225 /// - `?` (question mark)
226 /// - `*` (asterisk)
227 /// - The C0 control characters, 0–31 and 127 (U+0000–U+001F, U+007F); note that U+007F isn’t
228 /// actually part of C0, but Microsoft included it in this list so I do too.
229 ///
230 /// - Names must not end with a space or a dot (so these are removed recursively—for reasons of
231 /// technical convenience, `replace_with` is NOT employed).
232 ///
233 /// - These names are reserved (and so a trailing underscore is added to the base name),
234 /// including with an extension if `extension_cleverness` is enabled:
235 ///
236 /// - CON, PRN, AUX, NUL,
237 /// - COM1, COM2, COM3, COM4, COM5, COM6, COM7, COM8, COM9,
238 /// - LPT1, LPT2, LPT3, LPT4, LPT5, LPT6, LPT7, LPT8, and LPT9
239 ///
240 /// Most of these restrictions are actually not quite universal in Windows, but getting around
241 /// them requires switching into POSIX mode or using long UNC paths (e.g. `\\.\C:\CON`,
242 /// `\\?\D:\aux.h`), and your life will certainly be miserable if you try using them; so
243 /// they’re all considered not Windows-safe.
244 pub windows_safe: bool,
245
246 /// Remove characters that may be problematic in the usual places in URLs. Default `false`.
247 ///
248 /// If you want something URL-safe, consider slugifying instead (see below).
249 ///
250 /// This removes any character that is not what’s called a [*URL code point*], also removes the
251 /// characters `&`, `/` and `?`, and forbids the names `.` and `..` which have a special
252 /// meaning in paths. The result is either an empty string, or suitable for use as a path
253 /// component, query string value or fragment, without generally *needing* percent-encoding:
254 /// such a URL will be correctly parsed by a WHATWG URL Standard parser, though nominally
255 /// invalid¹, but older or poorer-quality URL parsers may need percent-encoding to cope with
256 /// the non-ASCII that is retained.
257 ///
258 /// Some notable characters that are removed: `/`, `\`, `%`, `?`, `#`, `&`, `"`, and space.
259 ///
260 /// Almost all non-ASCII is retained.
261 ///
262 /// Notes on using these URLs in some common formats:
263 ///
264 /// - In HTML, no escaping is needed in `<a href="http://www.example/fïle_ñamê">`, because `&`
265 /// and `"` are the only two characters needing escaping in a double-quoted attribute value,
266 /// and both are removed by `url_safe`.
267 ///
268 /// - In plain text formats following the longstanding convention of angle bracket delimition
269 /// (`<http://www.example/lïke_τhis>`), no escaping should be required as `>` is removed by
270 /// `url_safe`. This includes Markdown. However, some such parsers could be stricter about
271 /// what’s allowed inside the angle brackets, so you may need or want to use a URL Standard
272 /// serialiser to do percent-encoding of the non-ASCII.
273 ///
274 /// - In Markdown `[text](href)` links, you’ll want to manually percent-encode `(` to `%28` and
275 /// `)` to `%29`. This is yet another bad choice in Markdown’s technical foundation:
276 /// parentheses aren’t percent-encoded, never have been; so using a URL Standard serialiser
277 /// won’t help you, you’ll instead need to manually encode them, or unpaired parentheses will
278 /// break the link and possibly eat your laundry².
279 ///
280 /// Given that this produces nominally-invalid URLs, you may be wondering why to bother at all;
281 /// it really comes down to characters like `?`, `/` and `#`: you *can* include them in paths
282 /// by percent-encoding, but it’s too likely that *somewhere* along the way, *something* will
283 /// mangle your path, not encoding it properly, and everything will break—basically the entire
284 /// *system* has to process the URL correctly; ever tried a path component containing `%2F`?
285 /// But if you’ve removed the genuinely problematic characters, then in theory things can no
286 /// longer go wrong once you’re past the parser. And being able to skip percent-encoding your
287 /// URLs when you know you’ll be using a proper URL parser is nice.
288 ///
289 /// I deliberately haven’t provided an option for removing characters that would make a URL
290 /// nominally invalid (which is “non-ASCII”), because I think that goes too far: in such a
291 /// case, I don’t think you should *strip* such characters, but rather slugify the whole thing
292 /// (which can do things like `Voilà!` → `voila`).
293 ///
294 /// `replace_with` is used for the character removals, but NOT for the forbidding of the names
295 /// `"."` and `".."`, for which it will instead yield an empty string.
296 ///
297 /// —⁂—
298 ///
299 /// ¹ “Invalid” is just a label in WHATWG specs; it doesn’t change anything, and parsing is
300 /// still well-defined, it’s generally just a hint that either you may have made a mistake,
301 /// or that older tools might not handle this case the same way.
302 ///
303 /// ² When Americans say “eat your laundry” they mean the *clothes*. An Australian seeking to
304 /// express *that* concept would say “eat your washing” (and probably be looked at strangely
305 /// because it’s not an expression in common use). The laundry is the room in which clothes
306 /// are washed; so when I say injection attacks might eat your laundry——
307 ///
308 /// [*URL code point*]: https://url.spec.whatwg.org/#url-code-points
309 pub url_safe: bool,
310
311 /// Replace all sequences of whitespace with one space. Default `true`.
312 ///
313 /// This uses the Unicode `White_Space` property to decide ([`char::is_whitespace`]).
314 ///
315 /// This is done in two phases:
316 ///
317 /// 1. Before safety character replacements, each whitespace character is normalised to a
318 /// U+0020 SPACE; `replace_with` is not invoked.
319 ///
320 /// 2. After all character replacements, adjacent spaces (including any produced by
321 /// `replace_with`, independent of `collapse_replacements`) are collapsed to just one.
322 pub normalise_whitespace: bool,
323
324 /// Remove spaces and full stops (`.`) from the start and end of the name. Default `true`.
325 ///
326 /// `normalise_whitespace` is performed before this; with it on, this will trim all whitespace,
327 /// with it off it’ll only trim U+0020 SPACE.
328 ///
329 /// All things that invoke `replace_with` are performed before this; thus, if you replace a
330 /// character with a space or full stop, that could get trimmed. `replace_with` is not invoked
331 /// on any characters removed by this.
332 ///
333 /// If `extension_cleverness` is enabled (which it is by default), on names with an extension
334 /// this trims from the start and end of the base name and extension independently, rather than
335 /// just the start and end of the full string. That is, `" foo . bar . baz "` will become
336 /// `"foo . bar.baz"` with `extension_cleverness`, and `"foo . bar . baz"` without.
337 ///
338 /// This is independent of `windows_safe`, which also trims trailing spaces and dots from the
339 /// complete name.
340 // BTW: U+002E is named FULL STOP, oh uncouth Americans. 😀
341 pub trim_spaces_and_full_stops: bool,
342
343 /// Remove a few more punctuationy characters from the start and end of the name.
344 /// Default `true`.
345 ///
346 /// This is a more aggressive supplement to `trim_spaces_and_full_stops`, trimming from the
347 /// same places in the same way. These characters are removed:
348 ///
349 /// - `_` (underscore; especially significant because `replace_with` defaults to an underscore)
350 /// - `-` (hyphen/dash/minus)
351 /// - `,` (comma)
352 /// - `;` (semicolon)
353 pub trim_more_punctuation: bool,
354
355 /// Remove control characters. Default `true`.
356 ///
357 /// This removes all characters with the general category *Control*: C0 controls U+0000–U+001F,
358 /// control character U+007F, and C1 controls U+0080–U+009F.
359 ///
360 /// `replace_with` is invoked on these removals.
361 pub remove_control_characters: bool,
362
363 /// Remove BiDi control characters that are relevant to reordering attacks. Default `true`.
364 ///
365 /// <https://trojansource.codes/trojan-source.pdf> is a paper with info about the attack.
366 ///
367 /// This removes U+202A–U+202E and U+2066–U+2069. It does NOT remove the remaining three
368 /// Bidi_Control characters U+061C, U+200E and U+200F (ALM, LRM, RLM),
369 /// which are not implicated in the attack and are conceivably useful in file names.
370 ///
371 /// `replace_with` is invoked on these removals.
372 pub remove_reordering_characters: bool,
373
374 /// Where characters are removed (except as marked), replace them with this.
375 /// Default `Some('_')`.
376 ///
377 /// If you provide a character that would normally be removed, it will not be removed: that
378 /// processing is done once only.
379 ///
380 /// If you provide a character that would be trimmed, it may or may not be trimmed: end matches
381 /// will be trimmed, start matches only will be if ridiculously long names and/or extensions
382 /// force unusual truncation, exposing the start of the string (so that it gets trimmed to
383 /// nothing).
384 pub replace_with: R,
385
386 /// Where multiple adjacent characters are to be replaced, only replace the first, and remove
387 /// any subsequent ones. Default `false`.
388 ///
389 /// See also `normalise_whitespace`, which can collapse replacements if you replace with
390 /// whitespace.
391 pub collapse_replacements: bool,
392
393 /// If sanitisation would leave the path empty, return this string instead. Default `"_"`.
394 ///
395 /// This exists because I found myself writing `if name.is_empty() { name.push('_') }` after
396 /// every time I called `sanitise`. I think most of the time you don’t want to be left with an
397 /// empty string, and inserting *something* is tolerable, so this is on by default as something
398 /// fairly neutral that aligns with the `replace_with` default as well. You can effectively
399 /// disable this by setting this to an empty string.
400 ///
401 /// `length_limit` is not taken into account on this. If you put something ridiculously long in
402 /// it, you brought it on yourself and I wash my hands of it, as Pontius Pilate of old.
403 ///
404 /// (Read *Ruth 3:15–17* from the Bible to understand the name of this option.)
405 pub six_measures_of_barley: &'static str,
406 }
407
408 // Implemented on just one type for inference reasons. One might wonder why I use an associated
409 // constant at all. This would not be an unreasonable thing to wonder.
410 impl Options<Option<char>> {
411 /// The default options. This is more useful than `Options::default()` (which just returns
412 /// this) because it’s const, so you can access `Options::DEFAULT.length_limit` in const
413 /// context.
414 pub const DEFAULT: Self = Options {
415 length_limit: 255,
416 reserve_extra: 0,
417 extension_cleverness: true,
418 most_fs_safe: true,
419 windows_safe: true,
420 url_safe: false,
421 normalise_whitespace: true,
422 trim_spaces_and_full_stops: true,
423 trim_more_punctuation: true,
424 remove_control_characters: true,
425 remove_reordering_characters: true,
426 replace_with: Some('_'),
427 collapse_replacements: false,
428 six_measures_of_barley: "_",
429 };
430 }
431
432 impl Default for Options<Option<char>> {
433 fn default() -> Self {
434 Self::DEFAULT
435 }
436 }
437
438 impl<R: Replace> Options<R> {
439 /// A workaround for an otherwise-messy type situation with filling in defaults.
440 ///
441 /// This solves the problem that you can’t write this:
442 ///
443 /// ```rust,ignore
444 /// Options { replace_with: |c| /* … */, ..Options::DEFAULT }
445 /// ```
446 ///
447 /// … because struct update syntax doesn’t currently allow you to change types, and
448 /// `Options::DEFAULT` is an `Options<Option<char>>`, but with a closure for `replace_with`
449 /// you’re needing to change it to `Options<[closure@…]>`. So instead, write like one of these:
450 ///
451 /// ```rust,ignore
452 /// Options::DEFAULT.with_replace_with(|c| /* … */)
453 /// Options { /* … */, ..Options::DEFAULT }.with_replace_with(|c| /* … */)
454 /// ```
455 ///
456 /// If you’re using nightly rustc, you can try the [incomplete type-changing-struct-update
457 /// feature](https://github.com/rust-lang/rust/issues/86555) instead, which lets the first code
458 /// work (so long as this unstable and incomplete feature is working):
459 ///
460 /// ```rust,ignore
461 /// #![feature(type_changing_struct_update)]
462 /// use sanitise_file_name::Options;
463 ///
464 /// fn main() {
465 /// Options { replace_with: |c| /* … */, ..Options::DEFAULT }
466 /// }
467 /// ```
468 pub fn with_replace_with<R2: Replace>(self, new_replace_with: R2) -> Options<R2> {
469 Options {
470 length_limit: self.length_limit,
471 reserve_extra: self.reserve_extra,
472 extension_cleverness: self.extension_cleverness,
473 most_fs_safe: self.most_fs_safe,
474 windows_safe: self.windows_safe,
475 url_safe: self.url_safe,
476 normalise_whitespace: self.normalise_whitespace,
477 trim_spaces_and_full_stops: self.trim_spaces_and_full_stops,
478 trim_more_punctuation: self.trim_more_punctuation,
479 remove_control_characters: self.remove_control_characters,
480 remove_reordering_characters: self.remove_reordering_characters,
481 replace_with: new_replace_with,
482 collapse_replacements: self.collapse_replacements,
483 six_measures_of_barley: self.six_measures_of_barley,
484 }
485 }
486 }
487
488 /// See [`Options::replace_with`].
489 pub trait Replace {
490 // “Why no *string* replacement?” I hear you ask.
491 // Because then I couldn’t guarantee one allocation.
492 fn replace(&self, char_being_removed: char) -> Option<char>;
493 }
494
495 /// `None`: just remove the character, don’t replace it.
496 /// `Some`: replace the character with this character.
497 impl Replace for Option<char> {
498 fn replace(&self, _: char) -> Option<char> {
499 *self
500 }
501 }
502
503 /// Call this function with the character that is being removed,
504 /// and if it returns a character, replace it with that.
505 impl<F: Fn(char) -> Option<char>> Replace for F {
506 fn replace(&self, c: char) -> Option<char> {
507 self(c)
508 }
509 }
510
511 fn is_most_fs_safe_char(c: char) -> bool {
512 c != '/' && c != '\0'
513 }
514
515 fn is_url_safe_char(c: char) -> bool {
516 // Safe characters are those in the *URL code point* set, minus &, / and ?.
517 //
518 // Definitions from the URL and Infra Standards:
519 //
520 // > The *URL code points* are ASCII alphanumeric, U+0021 (!), U+0024 ($), U+0026 (&),
521 // > U+0027 ('), U+0028 LEFT PARENTHESIS, U+0029 RIGHT PARENTHESIS, U+002A (*), U+002B (+),
522 // > U+002C (,), U+002D (-), U+002E (.), U+002F (/), U+003A (:), U+003B (;), U+003D (=),
523 // > U+003F (?), U+0040 (@), U+005F (_), U+007E (~), and code points in the range U+00A0 to
524 // > U+10FFFD, inclusive, excluding surrogates and noncharacters.
525 //
526 // > A *noncharacter* is a code point that is in the range U+FDD0 to U+FDEF, inclusive, or
527 // > U+FFFE, U+FFFF, U+1FFFE, U+1FFFF, U+2FFFE, U+2FFFF, U+3FFFE, U+3FFFF, U+4FFFE, U+4FFFF,
528 // > U+5FFFE, U+5FFFF, U+6FFFE, U+6FFFF, U+7FFFE, U+7FFFF, U+8FFFE, U+8FFFF, U+9FFFE, U+9FFFF,
529 // > U+AFFFE, U+AFFFF, U+BFFFE, U+BFFFF, U+CFFFE, U+CFFFF, U+DFFFE, U+DFFFF, U+EFFFE, U+EFFFF,
530 // > U+FFFFE, U+FFFFF, U+10FFFE, or U+10FFFF.
531 //
532 // Surrogates are already excluded by the `char` data type.
533
534 matches!(c,
535 'A'..='Z' | 'a'..='z' | '0'..='9' |
536 '!' | '$' | /* '&' deliberately excluded */ '\'' | '(' | ')' | '*' | '+' | ',' | '-' |
537 '.' | /* '/' deliberately excluded */ ':' | ';' | '=' | /* '?' deliberately excluded */
538 '@' | '_' | '~' | '\u{a0}'..='\u{fdcf}' | '\u{fdf0}'..='\u{10fffd}')
539
540 // Exclude the remaining noncharacters U+??FFFE and U+??FFFF:
541 && (c as u32) & 0xfffe != 0xfffe
542 }
543
544 fn is_windows_safe_char(char: char) -> bool {
545 !matches!(char,
546 '<' | '>' | ':' | '"' | '/' | '\\' | '|' | '?' | '*' |
547 '\u{0}'..='\u{1f}' | '\u{7f}')
548 }
549
550 fn is_space_or_full_stop(c: char) -> bool {
551 matches!(c, ' ' | '.')
552 }
553
554 fn is_more_punctuation_character(c: char) -> bool {
555 matches!(c, '_' | '-' | ',' | ';')
556 }
557
558 fn is_reordering_character(c: char) -> bool {
559 matches!(c, '\u{202A}'..='\u{202E}' | '\u{2066}'..='\u{2069}')
560 }
561
562 fn is_reserved_windows_file_name(name: &str) -> bool {
563 matches!(name.as_bytes(),
564 | [b'C' | b'c', b'O' | b'o', b'N' | b'n']
565 | [b'P' | b'p', b'R' | b'r', b'N' | b'n']
566 | [b'A' | b'a', b'U' | b'u', b'X' | b'x']
567 | [b'N' | b'n', b'U' | b'u', b'L' | b'l']
568 | [b'C' | b'c', b'O' | b'o', b'M' | b'm', b'1'..=b'9']
569 | [b'L' | b'l', b'P' | b'p', b'T' | b't', b'1'..=b'9'])
570 }
571
572 /// Split a name on its final '.', returning (base name, extension) if there is one.
573 /// Both could be empty.
574 fn split_extension(input: &str) -> Option<(&str, &str)> {
575 input
576 .as_bytes()
577 .iter()
578 .enumerate()
579 .rev()
580 .find(|(_, c)| **c == b'.')
581 .map(|(dot_index, _)| (&input[..dot_index], &input[dot_index + 1..]))
582 }
583
584 /// Sanitise a file name with the default options.
585 /// See [`Options`] for a description of what all the options do.
586 ///
587 /// The return value should be suitable as a file name, and will not be empty (if it *would* be
588 /// empty, it’ll be `_` instead, per [`Options::six_measures_of_barley`]’s default).
589 #[cfg(feature = "alloc")]
590 #[cfg_attr(docsrs, doc(cfg(feature = "alloc")))]
591 pub fn sanitise(s: &str) -> String {
592 sanitise_with_options(s, &Options::DEFAULT)
593 }
594
595 /// Calculate a sufficient allocation size for the string used. This number will never exceed
596 /// `input.len() + 1 + options.reserve_extra`, and will be less on ridiculously long inputs.
597 ///
598 /// Only intended for use by crazy allocation-counters like me.
599 pub fn sufficient_alloc_size<R: Replace>(input: &str, options: &Options<R>) -> usize {
600 if options.length_limit < 10 {
601 return 0;
602 }
603 if options.extension_cleverness {
604 if let Some((base_name, extension)) = split_extension(input) {
605 let extension_length_limit = options.length_limit - 6;
606 let might_add_underscore = |n| {
607 if (n == 3 || n == 4) && options.windows_safe {
608 n + '_'.len_utf8()
609 } else {
610 n
611 }
612 };
613 return (
614 might_add_underscore(base_name.len()).min(options.length_limit) +
615 '.'.len_utf8() +
616 extension.len().min(extension_length_limit)
617 // No reserve_extra on this side because this is the size needed *while working*,
618 // but reserve_extra is only needed when we’re done.
619 ).max(
620 might_add_underscore(input.len())
621 .min(options.length_limit)
622 .max(options.six_measures_of_barley.len()) +
623 options.reserve_extra
624 )
625 }
626 }
627 (input.len().min(options.length_limit) + if options.windows_safe { '_'.len_utf8() } else { 0 })
628 .max(options.six_measures_of_barley.len()) + options.reserve_extra
629 }
630
631 // Alas, <usize as Ord>::max isn’t const.
632 const fn max(a: usize, b: usize) -> usize { if a > b { a } else { b } }
633
634 macro_rules! max_alloc_size_body {
635 ($options:ident) => {{
636 if $options.length_limit < 10 {
637 return 0;
638 }
639 let baseline = max($options.length_limit, $options.six_measures_of_barley.len())
640 + $options.reserve_extra;
641 if $options.extension_cleverness {
642 let extension_length_limit = $options.length_limit - 6;
643 max($options.length_limit + '.'.len_utf8() + extension_length_limit, baseline)
644 } else {
645 baseline
646 }
647 }}
648 }
649
650 #[cfg(not(feature = "const-fn-trait-bound"))]
651 /// Calculate the maximum allocation size required for a given set of options, to correctly handle
652 /// any input.
653 ///
654 /// This is intended for the scratch buffer approach, where you keep one string around and keep on
655 /// sanitising a whole bunch of inputs into it in turn, or for array-allocated strings like with
656 /// `tinyvec_string`.
657 ///
658 /// This is unfortunately not currently a const fn. If you need a const fn (e.g. to craft an
659 /// precisely-sized `ArrayString`), you may:
660 ///
661 /// 1. Enable the `const-fn-trait-bound` feature on this crate (requires nightly rustc), which will
662 /// change this function to be const, or
663 ///
664 /// 2. Use [`max_alloc_size_const`] instead, which requires `R = Option<char>`.
665 /// (There’s also a `tinyvec_string` usage demonstration there.)
666 pub fn max_alloc_size<R: Replace>(options: &Options<R>) -> usize {
667 max_alloc_size_body!(options)
668 }
669
670 #[cfg(feature = "const-fn-trait-bound")]
671 /// Calculate the maximum allocation size required for a given set of options, to correctly handle
672 /// any input.
673 ///
674 /// This is intended for the scratch buffer approach, where you keep one string around and keep on
675 /// sanitising a whole bunch of inputs into it in turn, or for array-allocated strings like with
676 /// `tinyvec_string`.
677 ///
678 /// This is a const fn because this crate was compiled with the `const-fn-trait-bound` feature
679 /// enabled (which requires nightly rustc at the time of writing).
680 ///
681 /// See also [`max_alloc_size_const`] for an example of using this with `tinyvec_string`.
682 pub const fn max_alloc_size<R: Replace>(options: &Options<R>) -> usize {
683 max_alloc_size_body!(options)
684 }
685
686 /// A `const` variant of [`max_alloc_size`].
687 ///
688 /// Sample usage, combined with `tinyvec_string` (with its `rustc_1_55` feature enabled):
689 ///
690 /// ```rust,ignore
691 /// use tinyvec_string::ArrayString;
692 /// let mut string =
693 /// ArrayString::<[u8; max_alloc_size_const(&Options::DEFAULT)]>::new();
694 /// sanitise_to("input name", &Options::DEFAULT, &mut string);
695 /// ```
696 ///
697 /// Once `const-fn-trait-bound` is stabilised, this method will be deprecated.
698 #[cfg_attr(feature = "const-fn-trait-bound", doc = "\n \
699 Since you compiled this crate with the `const-fn-trait-bound` feature, you don’t need this
700 method. Be cheerful and use `max_alloc_size` instead!")]
701 pub const fn max_alloc_size_const(options: &Options<Option<char>>) -> usize {
702 max_alloc_size_body!(options)
703 }
704
705 /// Sanitise a file name. See [`Options`] for a description of what all the options do.
706 ///
707 /// The return value should be suitable as a file name for the specified options,
708 /// unless it’s empty which can only happen if the option `six_measures_of_barley` is empty (or if
709 /// the `length_limit` option is illegally small, actually).
710 #[cfg(feature = "alloc")]
711 #[cfg_attr(docsrs, doc(cfg(feature = "alloc")))]
712 pub fn sanitise_with_options<R: Replace>(input: &str, options: &Options<R>) -> String {
713 let mut out = String::with_capacity(sufficient_alloc_size(input, options));
714
715 #[cfg(test)]
716 let initial_capacity = out.capacity();
717
718 sanitise_to(input, options, &mut out);
719
720 #[cfg(test)]
721 if initial_capacity != out.capacity() {
722 // I’m serious about this making exactly one allocation. No reallocating allowed.
723 panic!("Capacity changed from {initial_capacity} to {} (on {:?} → {:?})",
724 out.capacity(), input, out);
725 }
726
727 out
728 }
729
730 /// A target for sanitisation: essentially the subset of `String` functionality used.
731 ///
732 /// It might have been nice to use something like `Read + Write + Seek` instead, but the need to
733 /// delete things after writing means that you need still more, and in the end it’s much easier to
734 /// treat it as a string.
735 ///
736 /// I’ve provided implementations for `String` (if the *alloc* feature is enabled, which it is by
737 /// default) and `tinyvec_string::ArrayString` (if the *tinyvec_string* feature is enabled),
738 /// but there’s nothing preventing you from implementing it on other similar string types.
739 pub trait Stringy:
740 Index<Range<usize>, Output = str> +
741 Index<RangeFrom<usize>, Output = str> +
742 Deref<Target = str> +
743 Extend<char>
744 {
745 fn push(&mut self, ch: char);
746 fn push_str(&mut self, string: &str);
747 fn pop(&mut self) -> Option<char>;
748 fn truncate(&mut self, new_len: usize);
749 fn replace_range<R>(&mut self, range: R, replace_with: &str) where R: RangeBounds<usize>;
750 }
751
752 #[cfg(feature = "alloc")]
753 #[cfg_attr(docsrs, doc(cfg(feature = "alloc")))]
754 impl Stringy for String {
755 #[inline] fn push(&mut self, ch: char) { self.push(ch) }
756 #[inline] fn push_str(&mut self, string: &str) { self.push_str(string) }
757 #[inline] fn pop(&mut self) -> Option<char> { self.pop() }
758 #[inline] fn truncate(&mut self, new_len: usize) { self.truncate(new_len) }
759 #[inline] fn replace_range<R>(&mut self, range: R, replace_with: &str)
760 where R: RangeBounds<usize>
761 { self.replace_range(range, replace_with) }
762 }
763
764 #[cfg(feature = "tinyvec_string")]
765 #[cfg_attr(docsrs, doc(cfg(feature = "tinyvec_string")))]
766 impl<A: tinyvec_string::bytearray::ByteArray> Stringy for tinyvec_string::ArrayString<A> {
767 #[inline] fn push(&mut self, ch: char) { self.push(ch) }
768 #[inline] fn push_str(&mut self, string: &str) { self.push_str(string) }
769 #[inline] fn pop(&mut self) -> Option<char> { self.pop() }
770 #[inline] fn truncate(&mut self, new_len: usize) { self.truncate(new_len) }
771 #[inline] fn replace_range<R>(&mut self, range: R, replace_with: &str)
772 where R: RangeBounds<usize>
773 { self.replace_range(range, replace_with) }
774 }
775
776 /// Sanitise a file name into an existing `String`. Intended for power users only.
777 ///
778 /// When you use [`sanitise`] or [`sanitise_with_options`], the perfect allocation is artisanally
779 /// crafted (or something). If you use this carelessly, you may actually cause *more* allocations
780 /// to be made, rather than less. You may therefore wish to use [`sufficient_alloc_size`] in some
781 /// cases to calculate how much more to reserve ahead of time.
782 ///
783 /// See [`Options`] for a description of what all the options do.
784 ///
785 /// After calling this, `out` will be the same length or longer, never shorter. If you want to know
786 /// *how much* longer, store and compare the length yourself.
787 pub fn sanitise_to<R: Replace, S: Stringy>(input: &str, options: &Options<R>, out: &mut S) {
788 let protected = out.len();
789 // I said in the docs don’t set it to less than 10, but without this zero leads to some
790 // unreachable!() being reached, which is æsthetically displeasing, so I’m just going to return
791 // empty strings for unreasonably small length limits. 🙂
792 if options.length_limit < 10 {
793 return;
794 }
795
796 // When label-break-value stabilises I’ll switch to that, but until then, loop it is.
797 #[allow(clippy::never_loop)]
798 loop { // breaks after exactly one iteration.
799 if options.extension_cleverness {
800 if let Some((base_name, extension)) = split_extension(input) {
801 // With extension-awareness, when the path exceeds length_limit, we prefer
802 // to truncate from the base name rather than from the extension. But we don’t
803 // know how much we’ll need to truncate until we’ve finished processing the
804 // extension, so we have to allocate a maximum of roughly twice as much as
805 // we’ll end up needing.
806 //
807 // For implementation convenience in this corner case, we’ve declared a maximum
808 // extension length of six less than length_limit (explained on
809 // Options::extension_cleverness).
810 let extension_length_limit = options.length_limit - 6;
811 sanitise_part(base_name, options, options.length_limit, false, out, protected);
812 let base_len = out.len() - protected;
813 out.push('.');
814 let extension_truncated = sanitise_part(
815 extension,
816 options,
817 extension_length_limit,
818 true,
819 out,
820 // It’s OK for trimming to take out the entire file name (so we’re deliberately
821 // not including base_len in this), but we mustn’t touch what’s not ours.
822 protected,
823 );
824
825 let mut total_len = out.len() - protected;
826
827 if total_len > options.length_limit {
828 if extension_truncated {
829 // Extension is unsalvageable: truncate from the end.
830 while total_len > options.length_limit {
831 match out.pop() {
832 Some(c) => total_len -= c.len_utf8(),
833 None => unreachable!(),
834 }
835 }
836 // Length is now acceptable, but that could have left us with
837 // undesirable trailing characters, so run trim again.
838 out.truncate(protected + trim_end(&out[protected..], options, true).len());
839 } else {
840 // (Sigh. Whose brilliant idea was it to try to preserve extensions anyway?
841 // Maybe if I’d realised the pain it’d cause I wouldn’t have bothered.
842 // It’s not like anyone *else* does it. But it is good, say I. And now I’m
843 // even mulling over grapheme-cluster-aware truncation. Am I mad?)
844 let base_name_end_index = base_len;
845 let mut base_chars = out[protected..protected + base_name_end_index].chars();
846 while total_len > options.length_limit {
847 match base_chars.next_back() {
848 Some(c) => {
849 total_len -= c.len_utf8();
850 }
851 None => unreachable!(),
852 }
853 }
854 let base_name = trim_end(
855 &out[protected..protected + base_chars.as_str().len()],
856 options,
857 false,
858 );
859 let range = protected + base_name.len()..protected + base_name_end_index;
860 out.replace_range(range, "");
861 }
862 }
863
864 break;
865 }
866 }
867 // Extension cleverness disabled, or no extension found: the much simpler path!
868 sanitise_part(input, options, options.length_limit, false, out, protected);
869 break;
870 }
871
872 // Finally one last bit of processing: checking names that are all dots
873 // (though normally windows_safe will already have truncated them to zero).
874 if (options.url_safe && (&out[protected..] == "." || &out[protected..] == "..")) ||
875 (options.most_fs_safe && out[protected..].chars().all(|c| c == '.'))
876 {
877 out.truncate(protected);
878 }
879
880 if out[protected..].is_empty() {
881 out.push_str(options.six_measures_of_barley);
882 }
883 }
884
885 fn sanitise_part<R: Replace, S: Stringy>(
886 input: &str,
887 options: &Options<R>,
888 length_limit: usize,
889 is_extension: bool,
890 out: &mut S,
891 protected: usize,
892 ) -> bool {
893 let mut len = 0;
894 let mut did_truncate = false;
895
896 let mut last_was_remove = false;
897 let mut last_was_whitespace = false;
898 out.extend(input.chars()
899 .map(|mut c| {
900 c = if options.normalise_whitespace && c.is_whitespace() { ' ' } else { c };
901 (c,
902 (options.most_fs_safe && !is_most_fs_safe_char(c)) ||
903 (options.windows_safe && !is_windows_safe_char(c)) ||
904 (options.url_safe && !is_url_safe_char(c)) ||
905 (options.remove_control_characters && c.is_control()) ||
906 (options.remove_reordering_characters && is_reordering_character(c))
907 )
908 })
909 .filter_map(|(c, remove)| {
910 if options.collapse_replacements {
911 if remove && last_was_remove {
912 return None;
913 }
914 last_was_remove = remove;
915 }
916 if remove { options.replace_with.replace(c) } else { Some(c) }
917 .filter(|&c| {
918 if options.normalise_whitespace {
919 let is_whitespace = c == ' ';
920 let drop = last_was_whitespace && is_whitespace;
921 last_was_whitespace = is_whitespace;
922 !drop
923 } else {
924 true
925 }
926 })
927 })
928 .skip_while(|&c| {
929 (options.trim_spaces_and_full_stops && is_space_or_full_stop(c)) ||
930 (options.trim_more_punctuation && is_more_punctuation_character(c))
931 })
932 .take_while(|&c| {
933 let new_len = len + c.len_utf8();
934 if new_len <= length_limit {
935 len = new_len;
936 true
937 } else {
938 did_truncate = true;
939 false
940 }
941 })
942 );
943
944 if len > 0 {
945 // We’ve added something non-trimmed, that’ll guard the potential reserved name underscore.
946 out.truncate(protected + trim_end(&out[protected..], options, is_extension).len());
947 }
948
949 if !is_extension && options.windows_safe && is_reserved_windows_file_name(&out[protected..]) {
950 // This underscore looks to be in danger of being end-trimmed,
951 // but in practice we’ve ensured that it won’t be
952 // (except maybe one case with a lower length_limit than permitted).
953 out.push('_');
954 }
955
956 // Whew. Finally done. Breathe a sigh of relief.
957 did_truncate
958 }
959
960 fn trim_end<'a, R: Replace>(out: &'a str, options: &Options<R>, is_extension: bool) -> &'a str {
961 let trim_space_or_full_stop = options.trim_spaces_and_full_stops ||
962 ((is_extension || !options.extension_cleverness) && options.windows_safe);
963 out.trim_end_matches(|c| {
964 (trim_space_or_full_stop && is_space_or_full_stop(c)) ||
965 (options.trim_more_punctuation && is_more_punctuation_character(c))
966 })
967 }
968
969 // A concession to those poor Americans et al. 😀
970 #[cfg(feature = "alloc")]
971 #[cfg_attr(docsrs, doc(cfg(feature = "alloc")))]
972 pub use sanitise as sanitize;
973 #[cfg(feature = "alloc")]
974 #[cfg_attr(docsrs, doc(cfg(feature = "alloc")))]
975 pub use sanitise_with_options as sanitize_with_options;
976 pub use sanitise_to as sanitize_to;
977
978 // How did this get to almost a thousand lines by this point? I’m sure I started out with only
979 // twenty or so. But then I got careful about allocations, and added extension cleverness, and
980 // added more features, and documented exhaustingly, and oops, a thousand lines, lines that are
981 // convoluted at times. Well, I succeeded in all my *functional* goals, with better precision,
982 // theoretically better but untested speed, better behaviour around extensions, single-allocation
983 // and even *no*-allocation operation; but utterly lost sight of simple and obviously-correct code.
984 // Was it worth it? Eh, probably.
985
986 // --- Tests ---
987
988 #[cfg(feature = "alloc")]
989 #[test]
990 fn test_length_limit_things() {
991 // I wrote these tests before I wrote the matrix. I might as well delete them, but I haven’t.
992
993 let short = Options {
994 length_limit: 10,
995 ..Options::DEFAULT
996 };
997
998 assert_eq!(sanitise_with_options("abcdef.ghij", &short), "abcde.ghij");
999
1000 // Unsalvageable extension
1001 assert_eq!(sanitise_with_options("abcde.fghij", &short), "abcde.fghi");
1002
1003 // Windows reserved name protection
1004 assert_eq!(sanitise_with_options("AUX.abcdef", &short), "AUX_.abcd");
1005 assert_eq!(sanitise_with_options("AUX.abcdef", &Options { windows_safe: false, ..short }),
1006 "AUX.abcd");
1007 assert_eq!(sanitise_with_options("lpT7.abcdef", &short), "lpT7_.abcd");
1008 assert_eq!(sanitise_with_options("cOm6.abcdef", &Options { windows_safe: false, ..short }),
1009 "cOm6.abcd");
1010
1011 assert_eq!(sanitise("CON"), "CON_");
1012 assert_eq!(sanitise("aux.h"), "aux_.h");
1013 assert_eq!(sanitise("Lpt1.exe"), "Lpt1_.exe");
1014 assert_eq!(sanitise("xyz"), "xyz");
1015 assert_eq!(sanitise(""), "_");
1016 assert_eq!(sanitise("nül"), "nül");
1017 assert_eq!(sanitise("COM1.jpg.png"), "COM1.jpg.png");
1018 }
1019
1020 #[cfg(feature = "alloc")]
1021 #[test]
1022 fn matrix() {
1023 // Look, I know I said I didn’t want std, but I *need* it for these tests, y’see?
1024 #[cfg(not(feature = "std"))]
1025 extern crate std;
1026 use std::prelude::rust_2021::*;
1027 use std::fmt::Write;
1028 use std::{eprintln, println, format, vec};
1029
1030 fn case<
1031 R: Replace,
1032 #[cfg(feature = "tinyvec_string")]
1033 A: tinyvec_string::bytearray::ByteArray,
1034 >(
1035 set_name: &'static str,
1036 options_name: &'static str,
1037 options: &Options<R>,
1038 paths: &mut Vec<String>,
1039 unsteady_state: &mut Vec<(&'static str, &'static str, String, String, String)>,
1040 // Apparently you can’t do `case::<#[cfg] A>()`, so we have to do this instead.
1041 #[cfg(feature = "tinyvec_string")]
1042 _: std::marker::PhantomData<A>,
1043 ) {
1044 println!("Sanitising {set_name} with options {options_name}");
1045 #[cfg(feature = "tinyvec_string")]
1046 let mut array_string = tinyvec_string::ArrayString::<A>::new();
1047 let mut sanitised = String::new();
1048 let mut capacity = String::new();
1049 let mut scratch = if options_name == "passthrough" {
1050 // “memory allocation of 18446744073709551615 bytes failed” 😀
1051 String::new()
1052 } else {
1053 String::with_capacity(max_alloc_size(options) + 1)
1054 };
1055 let scratch_size = scratch.capacity();
1056 for input in std::fs::read_to_string(format!("tests/{set_name}.txt")).unwrap().lines() {
1057 let output = sanitise_with_options(input, options);
1058 // A couple of sanity checks make sense here.
1059 if output.len() > options.length_limit {
1060 panic!(
1061 "Input {input} sanitised to {output}, which at {len} is greater than the allowed {max}",
1062 len = output.len(),
1063 max = options.length_limit,
1064 );
1065 }
1066 if options.windows_safe && is_reserved_windows_file_name(
1067 options.extension_cleverness.then(|| &*output)
1068 .and_then(split_extension)
1069 .map(|(base, _)| base)
1070 .unwrap_or(&output)
1071 ) {
1072 panic!("Input {input} sanitised to {output}, which is a reserved Windows file name");
1073 }
1074
1075 // And ensure sanitise_to is working properly also.
1076 scratch.truncate(0);
1077 scratch.push('.'); // A trimmable character, and not six_measures_of_barley.
1078 sanitise_to(input, options, &mut scratch);
1079 assert_eq!(scratch.chars().next(), Some('.'));
1080 assert_eq!(scratch[1..], output);
1081 #[cfg(feature = "tinyvec_string")]
1082 {
1083 if array_string.capacity() > 0 {
1084 array_string.truncate(0);
1085 array_string.push(' '); // A trimmable character, and not six_measures_of_barley.
1086 sanitise_to(input, options, &mut array_string);
1087 assert_eq!(array_string.chars().next(), Some(' '));
1088 assert_eq!(array_string[1..], output);
1089 }
1090 }
1091
1092 sanitised.push_str(&output);
1093 sanitised.push('\n');
1094 let _ = writeln!(capacity, "{}", output.capacity());
1095 if input != output {
1096 if options_name == "passthrough" {
1097 unsteady_state.push(
1098 (set_name, options_name, input.to_owned(), output.clone(), output),
1099 );
1100 } else {
1101 let repeated = sanitise_with_options(&output, options);
1102 if repeated != output {
1103 sanitised.push_str("⚠ Sanitisation did not reach a steady state. Next line shows the effect of resanitising the line above. ⚠\n");
1104 sanitised.push_str(&repeated);
1105 sanitised.push('\n');
1106 unsteady_state.push(
1107 (set_name, options_name, input.to_owned(), output, repeated),
1108 );
1109 }
1110 }
1111 }
1112 }
1113 let sanitised_name = format!("tests/{set_name}.{options_name}.sanitised");
1114 let capacity_name = format!("tests/{set_name}.{options_name}.capacity");
1115 std::fs::write(&sanitised_name, sanitised).unwrap();
1116 std::fs::write(&capacity_name, capacity).unwrap();
1117 paths.push(sanitised_name);
1118 paths.push(capacity_name);
1119 if options_name != "passthrough" {
1120 assert_eq!(scratch_size, scratch.capacity(), "scratch buffer reallocated");
1121 }
1122 }
1123
1124 let mut unsteady_state = vec![];
1125 let mut paths = vec![];
1126 let d = Options::DEFAULT;
1127 for name in ["blns", "misc"] {
1128 macro_rules! case {
1129 // On $array_size: I tried using roughly { max_alloc_size_const(&options) + 1 },
1130 // but threading it all through was just too painful, especially in the absence of
1131 // const-fn-trait-bound. So I’ll just do one separate test for that.
1132 ($array_size:literal, $options_name:expr, $options:expr) => {
1133 let options = &$options;
1134 // +1 for the ' ' we prefix.
1135 let required_size = max_alloc_size(options).saturating_add(1);
1136 assert!($array_size == 0 || required_size <= $array_size,
1137 "Test case design error: array being given {} bytes, but {} are needed",
1138 $array_size,
1139 required_size
1140 );
1141 case(
1142 name,
1143 $options_name,
1144 options,
1145 &mut paths,
1146 &mut unsteady_state,
1147 // TODO: after https://github.com/ThatsNoMoon/tinyvec_string/issues/3 is
1148 // resolved, ditch $array_size and use max_alloc_size instead.
1149 #[cfg(feature = "tinyvec_string")]
1150 std::marker::PhantomData::<[u8; $array_size]>,
1151 );
1152 }
1153 }
1154 // Assumption that I decline to “test” because it’d be silly:
1155 // sanitise(…) == sanitise_with_options(…, &Options::DEFAULT).
1156 case!(512, "default", d);
1157 case!(512, "realistic-length_limit-reduction", Options { length_limit: Options::DEFAULT.length_limit - 4, ..d });
1158 case!(512, "url_safe", Options { url_safe: true, ..d });
1159 case!(512, "silly-replace_with", Options::DEFAULT.with_replace_with(|c| char::from_u32(c as u32 + 1)));
1160 case!(512, "no-windows_safe", Options { windows_safe: false, ..d });
1161 case!(256, "no-extension_cleverness", Options { extension_cleverness: false, ..d });
1162 // 10 + 1
1163 case!(11, "short-sans-extension_cleverness", Options { length_limit: 10, extension_cleverness: false, ..d });
1164 // 15 + 1
1165 case!(16, "short", Options { length_limit: 10, ..d });
1166 case!(0, "passthrough", Options {
1167 length_limit: usize::MAX,
1168 reserve_extra: 0,
1169 extension_cleverness: false,
1170 most_fs_safe: false,
1171 windows_safe: false,
1172 url_safe: false,
1173 normalise_whitespace: false,
1174 trim_spaces_and_full_stops: false,
1175 trim_more_punctuation: false,
1176 remove_control_characters: false,
1177 remove_reordering_characters: false,
1178 replace_with: None,
1179 collapse_replacements: false,
1180 six_measures_of_barley: "",
1181 });
1182 macro_rules! case_only {
1183 ($option:ident) => {{
1184 let mut options = Options {
1185 most_fs_safe: false,
1186 windows_safe: false,
1187 url_safe: false,
1188 normalise_whitespace: false,
1189 trim_spaces_and_full_stops: false,
1190 trim_more_punctuation: false,
1191 remove_control_characters: false,
1192 remove_reordering_characters: false,
1193 ..d
1194 };
1195 options.$option = true;
1196 case!(512, concat!("just-", stringify!($option)), options);
1197 }}
1198 }
1199 case_only!(most_fs_safe);
1200 case_only!(windows_safe);
1201 case_only!(url_safe);
1202 case_only!(normalise_whitespace);
1203 case_only!(trim_spaces_and_full_stops);
1204 case_only!(trim_more_punctuation);
1205 case_only!(remove_control_characters);
1206 case_only!(remove_reordering_characters);
1207 // Eh, I’m bored now. That’ll do.
1208 }
1209
1210 let mut complain_of_unsteady_states = false;
1211 if !unsteady_state.is_empty() {
1212 for (set, options, original, first, second) in &unsteady_state {
1213 match (*set, *options, &**original, &**first, &**second) {
1214 ("blns", "short", "Dr. Herman I. Libshitz", "Dr. Herman", "Dr.Herm") |
1215 ("blns", "short", r#"{{ "".__class__.__mro__[2].__subclasses__()[40]("/etc/passwd").read() }}"#, "{{ __.__cl", "{{.cl") => {
1216 // Skip known cases of unsalvageable extensions combining with dots in the base
1217 // name to effectively give a new extension, making quiescence take two steps.
1218 // Making these steady-state would take too much effort, and the harm is
1219 // minimal (the unsteady state is still a correctly sanitised name).
1220 },
1221 (_, "silly-replace_with", _, _, _) => {
1222 // Certainly this one isn’t steady-state!
1223 },
1224 (_, "passthrough", _, _, _) => {
1225 complain_of_unsteady_states = true;
1226 eprintln!("Unsteady state in {set} with {options} options, diff tests/{set}.{options}.txt and tests/{set}.{options}.sanitised");
1227 },
1228 _ => {
1229 complain_of_unsteady_states = true;
1230 eprintln!("Unknown unsteady state in {set} with {options} options, look for the ⚠ symbol in tests/{set}.{options}.sanitised");
1231 },
1232 }
1233 }
1234 }
1235
1236 if !std::process::Command::new("git")
1237 .arg("diff")
1238 .arg("--exit-code")
1239 .arg("--text")
1240 .args(&paths)
1241 .status()
1242 .unwrap()
1243 .success()
1244 {
1245 panic!("sanitisation produced different results than are known, review the diffs");
1246 }
1247
1248 // A guard against committing an unsteady state.
1249 if complain_of_unsteady_states {
1250 panic!("Some sanitisations unexpectedly failed to reach a steady state.");
1251 }
1252 }
1253
1254 #[cfg(feature = "tinyvec_string")]
1255 #[test]
1256 #[should_panic]
1257 fn test_tinyvec_string_panic() {
1258 // I’ve already tested various normal cases, including that ridiculously long strings don’t
1259 // cause overflow on moderately limited arrays with moderate length limits; but I haven’t
1260 // demonstrated the panic that occurs if the array is too short. So here’s this now. 🙂
1261 sanitise_to(
1262 "Watch me panic!",
1263 &Options::DEFAULT,
1264 &mut tinyvec_string::ArrayString::<[u8; 12]>::new(),
1265 );
1266 }
1267
1268 #[cfg(feature = "tinyvec_string")]
1269 #[test]
1270 fn test_tinyvec_string_max_alloc_size() {
1271 use tinyvec_string::ArrayString;
1272 // Note: this is *currently* 505, but I declare that not part of the compatibility contract;
1273 // extension cleverness/grapheme cluster changes could lead to it increasing to 510.
1274 let mut string: ArrayString<[u8; 505]> =
1275 ArrayString::<[u8; max_alloc_size_const(&Options::DEFAULT)]>::new();
1276 assert_eq!(string.capacity(), 505);
1277 sanitise_to(
1278 "Watch everything being hunky dory, even when I throw unreasonably long values at it all. \
1279 Even when dots become extension separators; yes, even then. Then further: into the deep \
1280 reaches of testing, where things start to get garbled, and having written at least 255 \
1281 characters, I have to now write just as much of extension—horror. But that was a full \
1282 stop so that this could now be the extension, and I can’t put a dot for the next while› \
1283 ¿Whatever will I do? I suspect things are getting out of hand here, but I can’t stop now; \
1284 I’m running out of things to write, but it should be enough by now!",
1285 &Options::DEFAULT,
1286 &mut string,
1287 );
1288 assert_eq!(string, "Watch everything being hunky dory, even when I throw unreasonably long \
1289 values at it all. Even when dots become extension separators; yes, even then. Then \
1290 further_ into the deep reaches of testing, where things start to get garbled, and having \
1291 written at l");
1292 }