logos/
source.rs

1//! This module contains a bunch of traits necessary for processing byte strings.
2//!
3//! Most notable are:
4//! * `Source` - implemented by default for `&str`, `&[u8]` and wrapper types, used by the `Lexer`.
5//! * `Slice` - slices of `Source`, returned by `Lexer::slice`.
6
7use core::fmt::Debug;
8use core::ops::{Deref, Range};
9
10/// Trait for types the `Lexer` can read from.
11///
12/// Most notably this is implemented for `&str`. It is unlikely you will
13/// ever want to use this Trait yourself, unless implementing a new `Source`
14/// the `Lexer` can use.
15///
16/// SAFETY: Unless the unsafe functions of this trait are disabled with the `forbid_unsafe`
17/// feature, the correctness of the unsafe functions of this trait depend on the correct
18/// implementation of the `len` and `find_boundary` functions so generated code does not request
19/// out-of-bounds access.
20#[allow(clippy::len_without_is_empty)]
21pub trait Source {
22    /// A type this `Source` can be sliced into.
23    type Slice<'a>: PartialEq + Eq + Debug
24    where
25        Self: 'a;
26
27    /// Length of the source
28    fn len(&self) -> usize;
29
30    /// Read a chunk of bytes into an array. Returns `None` when reading
31    /// out of bounds would occur.
32    ///
33    /// This is very useful for matching fixed-size byte arrays, and tends
34    /// to be very fast at it too, since the compiler knows the byte lengths.
35    ///
36    /// ```rust
37    /// use logos::Source;
38    ///
39    /// let foo = "foo";
40    ///
41    /// assert_eq!(foo.read(0), Some(b"foo"));     // Option<&[u8; 3]>
42    /// assert_eq!(foo.read(0), Some(b"fo"));      // Option<&[u8; 2]>
43    /// assert_eq!(foo.read(2), Some(b'o'));       // Option<u8>
44    /// assert_eq!(foo.read::<&[u8; 4]>(0), None); // Out of bounds
45    /// assert_eq!(foo.read::<&[u8; 2]>(2), None); // Out of bounds
46    /// ```
47    fn read<'a, Chunk>(&'a self, offset: usize) -> Option<Chunk>
48    where
49        Chunk: self::Chunk<'a>;
50
51    /// Get a slice of the source at given range. This is analogous to
52    /// `slice::get(range)`.
53    ///
54    /// ```rust
55    /// use logos::Source;
56    ///
57    /// let foo = "It was the year when they finally immanentized the Eschaton.";
58    /// assert_eq!(<str as Source>::slice(&foo, 51..59), Some("Eschaton"));
59    /// ```
60    fn slice(&self, range: Range<usize>) -> Option<Self::Slice<'_>>;
61
62    /// Get a slice of the source at given range. This is analogous to
63    /// `slice::get_unchecked(range)`.
64    ///
65    /// # Safety
66    ///
67    /// Range should not exceed bounds.
68    ///
69    /// ```rust
70    /// use logos::Source;
71    ///
72    /// let foo = "It was the year when they finally immanentized the Eschaton.";
73    ///
74    /// unsafe {
75    ///     assert_eq!(<str as Source>::slice_unchecked(&foo, 51..59), "Eschaton");
76    /// }
77    /// ```
78    #[cfg(not(feature = "forbid_unsafe"))]
79    unsafe fn slice_unchecked(&self, range: Range<usize>) -> Self::Slice<'_>;
80
81    /// For `&str` sources attempts to find the closest `char` boundary at which source
82    /// can be sliced, starting from `index`.
83    ///
84    /// For binary sources (`&[u8]`) this should just return `index` back.
85    #[inline]
86    fn find_boundary(&self, index: usize) -> usize {
87        index
88    }
89
90    /// Check if `index` is valid for this `Source`, that is:
91    ///
92    /// + It's not larger than the byte length of the `Source`.
93    /// + (`str` only) It doesn't land in the middle of a UTF-8 code point.
94    fn is_boundary(&self, index: usize) -> bool;
95}
96
97impl Source for str {
98    type Slice<'a> = &'a str;
99
100    #[inline]
101    fn len(&self) -> usize {
102        self.len()
103    }
104
105    #[inline]
106    fn read<'a, Chunk>(&'a self, offset: usize) -> Option<Chunk>
107    where
108        Chunk: self::Chunk<'a>,
109    {
110        #[cfg(not(feature = "forbid_unsafe"))]
111        if offset + (Chunk::SIZE - 1) < self.len() {
112            // # Safety: we just performed a bound check.
113            Some(unsafe { Chunk::from_ptr(self.as_ptr().add(offset)) })
114        } else {
115            None
116        }
117
118        #[cfg(feature = "forbid_unsafe")]
119        Chunk::from_slice(self.as_bytes().slice(offset..Chunk::SIZE + offset)?)
120    }
121
122    #[inline]
123    fn slice(&self, range: Range<usize>) -> Option<&str> {
124        self.get(range)
125    }
126
127    #[cfg(not(feature = "forbid_unsafe"))]
128    #[inline]
129    unsafe fn slice_unchecked(&self, range: Range<usize>) -> &str {
130        debug_assert!(
131            range.start <= self.len() && range.end <= self.len(),
132            "Reading out of bounds {:?} for {}!",
133            range,
134            self.len()
135        );
136
137        self.get_unchecked(range)
138    }
139
140    #[inline]
141    fn find_boundary(&self, mut index: usize) -> usize {
142        while !self.is_char_boundary(index) {
143            index += 1;
144        }
145
146        index
147    }
148
149    #[inline]
150    fn is_boundary(&self, index: usize) -> bool {
151        self.is_char_boundary(index)
152    }
153}
154
155impl Source for [u8] {
156    type Slice<'a> = &'a [u8];
157
158    #[inline]
159    fn len(&self) -> usize {
160        self.len()
161    }
162
163    #[inline]
164    fn read<'a, Chunk>(&'a self, offset: usize) -> Option<Chunk>
165    where
166        Chunk: self::Chunk<'a>,
167    {
168        #[cfg(not(feature = "forbid_unsafe"))]
169        if offset + (Chunk::SIZE - 1) < self.len() {
170            Some(unsafe { Chunk::from_ptr(self.as_ptr().add(offset)) })
171        } else {
172            None
173        }
174
175        #[cfg(feature = "forbid_unsafe")]
176        Chunk::from_slice(self.slice(offset..Chunk::SIZE + offset)?)
177    }
178
179    #[inline]
180    fn slice(&self, range: Range<usize>) -> Option<&[u8]> {
181        self.get(range)
182    }
183
184    #[cfg(not(feature = "forbid_unsafe"))]
185    #[inline]
186    unsafe fn slice_unchecked(&self, range: Range<usize>) -> &[u8] {
187        debug_assert!(
188            range.start <= self.len() && range.end <= self.len(),
189            "Reading out of bounds {:?} for {}!",
190            range,
191            self.len()
192        );
193
194        self.get_unchecked(range)
195    }
196
197    #[inline]
198    fn is_boundary(&self, index: usize) -> bool {
199        index <= self.len()
200    }
201}
202
203impl<T> Source for T
204where
205    T: Deref,
206    <T as Deref>::Target: Source,
207{
208    type Slice<'a>
209        = <T::Target as Source>::Slice<'a>
210    where
211        T: 'a;
212
213    fn len(&self) -> usize {
214        self.deref().len()
215    }
216
217    fn read<'a, Chunk>(&'a self, offset: usize) -> Option<Chunk>
218    where
219        Chunk: self::Chunk<'a>,
220    {
221        self.deref().read(offset)
222    }
223
224    fn slice(&self, range: Range<usize>) -> Option<Self::Slice<'_>> {
225        self.deref().slice(range)
226    }
227
228    #[cfg(not(feature = "forbid_unsafe"))]
229    unsafe fn slice_unchecked(&self, range: Range<usize>) -> Self::Slice<'_> {
230        self.deref().slice_unchecked(range)
231    }
232
233    fn is_boundary(&self, index: usize) -> bool {
234        self.deref().is_boundary(index)
235    }
236
237    fn find_boundary(&self, index: usize) -> usize {
238        self.deref().find_boundary(index)
239    }
240}
241
242/// A fixed, statically sized chunk of data that can be read from the `Source`.
243///
244/// This is implemented for `u8`, as well as byte arrays `&[u8; 1]` to `&[u8; 32]`.
245pub trait Chunk<'source>: Sized + Copy + PartialEq + Eq {
246    /// Size of the chunk being accessed in bytes.
247    const SIZE: usize;
248
249    /// Create a chunk from a raw byte pointer.
250    ///
251    /// # Safety
252    ///
253    /// Raw byte pointer should point to a valid location in source.
254    #[cfg(not(feature = "forbid_unsafe"))]
255    unsafe fn from_ptr(ptr: *const u8) -> Self;
256
257    /// Create a chunk from a slice.
258    /// Returns None if the slice is not long enough to produce the chunk.
259    #[cfg(feature = "forbid_unsafe")]
260    fn from_slice(s: &'source [u8]) -> Option<Self>;
261}
262
263#[allow(clippy::needless_lifetimes)]
264impl<'source> Chunk<'source> for u8 {
265    const SIZE: usize = 1;
266
267    #[inline]
268    #[cfg(not(feature = "forbid_unsafe"))]
269    unsafe fn from_ptr(ptr: *const u8) -> Self {
270        *ptr
271    }
272
273    #[inline]
274    #[cfg(feature = "forbid_unsafe")]
275    fn from_slice(s: &'source [u8]) -> Option<Self> {
276        s.first().copied()
277    }
278}
279
280impl<'source, const N: usize> Chunk<'source> for &'source [u8; N] {
281    const SIZE: usize = N;
282
283    #[inline]
284    #[cfg(not(feature = "forbid_unsafe"))]
285    unsafe fn from_ptr(ptr: *const u8) -> Self {
286        &*(ptr as *const [u8; N])
287    }
288
289    #[inline]
290    #[cfg(feature = "forbid_unsafe")]
291    fn from_slice(s: &'source [u8]) -> Option<Self> {
292        s.slice(0..Self::SIZE).and_then(|x| x.try_into().ok())
293    }
294}
logos/source.rs

logos/
source.rs
