lsp_core/util/
token.rs

1use std::ops::Range;
2
3use bevy_ecs::prelude::*;
4use derive_more::{AsMut, AsRef, Deref, DerefMut, IsVariant};
5use tracing::{debug, instrument};
6
7use crate::{components::*, lang::TokenTrait, prelude::*};
8
9/// [`Component`] used to indicate the currently targeted [`Token`] during a request.
10#[derive(Component, Debug)]
11pub struct TokenComponent {
12    pub token: Spanned<Token>,
13    pub range: crate::lsp_types::Range,
14    pub text: String,
15}
16
17/// [`Component`] that contains the parsed tokens.
18///
19/// [`crate`] defines systems (like [`get_current_token`]) that depend
20/// on Tokens to deduce [`TokenComponent`] during the [`CompletionLabel`] schedule.
21#[derive(Component, AsRef, Deref, AsMut, DerefMut, Debug)]
22pub struct Tokens(pub Vec<Spanned<Token>>);
23
24#[instrument(skip(query, commands))]
25pub fn get_current_token(
26    mut query: Query<(Entity, &Tokens, &PositionComponent, &RopeC, &DynLang)>,
27    mut commands: Commands,
28) {
29    for (entity, tokens, position, rope, helper) in &mut query {
30        commands.entity(entity).remove::<TokenComponent>();
31        let Some(offset) = position_to_offset(position.0, &rope.0) else {
32            debug!("Couldn't transform to an offset ({:?})", position.0);
33            continue;
34        };
35
36        let Some(token) = tokens
37            .0
38            .iter()
39            .filter(|x| x.span().contains(&offset))
40            .min_by_key(|x| x.span().end - x.span().start)
41        else {
42            let closest = tokens.0.iter().min_by_key(|x| {
43                let start = if offset > x.span().start {
44                    offset - x.span().start
45                } else {
46                    x.span().start - offset
47                };
48
49                let end = if offset > x.span().end {
50                    offset - x.span().end
51                } else {
52                    x.span().end - offset
53                };
54
55                if start > end {
56                    end
57                } else {
58                    start
59                }
60            });
61            debug!(
62                "Failed to find a token, offset {} closest {:?}",
63                offset, closest
64            );
65            continue;
66        };
67
68        let (text, range) = helper.get_relevant_text(token, rope);
69        let Some(range) = range_to_range(&range, &rope.0) else {
70            debug!("Failed to transform span to range");
71            continue;
72        };
73
74        debug!("Current token {:?} {}", token, text);
75        commands.entity(entity).insert(TokenComponent {
76            token: token.clone(),
77            range,
78            text,
79        });
80    }
81}
82
83pub trait Membered: Sized + 'static {
84    const ITEMS: &'static [Self];
85
86    fn complete(&self) -> &'static str;
87}
88
89macro_rules! derive_enum {
90    // entry point
91    ($(#$meta:tt)? $vis:vis enum $name:ident {
92        $($xs:ident $(@ $st:tt)? $(=> $it:tt)?) ,* $(,)?
93    }) => {
94
95        $(#$meta)? $vis enum $name {
96            $($xs),*
97        }
98
99
100        derive_enum!(@membered $name {$($xs)* } {}: $($xs $(=> $it)? ),* ,);
101        derive_enum!(@fromStr $name {}: $($xs $(@ $st)? ),* ,);
102    };
103
104    // fromstr implementation
105    (@fromStr $name:ident {$($eout:tt)*}: $member:ident @ $str:tt , $($xs:tt)*) => {
106        derive_enum!(@fromStr $name
107            { $str => Ok($name::$member), $($eout)*}:
108            $($xs)*
109        );
110    };
111    (@fromStr $name:ident {$($eout:tt)*}: $member:ident , $($xs:tt)*) => {
112        derive_enum!(@fromStr $name
113            { x if x.eq_ignore_ascii_case(stringify!($member))  => Ok($name::$member), $($eout)*}:
114            $($xs)*
115        );
116    };
117
118    (@fromStr $name:ident {$($eout:tt)*}:) => {
119        impl std::str::FromStr for $name {
120            type Err = ();
121
122            fn from_str(st: &str) -> Result<Self, Self::Err> {
123                match st {
124                    $($eout)*
125                    _ => Err(()),
126                }
127            }
128        }
129    };
130
131    // membered implementation
132    (@membered $name:ident {$($els:ident)*} {$($eout:tt)*}: $member:ident => $str:tt , $($xs:tt)*) => {
133        derive_enum!(@membered $name {$($els)*}
134            {
135                $name::$member => $str,
136                $($eout)*
137            }:
138            $($xs)*
139        );
140    };
141    (@membered $name:ident {$($els:ident)*} {$($eout:tt)*}: $member:ident , $($xs:tt)*) => {
142        derive_enum!(@membered $name {$($els)*}
143            {
144                $name::$member => stringify!($member),
145                $($eout)*
146            }:
147            $($xs)*
148        );
149    };
150
151    (@membered $name:ident {$($xs:ident)*} {$($eout:tt)*}:) => {
152        impl Membered for $name {
153            const ITEMS: &'static [Self] =  &[
154                $($name::$xs),*
155            ];
156
157            fn complete(&self) -> &'static str {
158                match self {
159                    $($eout)*
160                }
161            }
162        }
163    };
164}
165
166derive_enum!(
167    #[derive(Debug, Clone, PartialEq)]
168    pub enum SparqlExpr2 {
169        Or => "PLUS",
170        Plus @ "+" => "PLUS"
171    }
172);
173
174derive_enum!(
175    pub enum SparqlExpr3 {
176        Or => "PLUS",
177        Plus @ "+" => "PLUS",
178    }
179);
180
181#[cfg(test)]
182mod tests {
183    use std::str::FromStr as _;
184
185    use super::SparqlExpr2;
186
187    #[test]
188    fn test_sparql_expr_2() {
189        let or = SparqlExpr2::from_str("or");
190        assert_eq!(or, Ok(SparqlExpr2::Or));
191
192        let or = SparqlExpr2::from_str("+");
193        assert_eq!(or, Ok(SparqlExpr2::Plus));
194    }
195}
196
197pub mod semantic_token {
198    use crate::lsp_types::SemanticTokenType as STT;
199    pub const BOOLEAN: STT = STT::new("boolean");
200    pub const LANG_TAG: STT = STT::new("langTag");
201}
202
203derive_enum!(
204    #[derive(Clone, PartialEq, Eq,Ord, PartialOrd, Hash, Debug)]
205    pub enum SparqlExpr {
206        Or @ "||",
207        And @ "&&",
208        Equal @ "=",
209        NotEqual @ "!=",
210        Lt @ "<",
211        Gt @ ">",
212        Lte @ "<=",
213        Gte @ ">=",
214        In,
215        Not,
216        Plus @ "+",
217        Minus @ "-",
218        Times @ "*",
219        Divide @ "/",
220        Exclamation @ "!",
221    }
222);
223
224derive_enum!(
225    #[derive(Clone, PartialEq, Eq, Hash,Ord, PartialOrd, Debug)]
226    pub enum SparqlCall {
227        Str => "STR",
228        Lang => "LANG",
229        LangMatches => "langMatches",
230        LangDir => "LANGDIR",
231        Datatype => "datatype",
232        Bound => "BOUND",
233        Iri => "IRI",
234        Uri => "URI",
235        Bnode => "BNODE",
236        Rand => "RAND",
237        Abs => "ABS",
238        Ceil => "CEIL",
239        Floor => "FLOOR",
240        Round => "ROUND",
241        Concat => "CONCAT",
242        StrLen => "STRLEN",
243        Ucase => "UCASE",
244        Lcase => "lcase",
245        EncodeForUri => "ENCODE_FOR_URI",
246        Contains => "CONTAINS",
247        StrStarts => "STRSTARTS",
248        StrEnds => "STRENDS",
249        StrBefore => "STRBEFORE",
250        StrAfter => "STRAFTER",
251        Year => "YEAR",
252        Month => "MONTH",
253        Day => "DAY",
254        Hours => "HOURS",
255        Minutes => "MINUTES",
256        Seconds => "SECONDS",
257        Timezone => "TIMEZONE",
258        Tz => "TZ",
259        Now => "NOW",
260        Uuid => "UUID",
261        StrUuid => "STRUUID",
262        Md5 => "MD5",
263        Sha1 => "SHA1",
264        Sha256 => "SHA256",
265        Sha384 => "SHA384",
266        Sha512 => "SHA512",
267        Coalesce => "COALESCE",
268        If => "IF",
269        StrLang => "STRLANG",
270        StrLangDir => "STRLANGDIR",
271        StrDt => "STRDT",
272        SameTerm => "sameTerm",
273        IsIri => "isIRI",
274        IsUri => "isURI",
275        IsBlank => "isBLANK",
276        IsLiteral => "isLITERAL",
277        IsNumeric => "isNUMBERIC",
278        HasLang => "hasLANG",
279        HasLangDir => "hasLANGDIR",
280        IsTriple => "isTRIPLE",
281        Triple => "TRIPLE",
282        Subject => "SUBJECT",
283        Predicate => "PREDICATE",
284        Object => "OBJECT",
285    }
286);
287
288derive_enum!(
289    #[derive(Clone, PartialEq, Eq,Ord, PartialOrd, Hash, Debug)]
290    pub enum SparqlAggregate {
291        Count => "COUNT",
292        Sum => "SUM",
293        Min => "MIN",
294        Max => "MAX",
295        Avg => "AVG",
296        Sample => "SAMPLE",
297        GroupConcat => "GROUP_CONCAT",
298    }
299);
300
301derive_enum!(
302    #[derive(Clone, PartialEq, Eq,Ord, PartialOrd, Hash, Debug)]
303    pub enum SparqlKeyword {
304        Regex => "REGEX",
305        Substr => "SUBSTR",
306        Replace => "REPLACE",
307        Exists => "EXISTS",
308        Select => "SELECT",
309        Distinct => "DISTINCT",
310        Reduced => "REDUCED",
311        Optional => "OPTIONAL",
312        Union => "UNION",
313        As => "AS",
314        Construct => "CONSTRUCT",
315        Where => "WHERE",
316        Describe => "DESCRIBE",
317        Ask => "ASK",
318        From => "FROM",
319        Named => "NAMED",
320        Group => "GROUP",
321        By => "BY",
322        Having => "HAVING",
323        Order => "ORDER",
324        Asc => "ASC",
325        Desc => "DESC",
326        Limit => "LIMIT",
327        Offset => "OFFSET",
328        Values => "VALUES",
329        Load => "LOAD",
330        Silent => "SILENT",
331        Clear => "CLEAR",
332        Drop => "DROP",
333        Create => "CREATE",
334        Add => "ADD",
335        Move => "MOVE",
336        Copy => "COPY",
337        Insert => "INSERT",
338        Data => "DATA",
339        Delete => "DELETE",
340        With => "WITH",
341        Using => "USING",
342        Default => "DEFAULT",
343        All => "ALL",
344        Graph => "GRAPH",
345        Service => "SERVICE",
346        Bind => "BIND",
347        Undef => "UNDEF",
348        Minus => "MINUS",
349        Filter => "FILTER",
350    }
351);
352
353#[derive(Clone, PartialEq, Ord, PartialOrd, Eq, Hash, Debug, IsVariant)]
354pub enum Token {
355    /// Sparql expression
356    SparqlExpr(SparqlExpr),
357    /// Sparql keyword
358    SparqlKeyword(SparqlKeyword),
359    /// Sparql call
360    SparqlCall(SparqlCall),
361    /// Sparql aggregate
362    SparqlAggregate(SparqlAggregate),
363    /// Sparql variable
364    Variable(String),
365    // Turtle Tokens
366    /// @prefix
367    PrefixTag,
368    /// @base
369    BaseTag,
370    /// sparql prefix
371    SparqlPrefix,
372    /// sparql base
373    SparqlBase,
374
375    /// a
376    PredType,
377
378    /// [
379    SqOpen,
380    /// ]
381    SqClose,
382    /// {
383    CurlOpen,
384    /// }
385    CurlClose,
386    /// (
387    BracketOpen,
388    /// )
389    BracketClose,
390
391    /// ^^
392    DataTypeDelim,
393
394    /// .
395    Stop,
396    /// ;
397    PredicateSplit,
398    /// ,
399    Comma,
400
401    /// true
402    True,
403    /// false
404    False,
405    /// <...>
406    IRIRef(String),
407
408    /// ..:
409    PNameLN(Option<String>, String),
410    /// _:...
411    BlankNodeLabel(String),
412    /// @...
413    LangTag(String),
414
415    Number(String),
416    /// All string types
417    Str(String, StringStyle),
418
419    /// [ ]
420    ANON,
421    Comment(String),
422
423    /// :
424    Colon,
425    /// null
426    Null,
427
428    Invalid(String),
429}
430
431/// Token struct holding the token and the index in the token array
432#[derive(Clone, Ord, PartialOrd, Hash, Debug)]
433pub struct PToken(pub Token, pub usize);
434impl PartialEq for PToken {
435    fn eq(&self, other: &Self) -> bool {
436        self.0 == other.0
437    }
438}
439impl Into<PToken> for Token {
440    fn into(self) -> PToken {
441        PToken(self, 0)
442    }
443}
444
445impl Eq for PToken {}
446
447impl TokenTrait for Token {
448    fn token(&self) -> Option<crate::lsp_types::SemanticTokenType> {
449        match self {
450            Token::PrefixTag
451            | Token::BaseTag
452            | Token::SparqlPrefix
453            | Token::SparqlBase
454            | Token::PredType
455            | Token::SparqlKeyword(_)
456            | Token::SparqlCall(_) => Some(crate::lsp_types::SemanticTokenType::KEYWORD),
457            Token::True | Token::False => Some(semantic_token::BOOLEAN),
458            Token::IRIRef(_) => Some(crate::lsp_types::SemanticTokenType::PROPERTY),
459            Token::LangTag(_) => Some(semantic_token::LANG_TAG),
460            Token::Number(_) => Some(crate::lsp_types::SemanticTokenType::NUMBER),
461            Token::Str(_, _) => Some(crate::lsp_types::SemanticTokenType::STRING),
462            Token::Comment(_) => Some(crate::lsp_types::SemanticTokenType::COMMENT),
463            Token::Variable(_) => Some(crate::lsp_types::SemanticTokenType::VARIABLE),
464            _ => None,
465        }
466    }
467
468    fn span_tokens(
469        Spanned(this, span): &Spanned<Self>,
470    ) -> Vec<(crate::lsp_types::SemanticTokenType, Range<usize>)> {
471        if let Some(t) = this.token() {
472            return vec![(t, span.clone())];
473        }
474
475        match this {
476            Token::PNameLN(p, _) => {
477                let s = p.as_ref().map(|x| x.len()).unwrap_or(0);
478
479                vec![
480                    (
481                        crate::lsp_types::SemanticTokenType::NAMESPACE,
482                        span.start..span.start + 1 + s,
483                    ),
484                    (
485                        crate::lsp_types::SemanticTokenType::ENUM_MEMBER,
486                        span.start + s + 1..span.end,
487                    ),
488                ]
489            }
490            Token::BlankNodeLabel(_) => {
491                vec![
492                    (
493                        crate::lsp_types::SemanticTokenType::NAMESPACE,
494                        span.start..span.start + 2,
495                    ),
496                    (
497                        crate::lsp_types::SemanticTokenType::PROPERTY,
498                        span.start + 2..span.end,
499                    ),
500                ]
501            }
502            _ => vec![],
503        }
504    }
505}
506
507#[derive(Clone, PartialEq, Ord, PartialOrd, Eq, Hash, Debug)]
508pub enum StringStyle {
509    /// """..."""
510    DoubleLong,
511    /// "..."
512    Double,
513    /// '''...'''
514    SingleLong,
515    /// '...'
516    Single,
517}
518
519impl StringStyle {
520    pub fn quote(&self) -> &'static str {
521        match self {
522            StringStyle::DoubleLong => "\"\"\"",
523            StringStyle::Double => "\"",
524            StringStyle::SingleLong => "'''",
525            StringStyle::Single => "'",
526        }
527    }
528}
529impl std::fmt::Display for PToken {
530    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
531        self.0.fmt(f)
532    }
533}
534
535impl std::fmt::Display for Token {
536    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
537        match self {
538            Token::PrefixTag => write!(f, "'@prefix'"),
539            Token::BaseTag => write!(f, "'@base'"),
540            Token::SparqlPrefix => write!(f, "'PREFIX'"),
541            Token::SparqlBase => write!(f, "'BASE'"),
542            Token::PredType => write!(f, "'a'"),
543            Token::SqOpen => write!(f, "'['"),
544            Token::SqClose => write!(f, "']'"),
545            Token::BracketOpen => write!(f, "'('"),
546            Token::BracketClose => write!(f, "')'"),
547            Token::DataTypeDelim => write!(f, "'^^'"),
548            Token::Stop => write!(f, "'.'"),
549            Token::PredicateSplit => write!(f, "';'"),
550            Token::Comma => write!(f, "','"),
551            Token::True => write!(f, "'true'"),
552            Token::False => write!(f, "'false'"),
553            Token::IRIRef(_) => write!(f, "a named node"),
554            Token::PNameLN(_, _) => write!(f, "a prefixed node"),
555            Token::BlankNodeLabel(_) => write!(f, "a blank node"),
556            Token::LangTag(_) => write!(f, "a language tag"),
557            Token::Number(_) => write!(f, "a number"),
558            Token::Str(_, _) => write!(f, "a string"),
559            Token::ANON => write!(f, "an inline blank node"),
560            Token::Comment(_) => write!(f, "a comment"),
561            Token::Invalid(_) => write!(f, "invalid token"),
562            Token::CurlOpen => write!(f, "'{{'"),
563            Token::CurlClose => write!(f, "'}}'"),
564            Token::Colon => write!(f, "':'"),
565            Token::Null => write!(f, "'null'"),
566            Token::SparqlExpr(_) => write!(f, "sparql expr token"),
567            Token::SparqlKeyword(_) => write!(f, "sparql keyword"),
568            Token::SparqlCall(_) => write!(f, "sparql call"),
569            Token::SparqlAggregate(_) => write!(f, "sparql aggregate"),
570            Token::Variable(_) => write!(f, "sparql variable"),
571        }
572    }
573}
lsp_core/util/token.rs

lsp_core/util/
token.rs
