improv_gui/
formula_highlight.rs

1//! Pure syntax highlighting for the formula editor: `&str -> LayoutJob`.
2//!
3//! A small independent char-class scanner (identifiers / numbers / strings /
4//! date literals / operators), good enough for visually useful highlighting —
5//! it does not need to match `core_model::parser`'s exact grammar. Kept free
6//! of any `egui::Ui` access so it's unit-testable without a window.
7
8use crate::theme::{BLACK, DARK_GRAY};
9use egui::text::LayoutJob;
10use egui::{Color32, FontId, Stroke, TextFormat};
11use std::ops::Range;
12
13/// A known scalar/aggregation function name, highlighted distinctly from a
14/// plain measure/category identifier.
15const FUNCTIONS: &[&str] = &[
16    "SUM", "AVG", "MIN", "MAX", "ABS", "ROUND", "FLOOR", "CEIL", "SQRT", "NEG", "MIN2", "MAX2",
17    "CALL", "SQL", "OVER", "TRUE", "FALSE", "AND", "OR", "NOT",
18];
19
20/// The lexical class of one highlighted token.
21#[derive(Debug, Clone, Copy, PartialEq, Eq)]
22pub enum TokenKind {
23    /// A measure/category name, or a keyword-like bare word not in
24    /// [`FUNCTIONS`].
25    Ident,
26    /// A recognized function/aggregation/keyword name (case-insensitive).
27    Function,
28    Number,
29    String,
30    /// A `#...#` date/time literal.
31    Date,
32    Operator,
33    Whitespace,
34}
35
36/// One scanned token: its kind and byte range in the source text.
37#[derive(Debug, Clone, PartialEq, Eq)]
38pub struct Token {
39    pub kind: TokenKind,
40    pub start: usize,
41    pub end: usize,
42}
43
44/// The character at byte offset `i` of `text`. `i` must be a char boundary
45/// (every offset [`scan`] advances to is one, by construction).
46///
47/// This exists because `text.as_bytes()[i] as char` — the shape this scanner
48/// used to have, and the same bug once present in
49/// `core_model::parser::tokenize` — casts a single BYTE, which is wrong for
50/// any multi-byte UTF-8 sequence and walks the cursor onto a non-boundary
51/// offset, panicking on the eventual `&text[start..i]` slice.
52fn char_at(text: &str, i: usize) -> char {
53    text[i..]
54        .chars()
55        .next()
56        .expect("i < len implies a char here")
57}
58
59/// Scan `text` into a flat sequence of [`Token`]s covering every byte (no
60/// gaps), classifying identifiers/numbers/strings/date-literals/operators by
61/// simple char class. Never fails — unrecognized characters become single-char
62/// `Operator` tokens so the whole string is always covered.
63///
64/// Every `start`/`end` is a char boundary and the tokens tile `text` exactly
65/// (no gaps, no overlap): [`highlight_formula`] slices `text` by those ranges.
66pub fn scan(text: &str) -> Vec<Token> {
67    let bytes = text.as_bytes();
68    let mut out = Vec::new();
69    let mut i = 0;
70    while i < bytes.len() {
71        let start = i;
72        let c = char_at(text, i);
73        let kind = match c {
74            c if c.is_whitespace() => {
75                i += c.len_utf8();
76                while i < bytes.len() {
77                    let w = char_at(text, i);
78                    if w.is_whitespace() {
79                        i += w.len_utf8();
80                    } else {
81                        break;
82                    }
83                }
84                TokenKind::Whitespace
85            }
86            '"' => {
87                // Byte scanning is safe here: ASCII bytes never occur inside a
88                // multi-byte UTF-8 sequence, so `i` stays on a char boundary.
89                i += 1;
90                while i < bytes.len() && bytes[i] != b'"' {
91                    i += 1;
92                }
93                if i < bytes.len() {
94                    i += 1; // closing quote
95                }
96                TokenKind::String
97            }
98            '#' => {
99                i += 1;
100                while i < bytes.len() && bytes[i] != b'#' {
101                    i += 1;
102                }
103                if i < bytes.len() {
104                    i += 1; // closing '#'
105                }
106                TokenKind::Date
107            }
108            c if c.is_ascii_digit() => {
109                while i < bytes.len() && (bytes[i].is_ascii_digit() || bytes[i] == b'.') {
110                    i += 1;
111                }
112                TokenKind::Number
113            }
114            c if c.is_alphabetic() || c == '_' => {
115                i += c.len_utf8();
116                while i < bytes.len() {
117                    let w = char_at(text, i);
118                    if w.is_alphanumeric() || w == '_' {
119                        i += w.len_utf8();
120                    } else {
121                        break;
122                    }
123                }
124                let word = &text[start..i];
125                if FUNCTIONS.iter().any(|f| word.eq_ignore_ascii_case(f)) {
126                    TokenKind::Function
127                } else {
128                    TokenKind::Ident
129                }
130            }
131            _ => {
132                // Two-char operators first, then a single CHAR. `get` yields
133                // None on a non-boundary/out-of-range end, so this can never
134                // split a multi-byte sequence.
135                let two = text.get(i..i + 2);
136                if matches!(
137                    two,
138                    Some("<=") | Some(">=") | Some("<>") | Some("==") | Some("!=")
139                ) {
140                    i += 2;
141                } else {
142                    i += c.len_utf8();
143                }
144                TokenKind::Operator
145            }
146        };
147        out.push(Token {
148            kind,
149            start,
150            end: i,
151        });
152    }
153    out
154}
155
156/// Token colors, within the NeXTSTEP grayscale palette (`theme.rs`): names,
157/// functions and numbers in black on the white field, operators and literals
158/// in dark gray (7.5:1 on white).
159fn color_for(kind: TokenKind) -> Color32 {
160    match kind {
161        TokenKind::Function | TokenKind::Number | TokenKind::Ident | TokenKind::Whitespace => BLACK,
162        TokenKind::String | TokenKind::Date | TokenKind::Operator => DARK_GRAY,
163    }
164}
165
166/// The color of formula error text and of the error-position underline. The
167/// palette has no red, so an error is marked by the underline, not by hue.
168pub const ERROR_COLOR: Color32 = BLACK;
169
170/// The error-position underline: thicker than a text underline so it reads.
171const ERROR_UNDERLINE: Stroke = Stroke {
172    width: 2.0,
173    color: ERROR_COLOR,
174};
175
176/// Build a highlighted [`LayoutJob`] for `text`, coloring tokens per
177/// [`scan`]/[`color_for`]. If `error` is `Some(start..end)`, those bytes
178/// (clamped to the text and to char boundaries; a span may cover part of a
179/// token or several) are rendered in [`ERROR_COLOR`] with a 2 px underline so
180/// the error's location is visible inline. `crate::diagnostics` places it:
181/// the token at a parse error's position, or the name a compile error names.
182pub fn highlight_formula(text: &str, font: FontId, error: Option<Range<usize>>) -> LayoutJob {
183    let mut job = LayoutJob::default();
184    if text.is_empty() {
185        // One empty section, as egui's own layouter makes: a job with none
186        // lays out zero rows high, and the edit shrinks by a pixel.
187        job.append("", 0.0, TextFormat::simple(font, BLACK));
188        return job;
189    }
190    let (lo, hi) = error.map_or((0, 0), |r| {
191        let fix = |mut i: usize| {
192            i = i.min(text.len());
193            while !text.is_char_boundary(i) {
194                i -= 1;
195            }
196            i
197        };
198        (fix(r.start), fix(r.end))
199    });
200    for t in scan(text) {
201        let fmt = TextFormat {
202            font_id: font.clone(),
203            color: color_for(t.kind),
204            ..Default::default()
205        };
206        let err = TextFormat {
207            underline: ERROR_UNDERLINE,
208            color: ERROR_COLOR,
209            ..fmt.clone()
210        };
211        // Up to three pieces: before the span, inside it, after it.
212        let a = lo.clamp(t.start, t.end);
213        let b = hi.clamp(a, t.end);
214        for (s, e, f) in [(t.start, a, &fmt), (a, b, &err), (b, t.end, &fmt)] {
215            if s < e {
216                job.append(&text[s..e], 0.0, f.clone());
217            }
218        }
219    }
220    job
221}
222
223#[cfg(test)]
224#[cfg_attr(coverage_nightly, coverage(off))]
225mod tests {
226    use super::*;
227
228    fn kinds(text: &str) -> Vec<(TokenKind, &str)> {
229        scan(text)
230            .into_iter()
231            .filter(|t| t.kind != TokenKind::Whitespace)
232            .map(|t| (t.kind, &text[t.start..t.end]))
233            .collect()
234    }
235
236    #[test]
237    fn identifies_idents_and_operator() {
238        let toks = kinds("Price * Quantity");
239        assert_eq!(
240            toks,
241            vec![
242                (TokenKind::Ident, "Price"),
243                (TokenKind::Operator, "*"),
244                (TokenKind::Ident, "Quantity"),
245            ]
246        );
247    }
248
249    #[test]
250    fn known_function_names_are_distinct_from_idents() {
251        let toks = kinds("SUM(Revenue OVER Time)");
252        assert_eq!(toks[0], (TokenKind::Function, "SUM"));
253        assert_eq!(toks[1], (TokenKind::Operator, "("));
254        assert_eq!(toks[2], (TokenKind::Ident, "Revenue"));
255        assert_eq!(toks[3], (TokenKind::Function, "OVER"));
256        assert_eq!(toks[4], (TokenKind::Ident, "Time"));
257        assert_eq!(toks[5], (TokenKind::Operator, ")"));
258
259        let abs = kinds("ABS(x)");
260        assert_eq!(abs[0], (TokenKind::Function, "ABS"));
261        assert_eq!(abs[1].0, TokenKind::Operator);
262        assert_eq!(abs[2], (TokenKind::Ident, "x"));
263    }
264
265    #[test]
266    fn string_and_number_and_date_literals() {
267        let toks = kinds(r#"Name == "Widget A""#);
268        assert_eq!(toks[0], (TokenKind::Ident, "Name"));
269        assert_eq!(toks[1], (TokenKind::Operator, "=="));
270        assert_eq!(toks[2], (TokenKind::String, "\"Widget A\""));
271
272        let toks = kinds("3.14 + 2");
273        assert_eq!(toks[0], (TokenKind::Number, "3.14"));
274        assert_eq!(toks[1], (TokenKind::Operator, "+"));
275        assert_eq!(toks[2], (TokenKind::Number, "2"));
276
277        let toks = kinds("Date > #2025-01-01#");
278        assert_eq!(toks[0], (TokenKind::Ident, "Date"));
279        assert_eq!(toks[1], (TokenKind::Operator, ">"));
280        assert_eq!(toks[2], (TokenKind::Date, "#2025-01-01#"));
281    }
282
283    #[test]
284    fn scan_covers_every_byte_with_no_gaps() {
285        let text = "Price * Quantity - SUM(X OVER Y)";
286        let toks = scan(text);
287        let mut pos = 0;
288        for t in &toks {
289            assert_eq!(t.start, pos);
290            pos = t.end;
291        }
292        assert_eq!(pos, text.len());
293    }
294
295    /// The load-bearing invariant: tokens tile `text` exactly (start at 0, each
296    /// one begins where the last ended, the last ends at `text.len()`) and every
297    /// boundary is a char boundary — `highlight_formula` slices by these ranges.
298    fn assert_tiles(text: &str) {
299        let toks = scan(text);
300        let mut pos = 0;
301        for t in &toks {
302            assert_eq!(t.start, pos, "gap/overlap before {t:?} in {text:?}");
303            assert!(t.end > t.start, "empty token {t:?} in {text:?}");
304            assert!(
305                text.is_char_boundary(t.start),
306                "{:?}: start {} not a char boundary",
307                text,
308                t.start
309            );
310            assert!(
311                text.is_char_boundary(t.end),
312                "{:?}: end {} not a char boundary",
313                text,
314                t.end
315            );
316            pos = t.end;
317        }
318        assert_eq!(pos, text.len(), "tokens do not cover all of {text:?}");
319    }
320
321    #[test]
322    fn unicode_input_never_panics_and_tokens_stay_on_char_boundaries() {
323        // Each of these panicked before the byte-cast fix (the reproduced case
324        // on EC2 was `scan("é")`).
325        for text in [
326            "é",
327            "café",
328            "年度売上",          // CJK identifier
329            "年度 * 売上",       // CJK with ASCII operator
330            "🚀",                // emoji (4-byte, non-alphabetic)
331            "Price 🚀 Quantity", // emoji as an "operator"
332            "Prîx * Qté",        // mixed ASCII + accented identifiers
333            "SUM(Recéttes OVER Année)",
334            "\u{0301}",   // lone combining mark
335            "e\u{0301}x", // combining mark inside an identifier
336            "ééé",
337            "\"café\" == Nom", // non-ASCII inside a string literal
338            "#2025-01-é1#",    // non-ASCII inside a date literal
339            "3.14 ≤ é",        // multi-byte operator-ish char
340            "é<=é",            // two-char op adjacent to multi-byte chars
341            "é\"unterminated", // unterminated string after a multi-byte char
342            "é#unterminated",
343            " é", // non-ASCII whitespace (NBSP)
344        ] {
345            assert_tiles(text);
346            // The renderer slices by the same ranges; must not panic either.
347            let job = highlight_formula(text, FontId::default(), Some(1..2));
348            assert_eq!(job.text, text, "rendered text differs for {text:?}");
349        }
350    }
351
352    #[test]
353    fn multibyte_identifier_is_one_token() {
354        // "café" is 5 bytes / 4 chars: one Ident token spanning all of it.
355        let toks = kinds("café");
356        assert_eq!(toks, vec![(TokenKind::Ident, "café")]);
357        // A non-alphanumeric multi-byte char is a single-CHAR operator token,
358        // not a single-byte one.
359        let toks = scan("🚀");
360        assert_eq!(
361            toks,
362            vec![Token {
363                kind: TokenKind::Operator,
364                start: 0,
365                end: 4
366            }]
367        );
368    }
369
370    #[test]
371    fn error_span_inside_a_multibyte_char_does_not_panic() {
372        // byte 1 is a continuation byte of "é": not a char boundary.
373        let job = highlight_formula("é + 1", FontId::default(), Some(1..2));
374        assert_eq!(job.text, "é + 1");
375    }
376
377    #[test]
378    fn error_span_marks_the_token() {
379        let text = "Price * Widgets";
380        // "Widgets" is bytes 8..15.
381        let job = highlight_formula(text, FontId::default(), Some(8..15));
382        let bad = job
383            .sections
384            .iter()
385            .find(|s| &job.text[s.byte_range.clone()] == "Widgets")
386            .expect("Widgets token present");
387        assert_eq!(bad.format.color, ERROR_COLOR);
388        assert_eq!(bad.format.underline, ERROR_UNDERLINE);
389        // Other tokens are unaffected. (The palette has no error hue, so the
390        // underline, not the color, is what marks the error.)
391        let price = job
392            .sections
393            .iter()
394            .find(|s| &job.text[s.byte_range.clone()] == "Price")
395            .expect("Price token present");
396        assert_eq!(price.format.underline, Stroke::NONE);
397    }
398
399    /// A span underlines exactly its bytes, across or inside tokens, and
400    /// clamps an out-of-range or mid-char end.
401    #[test]
402    fn error_span_marks_exactly_its_bytes() {
403        let marked = |text: &str, r| {
404            let job = highlight_formula(text, FontId::default(), Some(r));
405            assert_eq!(job.text, text);
406            job.sections
407                .iter()
408                .filter(|s| s.format.underline == ERROR_UNDERLINE)
409                .map(|s| &text[s.byte_range.clone()])
410                .collect::<String>()
411        };
412        assert_eq!(marked("Price * Widgets", 0..7), "Price *");
413        assert_eq!(marked("Price * Widgets", 9..11), "id");
414        assert_eq!(marked("Price", 3..99), "ce");
415        assert_eq!(marked("é + 1", 0..1), "");
416        assert_eq!(marked("é + 1", 0..2), "é");
417        assert_eq!(marked("é + 1", 1..2), "é", "a mid-char start moves back");
418    }
419
420    #[test]
421    fn no_error_span_leaves_all_tokens_unmarked() {
422        let job = highlight_formula("Price * Quantity", FontId::default(), None);
423        assert!(job
424            .sections
425            .iter()
426            .all(|s| s.format.underline == Stroke::NONE));
427    }
428
429    #[test]
430    fn empty_text_keeps_one_section_for_the_row_height() {
431        let job = highlight_formula("", FontId::default(), Some(0..3));
432        assert_eq!(job.sections.len(), 1);
433        assert_eq!(job.sections[0].format.underline, Stroke::NONE);
434    }
435
436    #[test]
437    fn token_colors_are_grayscale() {
438        let job = highlight_formula(
439            r#"SUM(x) + 1 == "s" #2025-01-01#"#,
440            FontId::default(),
441            Some(0..3),
442        );
443        for s in &job.sections {
444            let c = s.format.color;
445            assert!(c == BLACK || c == DARK_GRAY, "{c:?} is not a palette gray");
446        }
447    }
448
449    #[test]
450    fn whitespace_runs_and_underscored_idents_are_single_tokens() {
451        let toks: Vec<_> = scan("_a_b  \t x  ")
452            .into_iter()
453            .map(|t| (t.kind, t.start, t.end))
454            .collect();
455        assert_eq!(
456            toks,
457            vec![
458                (TokenKind::Ident, 0, 4),
459                (TokenKind::Whitespace, 4, 8),
460                (TokenKind::Ident, 8, 9),
461                (TokenKind::Whitespace, 9, 11),
462            ]
463        );
464    }
465}
improv_gui/formula_highlight.rs

improv_gui/
formula_highlight.rs
