1#![cfg_attr(coverage_nightly, feature(coverage_attribute))]
46#![cfg_attr(coverage_nightly, allow(unused_features))]
49
50use improv_core_model::{
51 BinaryOp, CategoryId, DimensionSpec, Expr, Formula, FuncId, MeasureId, Model, Value,
52};
53use thiserror::Error;
54
55pub const FUNC_SUM: FuncId = FuncId(1);
56pub const FUNC_AVERAGE: FuncId = FuncId(2);
57pub const FUNC_MIN: FuncId = FuncId(3);
58pub const FUNC_MAX: FuncId = FuncId(4);
59
60pub struct NlContext<'m> {
62 model: &'m Model,
63}
64
65impl<'m> NlContext<'m> {
66 pub fn new(model: &'m Model) -> Self {
67 NlContext { model }
68 }
69
70 fn measure_id(&self, name: &str) -> Option<MeasureId> {
72 self.model
73 .measures
74 .values()
75 .find(|m| m.name.0.eq_ignore_ascii_case(name))
76 .map(|m| m.id)
77 }
78
79 fn category_id(&self, name: &str) -> Option<CategoryId> {
80 self.model
81 .categories
82 .values()
83 .find(|c| c.name.0.eq_ignore_ascii_case(name))
84 .map(|c| c.id)
85 }
86
87 fn measure_name(&self, id: MeasureId) -> Option<&str> {
88 self.model.measures.get(&id).map(|m| m.name.0.as_str())
89 }
90
91 fn category_name(&self, id: CategoryId) -> Option<&str> {
92 self.model.categories.get(&id).map(|c| c.name.0.as_str())
93 }
94}
95
96#[derive(Debug, Error, PartialEq)]
97pub enum NlError {
98 #[error("unknown measure: {0}")]
99 UnknownMeasure(String),
100 #[error("unknown category: {0}")]
101 UnknownCategory(String),
102 #[error("unexpected end of input")]
103 UnexpectedEof,
104 #[error("unexpected token: {0}")]
105 Unexpected(String),
106 #[error("empty input")]
107 Empty,
108 #[error("formula nested too deeply (limit {MAX_PARSE_DEPTH})")]
109 TooDeep,
110}
111
112pub const MAX_PARSE_DEPTH: usize = 200;
117
118fn tokenize(text: &str) -> Vec<String> {
126 let mut out = Vec::new();
127 let mut word = String::new();
128 for ch in text.chars() {
129 match ch {
130 '+' | '-' | '*' | '/' | '(' | ')' => {
131 if !word.is_empty() {
132 out.push(std::mem::take(&mut word));
133 }
134 out.push(ch.to_string());
135 }
136 c if c.is_whitespace() || c == '.' || c == ',' => {
137 if !word.is_empty() {
138 out.push(std::mem::take(&mut word));
139 }
140 }
141 c => word.push(c.to_ascii_lowercase()),
142 }
143 }
144 if !word.is_empty() {
145 out.push(word);
146 }
147 out
148}
149
150struct Parser<'a, 'm> {
155 toks: &'a [String],
156 pos: usize,
157 ctx: &'a NlContext<'m>,
158 depth: usize,
161}
162
163fn is_keyword(w: &str) -> bool {
165 matches!(
166 w,
167 "plus"
168 | "minus"
169 | "times"
170 | "divided"
171 | "by"
172 | "over"
173 | "for"
174 | "each"
175 | "and"
176 | "of"
177 | "the"
178 | "sum"
179 | "average"
180 | "min"
181 | "max"
182 | "equals"
183 | "is"
184 )
185}
186
187impl<'a, 'm> Parser<'a, 'm> {
188 fn peek(&self) -> Option<&str> {
189 self.toks.get(self.pos).map(String::as_str)
190 }
191
192 fn peek2(&self) -> Option<&str> {
193 self.toks.get(self.pos + 1).map(String::as_str)
194 }
195
196 fn bump(&mut self) -> Option<&str> {
197 let t = self.toks.get(self.pos).map(String::as_str);
198 if t.is_some() {
199 self.pos += 1;
200 }
201 t
202 }
203
204 fn eat(&mut self, w: &str) -> bool {
205 if self.peek() == Some(w) {
206 self.pos += 1;
207 true
208 } else {
209 false
210 }
211 }
212
213 fn deeper(&mut self) -> Result<(), NlError> {
216 self.depth += 1;
217 if self.depth > MAX_PARSE_DEPTH {
218 return Err(NlError::TooDeep);
219 }
220 Ok(())
221 }
222
223 fn parse_sentence(&mut self) -> Result<Expr, NlError> {
224 if self.toks.is_empty() {
225 return Err(NlError::Empty);
226 }
227 if self.peek2() == Some("equals") || self.peek2() == Some("is") {
231 self.bump(); self.bump(); }
234 let e = self.parse_expr()?;
235 match self.peek() {
236 None => Ok(e),
237 Some(t) => Err(NlError::Unexpected(t.to_string())),
238 }
239 }
240
241 fn parse_expr(&mut self) -> Result<Expr, NlError> {
243 let mut lhs = self.parse_term()?;
244 loop {
245 let op = match self.peek() {
246 Some("plus") | Some("+") => BinaryOp::Add,
247 Some("minus") | Some("-") => BinaryOp::Sub,
248 _ => break,
249 };
250 self.bump();
251 self.deeper()?;
252 let rhs = self.parse_term()?;
253 lhs = Expr::BinaryOp(op, Box::new(lhs), Box::new(rhs));
254 }
255 Ok(lhs)
256 }
257
258 fn parse_term(&mut self) -> Result<Expr, NlError> {
260 let mut lhs = self.parse_factor()?;
261 loop {
262 let op = match self.peek() {
263 Some("times") | Some("*") => BinaryOp::Mul,
264 Some("/") => BinaryOp::Div,
265 Some("divided") if self.peek2() == Some("by") => {
266 self.bump(); BinaryOp::Div }
269 _ => break,
270 };
271 self.bump(); self.deeper()?;
273 let rhs = self.parse_factor()?;
274 lhs = Expr::BinaryOp(op, Box::new(lhs), Box::new(rhs));
275 }
276 Ok(lhs)
277 }
278
279 fn parse_factor(&mut self) -> Result<Expr, NlError> {
281 match self.peek() {
282 None => Err(NlError::UnexpectedEof),
283 Some("(") => {
284 self.bump();
285 self.deeper()?;
286 let e = self.parse_expr()?;
287 self.depth -= 1; if !self.eat(")") {
289 return Err(NlError::Unexpected(
290 self.peek().unwrap_or("<eof>").to_string(),
291 ));
292 }
293 Ok(e)
294 }
295 Some("the") if self.peek2().is_some_and(is_agg_word) => {
297 self.bump(); self.parse_agg()
299 }
300 Some(w) if is_agg_word(w) => self.parse_agg(),
301 Some(w) => {
302 if let Ok(n) = w.parse::<f64>() {
303 self.bump();
304 return Ok(Expr::Literal(Value::Number(n)));
305 }
306 self.parse_measure_ref()
307 }
308 }
309 }
310
311 fn parse_agg(&mut self) -> Result<Expr, NlError> {
313 let func = match self.bump() {
314 Some("sum") => FUNC_SUM,
315 Some("average") => FUNC_AVERAGE,
316 Some("min") => FUNC_MIN,
317 Some("max") => FUNC_MAX,
318 other => return Err(NlError::Unexpected(other.unwrap_or("<eof>").to_string())),
319 };
320 if !self.eat("of") {
321 return Err(NlError::Unexpected(
322 self.peek().unwrap_or("<eof>").to_string(),
323 ));
324 }
325 let arg = self.parse_measure_ref()?;
326 Ok(Expr::Call(func, vec![arg]))
327 }
328
329 fn parse_measure_ref(&mut self) -> Result<Expr, NlError> {
331 let name = match self.peek() {
332 None => return Err(NlError::UnexpectedEof),
333 Some(w) if is_keyword(w) => return Err(NlError::Unexpected(w.to_string())),
334 Some(w) => w.to_string(),
335 };
336 let id = self
337 .ctx
338 .measure_id(&name)
339 .ok_or_else(|| NlError::UnknownMeasure(name.clone()))?;
340 self.bump();
341
342 let mut spec = DimensionSpec::default();
343 loop {
344 match self.peek() {
345 Some("over") => {
346 self.bump();
347 spec.over.extend(self.parse_cat_list()?);
348 }
349 Some("by") => {
350 self.bump();
351 spec.by.extend(self.parse_cat_list()?);
352 }
353 Some("for") if self.peek2() == Some("each") => {
354 self.bump(); self.bump(); spec.by.extend(self.parse_cat_list()?);
357 }
358 _ => break,
359 }
360 }
361 Ok(Expr::Ref(id, spec))
362 }
363
364 fn parse_cat_list(&mut self) -> Result<Vec<CategoryId>, NlError> {
366 let mut cats = vec![self.parse_cat()?];
367 while self.eat("and") {
368 cats.push(self.parse_cat()?);
369 }
370 Ok(cats)
371 }
372
373 fn parse_cat(&mut self) -> Result<CategoryId, NlError> {
374 let name = match self.peek() {
375 None => return Err(NlError::UnexpectedEof),
376 Some(w) if is_keyword(w) => return Err(NlError::Unexpected(w.to_string())),
377 Some(w) => w.to_string(),
378 };
379 let id = self
380 .ctx
381 .category_id(&name)
382 .ok_or_else(|| NlError::UnknownCategory(name.clone()))?;
383 self.bump();
384 Ok(id)
385 }
386}
387
388fn is_agg_word(w: &str) -> bool {
389 matches!(w, "sum" | "average" | "min" | "max")
390}
391
392pub fn parse_nl_formula(ctx: &NlContext, text: &str) -> Result<Formula, NlError> {
394 let toks = tokenize(text);
395 let mut p = Parser {
396 toks: &toks,
397 pos: 0,
398 ctx,
399 depth: 0,
400 };
401 Ok(Formula::new(p.parse_sentence()?))
402}
403
404pub fn describe_formula(ctx: &NlContext, formula: &Formula) -> String {
410 describe_expr(ctx, &formula.expr)
411}
412
413fn describe_expr(ctx: &NlContext, e: &Expr) -> String {
414 match e {
415 Expr::Literal(Value::Number(n)) => fmt_num(*n),
416 Expr::Literal(v) => format!("{v:?}"),
417 Expr::Ref(id, spec) => describe_ref(ctx, *id, spec),
418 Expr::UnaryOp(_, inner) => format!("negative {}", describe_expr(ctx, inner)),
419 Expr::BinaryOp(op, l, r) => {
420 let word = match op {
421 BinaryOp::Add => "plus",
422 BinaryOp::Sub => "minus",
423 BinaryOp::Mul => "times",
424 BinaryOp::Div => "divided by",
425 BinaryOp::And => "and",
428 BinaryOp::Or => "or",
429 BinaryOp::Eq => "equals",
430 BinaryOp::Ne => "does not equal",
431 BinaryOp::Lt => "is less than",
432 BinaryOp::Le => "is at most",
433 BinaryOp::Gt => "is greater than",
434 BinaryOp::Ge => "is at least",
435 BinaryOp::Pow => "to the power of",
436 BinaryOp::Concat => "joined with",
437 };
438 format!("{} {word} {}", describe_expr(ctx, l), describe_expr(ctx, r))
439 }
440 Expr::Call(func, args) => {
441 let markers = args.iter().any(|a| {
444 matches!(a, Expr::Call(id, _) if *id == improv_core_model::functions::WHERE
445 || *id == improv_core_model::functions::WINDOW
446 || *id == improv_core_model::functions::INTO)
447 });
448 if let Some(l) = improv_core_model::functions::named_lambda_of(*func)
450 .and_then(|id| ctx.model.lambdas.get(&id))
451 {
452 let rendered: Vec<String> = args.iter().map(|a| describe_expr(ctx, a)).collect();
453 return format!("{}({})", l.name.0, rendered.join(", "));
454 }
455 let name = match *func {
456 _ if markers || args.len() != 1 => {
457 let rendered: Vec<String> =
458 args.iter().map(|a| describe_expr(ctx, a)).collect();
459 let name = improv_core_model::functions::by_id(*func)
460 .map_or_else(|| format!("function {}", func.0), |d| d.name.to_string());
461 return format!("{name}({})", rendered.join(", "));
462 }
463 FUNC_SUM => "sum",
464 FUNC_AVERAGE => "average",
465 FUNC_MIN => "min",
466 FUNC_MAX => "max",
467 other => {
468 let name = improv_core_model::functions::by_id(other)
469 .map_or_else(|| format!("function {}", other.0), |d| d.name.to_string());
470 let rendered: Vec<String> =
471 args.iter().map(|a| describe_expr(ctx, a)).collect();
472 return format!("{name}({})", rendered.join(", "));
473 }
474 };
475 format!("the {name} of {}", join_args(ctx, args))
476 }
477 Expr::Category(c) => ctx
478 .category_name(*c)
479 .map_or_else(|| format!("category {}", c.0), str::to_string),
480 }
481}
482
483fn join_args(ctx: &NlContext, args: &[Expr]) -> String {
484 args.iter()
485 .map(|a| describe_expr(ctx, a))
486 .collect::<Vec<_>>()
487 .join(" and ")
488}
489
490fn describe_ref(ctx: &NlContext, id: MeasureId, spec: &DimensionSpec) -> String {
491 let mut s = ctx
492 .measure_name(id)
493 .map(str::to_string)
494 .unwrap_or_else(|| format!("measure {}", id.0));
495 if !spec.over.is_empty() {
496 s.push_str(" over ");
497 s.push_str(&cat_list(ctx, &spec.over));
498 }
499 if !spec.by.is_empty() {
500 s.push_str(" for each ");
501 s.push_str(&cat_list(ctx, &spec.by));
502 }
503 s
504}
505
506fn cat_list(ctx: &NlContext, cats: &[CategoryId]) -> String {
507 cats.iter()
508 .map(|c| {
509 ctx.category_name(*c)
510 .map(str::to_string)
511 .unwrap_or_else(|| format!("category {}", c.0))
512 })
513 .collect::<Vec<_>>()
514 .join(" and ")
515}
516
517fn fmt_num(n: f64) -> String {
518 if n.fract() == 0.0 && n.is_finite() {
519 format!("{}", n as i64)
520 } else {
521 format!("{n}")
522 }
523}
524
525#[cfg(test)]
530#[cfg_attr(coverage_nightly, coverage(off))]
531mod tests {
532 use super::*;
533 use improv_core_model::{CategoryId, Measure, MeasureId, MeasureKind, Model, Name, ValueType};
534
535 const TIME: CategoryId = CategoryId(1);
536 const PRODUCT: CategoryId = CategoryId(2);
537 const PRICE: MeasureId = MeasureId(100);
538 const QUANTITY: MeasureId = MeasureId(101);
539 const REVENUE: MeasureId = MeasureId(102);
540
541 fn fixture() -> Model {
542 let mut m = Model::new();
543 m.add_category(TIME, "Time");
544 m.add_category(PRODUCT, "Product");
545 for (id, name, cats) in [
546 (PRICE, "Price", vec![PRODUCT]),
547 (QUANTITY, "Quantity", vec![TIME, PRODUCT]),
548 (REVENUE, "Revenue", vec![TIME, PRODUCT]),
549 ] {
550 m.add_measure(Measure {
551 id,
552 name: Name(name.into()),
553 value_type: ValueType::Number,
554 categories: cats,
555 kind: MeasureKind::Input,
556 description: None,
557 });
558 }
559 m
560 }
561
562 fn refr(id: MeasureId) -> Expr {
563 Expr::Ref(id, DimensionSpec::default())
564 }
565
566 #[test]
567 fn parses_multiplication() {
568 let m = fixture();
569 let ctx = NlContext::new(&m);
570 let f = parse_nl_formula(&ctx, "price times quantity").unwrap();
571 assert_eq!(
572 f.expr,
573 Expr::BinaryOp(
574 BinaryOp::Mul,
575 Box::new(refr(PRICE)),
576 Box::new(refr(QUANTITY))
577 )
578 );
579 }
580
581 #[test]
582 fn strips_subject_and_copula() {
583 let m = fixture();
584 let ctx = NlContext::new(&m);
585 let a = parse_nl_formula(&ctx, "Revenue equals price times quantity.").unwrap();
586 let b = parse_nl_formula(&ctx, "price times quantity").unwrap();
587 assert_eq!(a, b);
588 }
589
590 #[test]
591 fn parses_sum_aggregation() {
592 let m = fixture();
593 let ctx = NlContext::new(&m);
594 let f = parse_nl_formula(&ctx, "the sum of revenue over time for each product").unwrap();
595 let expected = Expr::Call(
596 FUNC_SUM,
597 vec![Expr::Ref(
598 REVENUE,
599 DimensionSpec {
600 over: vec![TIME],
601 by: vec![PRODUCT],
602 except: vec![],
603 },
604 )],
605 );
606 assert_eq!(f.expr, expected);
607 }
608
609 #[test]
610 fn arithmetic_precedence() {
611 let m = fixture();
613 let ctx = NlContext::new(&m);
614 let f = parse_nl_formula(&ctx, "price times quantity plus revenue").unwrap();
615 let expected = Expr::BinaryOp(
616 BinaryOp::Add,
617 Box::new(Expr::BinaryOp(
618 BinaryOp::Mul,
619 Box::new(refr(PRICE)),
620 Box::new(refr(QUANTITY)),
621 )),
622 Box::new(refr(REVENUE)),
623 );
624 assert_eq!(f.expr, expected);
625 }
626
627 #[test]
628 fn divided_by_multiword() {
629 let m = fixture();
630 let ctx = NlContext::new(&m);
631 let f = parse_nl_formula(&ctx, "revenue divided by quantity").unwrap();
632 assert_eq!(
633 f.expr,
634 Expr::BinaryOp(
635 BinaryOp::Div,
636 Box::new(refr(REVENUE)),
637 Box::new(refr(QUANTITY))
638 )
639 );
640 }
641
642 #[test]
643 fn describes_expr() {
644 let m = fixture();
645 let ctx = NlContext::new(&m);
646 let e = Expr::Call(
647 FUNC_SUM,
648 vec![Expr::Ref(
649 REVENUE,
650 DimensionSpec {
651 over: vec![TIME],
652 by: vec![PRODUCT],
653 except: vec![],
654 },
655 )],
656 );
657 let s = describe_formula(&ctx, &Formula::new(e));
658 assert_eq!(s, "the sum of Revenue over Time for each Product");
659 }
660
661 #[test]
662 fn describes_multiplication() {
663 let m = fixture();
664 let ctx = NlContext::new(&m);
665 let e = Expr::BinaryOp(
666 BinaryOp::Mul,
667 Box::new(refr(PRICE)),
668 Box::new(refr(QUANTITY)),
669 );
670 assert_eq!(
671 describe_formula(&ctx, &Formula::new(e)),
672 "Price times Quantity"
673 );
674 }
675
676 #[test]
677 fn round_trip_stable() {
678 let m = fixture();
679 let ctx = NlContext::new(&m);
680 let originals = [
681 Expr::BinaryOp(
682 BinaryOp::Mul,
683 Box::new(refr(PRICE)),
684 Box::new(refr(QUANTITY)),
685 ),
686 Expr::Call(
687 FUNC_SUM,
688 vec![Expr::Ref(
689 REVENUE,
690 DimensionSpec {
691 over: vec![TIME],
692 by: vec![PRODUCT],
693 except: vec![],
694 },
695 )],
696 ),
697 Expr::BinaryOp(
698 BinaryOp::Add,
699 Box::new(Expr::BinaryOp(
700 BinaryOp::Mul,
701 Box::new(refr(PRICE)),
702 Box::new(refr(QUANTITY)),
703 )),
704 Box::new(refr(REVENUE)),
705 ),
706 ];
707 for orig in originals {
708 let f = Formula::new(orig.clone());
709 let text = describe_formula(&ctx, &f);
710 let reparsed = parse_nl_formula(&ctx, &text)
711 .unwrap_or_else(|e| panic!("reparse of {text:?} failed: {e}"));
712 assert_eq!(reparsed.expr, orig, "round trip via {text:?}");
713 }
714 }
715
716 #[test]
717 fn unknown_measure_errors() {
718 let m = fixture();
719 let ctx = NlContext::new(&m);
720 let err = parse_nl_formula(&ctx, "widgets times quantity").unwrap_err();
721 assert_eq!(err, NlError::UnknownMeasure("widgets".into()));
722 }
723
724 #[test]
725 fn unknown_category_errors() {
726 let m = fixture();
727 let ctx = NlContext::new(&m);
728 let err = parse_nl_formula(&ctx, "the sum of revenue over region").unwrap_err();
729 assert_eq!(err, NlError::UnknownCategory("region".into()));
730 }
731
732 fn on_small_stack<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'static) -> T {
735 std::thread::Builder::new()
736 .stack_size(2 * 1024 * 1024)
737 .spawn(f)
738 .unwrap()
739 .join()
740 .unwrap()
741 }
742
743 #[test]
744 fn deep_parens_error_without_overflow() {
745 let err = on_small_stack(|| {
747 let m = fixture();
748 let text = format!("{}1{}", "(".repeat(100_000), ")".repeat(100_000));
749 parse_nl_formula(&NlContext::new(&m), &text).unwrap_err()
750 });
751 assert_eq!(err, NlError::TooDeep);
752 }
753
754 #[test]
755 fn long_operator_chain_errors_without_overflow() {
756 let err = on_small_stack(|| {
759 let m = fixture();
760 let text = vec!["1"; 100_000].join(" plus ");
761 parse_nl_formula(&NlContext::new(&m), &text).unwrap_err()
762 });
763 assert_eq!(err, NlError::TooDeep);
764 }
765
766 #[test]
767 fn moderate_nesting_still_parses() {
768 let m = fixture();
769 let ctx = NlContext::new(&m);
770 let text = format!("{}price{}", "(".repeat(150), ")".repeat(150));
771 assert_eq!(parse_nl_formula(&ctx, &text).unwrap().expr, refr(PRICE));
772 let chain = vec!["price"; 150].join(" plus ");
773 assert!(parse_nl_formula(&ctx, &chain).is_ok());
774 }
775}