improv_data_source/
lib.rs

1//! Shared "data source" bookkeeping for Improv's import/export backends
2//! (`improv_storage_sql`, `improv_storage_csv`, and future formats).
3//!
4//! Post-v0.5.0 Phase D: `improv_storage_sql::import_query` and
5//! `improv_storage_csv::import_csv_from` independently reinvented the same
6//! ~40-line block — ensure the mapped categories exist, create the input
7//! measure, intern dimension values into item ids (reusing existing items by
8//! name on re-import), then register the interned items on the model. That
9//! block is genuinely format-agnostic: it only touches `CategoryId` +
10//! `MeasureId` + names, never a source-specific column reference or row type.
11//! This crate extracts it so a third tabular format (Parquet, JSON-lines, …)
12//! reuses it instead of copying it a third time.
13//!
14//! What deliberately did NOT move: `ImportSpec`/`DimensionMapping` in each
15//! backend keep their own shape (SQL's `column` is a plain query-result name
16//! plus a `refresh_policy`; CSV's is a name-or-index `ColumnRef` plus a
17//! `value_type`) — unifying those would touch the backends' public field
18//! types, which ripples into `improv_cli`'s call sites (out of scope, and not
19//! worth it: the two shapes differ for real reasons, not by accident). There
20//! is also no shared `DataSource` trait: SQL's source is an open connection,
21//! CSV's is a path/reader, and nothing in the workspace calls either backend
22//! generically — a trait with no generic caller is indirection, not reuse.
23#![cfg_attr(coverage_nightly, feature(coverage_attribute))]
24// The attribute is only used inside #[cfg(test)] modules, so non-test
25// coverage builds would otherwise warn that the feature is unused.
26#![cfg_attr(coverage_nightly, allow(unused_features))]
27
28use improv_core_model::{
29    Category, CategoryId, Item, ItemId, Measure, MeasureId, MeasureKind, Model, Name, ValueType,
30};
31use std::collections::HashMap;
32
33/// Ensure a category exists for each `(id, name)` pair. Idempotent: an
34/// existing category (e.g. from a prior import) is left untouched.
35pub fn ensure_categories<'a>(
36    model: &mut Model,
37    dims: impl IntoIterator<Item = (CategoryId, &'a str)>,
38) {
39    for (id, name) in dims {
40        model.categories.entry(id).or_insert_with(|| Category {
41            id,
42            name: Name(name.to_string()),
43            items: Vec::new(),
44        });
45    }
46}
47
48/// Create (or overwrite) the input measure an import populates.
49pub fn add_input_measure(
50    model: &mut Model,
51    measure_id: MeasureId,
52    measure_name: &str,
53    value_type: ValueType,
54    categories: Vec<CategoryId>,
55    description: Option<String>,
56) {
57    model.add_measure(Measure {
58        id: measure_id,
59        name: Name(measure_name.to_string()),
60        value_type,
61        categories,
62        kind: MeasureKind::Input,
63        description,
64    });
65}
66
67/// Per-category `name -> ItemId` interner used while importing rows: dimension
68/// values are looked up (reusing an existing item of that name) or minted a
69/// fresh sequential id. Seed from the model's current items so re-import /
70/// refresh reuses items instead of creating duplicates with the same name.
71#[derive(Debug, Default)]
72pub struct ItemInterner {
73    next_item: u32,
74    map: HashMap<CategoryId, HashMap<String, ItemId>>,
75}
76
77impl ItemInterner {
78    /// Seed the interner from `model`'s existing items. Minted ids start at
79    /// `item_id_base`, but never below `max(existing item id) + 1`: on a repeat
80    /// import the caller passes the same fixed base (the GUI wizard always
81    /// passes 1_000_000), so allocating straight from the base would re-mint an
82    /// id a *differently named* item already owns — `register` then keeps the
83    /// old name and the new row's value silently lands under the wrong label.
84    /// Allocating above the model's maximum guarantees freshness while the base
85    /// still acts as the floor for a fresh import.
86    pub fn seeded_from(model: &Model, item_id_base: u32) -> Self {
87        let mut map: HashMap<CategoryId, HashMap<String, ItemId>> = HashMap::new();
88        let mut max_id = 0u32;
89        for it in model.items.values() {
90            map.entry(it.category)
91                .or_default()
92                .insert(it.name.0.clone(), it.id);
93            max_id = max_id.max(it.id.0);
94        }
95        Self {
96            next_item: item_id_base.max(max_id.saturating_add(1)),
97            map,
98        }
99    }
100
101    /// Look up (or mint) the item id for `name` under `category`.
102    pub fn intern(&mut self, category: CategoryId, name: &str) -> ItemId {
103        if let Some(id) = self.map.entry(category).or_default().get(name) {
104            return *id;
105        }
106        let id = ItemId(self.next_item);
107        self.next_item += 1;
108        self.map
109            .get_mut(&category)
110            .expect("just inserted by or_default above")
111            .insert(name.to_string(), id);
112        id
113    }
114
115    /// Register every interned item into `model`'s `items`/`categories` maps.
116    /// Call once, after all rows are interned.
117    pub fn register(&self, model: &mut Model) {
118        for (cat, items) in &self.map {
119            for (name, id) in items {
120                model.items.entry(*id).or_insert_with(|| Item {
121                    id: *id,
122                    category: *cat,
123                    name: Name(name.clone()),
124                });
125                if let Some(c) = model.categories.get_mut(cat) {
126                    if !c.items.contains(id) {
127                        c.items.push(*id);
128                    }
129                }
130            }
131        }
132    }
133}
134
135#[cfg(test)]
136#[cfg_attr(coverage_nightly, coverage(off))]
137mod tests {
138    use super::*;
139    use improv_core_model::{Coordinate, ItemId};
140
141    #[test]
142    fn ensure_categories_is_idempotent_and_preserves_existing_items() {
143        let mut model = Model::new();
144        let cat = CategoryId(1);
145        ensure_categories(&mut model, [(cat, "Time")]);
146        model
147            .categories
148            .get_mut(&cat)
149            .unwrap()
150            .items
151            .push(ItemId(1));
152        // Calling again must not reset the category (or its items).
153        ensure_categories(&mut model, [(cat, "Time")]);
154        assert_eq!(model.categories.get(&cat).unwrap().items, vec![ItemId(1)]);
155    }
156
157    #[test]
158    fn interner_reuses_existing_items_by_name_and_mints_new_ones() {
159        let mut model = Model::new();
160        let cat = CategoryId(1);
161        ensure_categories(&mut model, [(cat, "Time")]);
162        model.items.insert(
163            ItemId(1),
164            Item {
165                id: ItemId(1),
166                category: cat,
167                name: Name("2025".into()),
168            },
169        );
170
171        let mut interner = ItemInterner::seeded_from(&model, 1000);
172        let existing = interner.intern(cat, "2025");
173        assert_eq!(existing, ItemId(1), "reuses the pre-existing item");
174        let fresh1 = interner.intern(cat, "2026");
175        let fresh2 = interner.intern(cat, "2026");
176        assert_eq!(fresh1, fresh2, "same name interns to the same id");
177        assert_ne!(fresh1, ItemId(1));
178
179        interner.register(&mut model);
180        assert_eq!(model.categories.get(&cat).unwrap().items.len(), 2);
181        assert!(model.items.contains_key(&fresh1));
182    }
183
184    #[test]
185    fn mints_above_existing_ids_even_when_base_is_lower() {
186        // A prior import left item 1_000_000 ⇒ re-seeding with the same fixed
187        // base must not hand that id to a different name.
188        let mut model = Model::new();
189        let cat = CategoryId(1);
190        ensure_categories(&mut model, [(cat, "Time")]);
191        model.items.insert(
192            ItemId(1_000_000),
193            Item {
194                id: ItemId(1_000_000),
195                category: cat,
196                name: Name("2025".into()),
197            },
198        );
199
200        let mut interner = ItemInterner::seeded_from(&model, 1_000_000);
201        let fresh = interner.intern(cat, "2026");
202        assert_ne!(fresh, ItemId(1_000_000));
203        interner.register(&mut model);
204        assert_eq!(model.items.get(&ItemId(1_000_000)).unwrap().name.0, "2025");
205        assert_eq!(model.items.get(&fresh).unwrap().name.0, "2026");
206    }
207
208    #[test]
209    fn add_input_measure_creates_an_input_measure_over_the_categories() {
210        let mut model = Model::new();
211        let cat = CategoryId(1);
212        add_input_measure(
213            &mut model,
214            MeasureId(1),
215            "Revenue",
216            ValueType::Number,
217            vec![cat],
218            Some("test".into()),
219        );
220        let m = model.measures.get(&MeasureId(1)).unwrap();
221        assert!(m.is_input());
222        assert_eq!(m.categories, vec![cat]);
223        // Sanity: a coordinate over that category can key an input cell.
224        let coord = Coordinate::new().with(cat, ItemId(1));
225        model.set_input(
226            MeasureId(1),
227            coord.clone(),
228            improv_core_model::Value::Number(1.0),
229        );
230        assert_eq!(
231            model.input(MeasureId(1), &coord),
232            Some(&improv_core_model::Value::Number(1.0))
233        );
234    }
235
236    #[test]
237    fn register_skips_known_items_and_unknown_categories() {
238        let mut model = Model::new();
239        model.add_category(CategoryId(1), "Time");
240        model.add_item(ItemId(5), CategoryId(1), "2025");
241        let mut interner = ItemInterner::seeded_from(&model, 100);
242        // Seeded: re-interning "2025" reuses id 5, already on the category.
243        assert_eq!(interner.intern(CategoryId(1), "2025"), ItemId(5));
244        let orphan = interner.intern(CategoryId(9), "x");
245        interner.register(&mut model);
246        assert_eq!(model.categories[&CategoryId(1)].items, [ItemId(5)]);
247        // An item for a category the model lacks is stored, attached to nothing.
248        assert_eq!(model.items[&orphan].category, CategoryId(9));
249        assert!(!model.categories.contains_key(&CategoryId(9)));
250    }
251}
improv_data_source/lib.rs

improv_data_source/
lib.rs
