use enumset::{EnumSet, EnumSetType}; use super::SyntaxElement; pub struct Input<'src, SyntaxKind: SyntaxElement> { raw: Vec<(SyntaxKind, &'src str)>, // enumset of meaningless tokens semantically_meaningless: EnumSet, // indices of non-meaningless tokens meaningful_toks: Vec, } impl<'src, SyntaxKind: SyntaxElement> Input<'src, SyntaxKind> { pub fn new( raw_toks: Vec<(SyntaxKind, &'src str)>, meaningless: Option>, ) -> Self { let mut meaningful_toks = Vec::new(); if let Some(meaningless) = meaningless { let meaningful_toks = raw_toks .iter() .enumerate() .filter_map(|(i, tok)| (!meaningless.contains(tok.0)).then_some(i)) .collect_into(&mut meaningful_toks); } Self { raw: raw_toks, semantically_meaningless: meaningless.unwrap_or_default(), meaningful_toks, } } pub fn kind(&self, idx: usize) -> SyntaxKind { let Some(meaningful_idx) = self.meaningful_toks.get(idx) else { return SyntaxKind::EOF; }; self.raw.get(*meaningful_idx).unwrap().0 } pub fn preceding_meaningless(&self, idx: usize) -> usize { assert!(self.meaningful_toks.len() > idx); if idx == 0 { // maybe should be `self.meaningful_toks[idx]` instead?? 1 } else { self.meaningful_toks[idx] - self.meaningful_toks[idx - 1] } } /// get the count of meaningless tokens at the end of the file. pub fn meaningless_tail_len(&self) -> usize { self.raw.len() - (self.meaningful_toks.last().unwrap() + 1) } pub fn dissolve(self) -> (Vec<(SyntaxKind, &'src str)>, EnumSet) { let Self { raw, semantically_meaningless, .. } = self; (raw, semantically_meaningless) } }