[fine] Type checking

oden-script/src/tokens.rs

@@ -74,6 +74,10 @@ impl<'a> Token<'a> {
         }
     }
 
+    pub fn start(&self) -> usize {
+        self.start
+    }
+
     pub fn kind(&self) -> TokenKind {
         self.kind
     }
@@ -95,23 +99,22 @@ impl<'a> std::fmt::Display for Token<'a> {
     }
 }
 
-pub struct Tokens<'a> {
-    source: &'a str,
-    chars: std::str::CharIndices<'a>,
-    next_char: Option<(usize, char)>,
+pub struct Lines {
     newlines: Vec<usize>,
+    eof: usize,
 }
 
-impl<'a> Tokens<'a> {
-    pub fn new(source: &'a str) -> Self {
-        let mut result = Tokens {
-            source,
-            chars: source.char_indices(),
-            next_char: None,
+impl Lines {
+    fn new(eof: usize) -> Self {
+        Lines {
             newlines: Vec::new(),
-        };
-        result.advance(); // Prime the pump
-        result
+            eof,
+        }
+    }
+
+    /// Record the position of a newline in the source.
+    pub fn add_line(&mut self, pos: usize) {
+        self.newlines.push(pos)
     }
 
     /// Return the position of the given token as a (line, column) pair. By
@@ -122,9 +125,15 @@ impl<'a> Tokens<'a> {
     pub fn token_position(&self, token: &Option<Token>) -> (usize, usize) {
         let start = match token {
             Some(t) => t.start,
-            None => self.source.len(),
+            None => self.eof,
         };
-        let line_end_index = match self.newlines.binary_search(&start) {
+        self.position(start)
+    }
+
+    /// Return the position of the given character offset as a (line,column)
+    /// pair. By convention, lines are 1-based and columns are 0-based.
+    pub fn position(&self, offset: usize) -> (usize, usize) {
+        let line_end_index = match self.newlines.binary_search(&offset) {
             Ok(index) => index,
             Err(index) => index,
         };
@@ -134,9 +143,39 @@ impl<'a> Tokens<'a> {
             self.newlines[line_end_index - 1] + 1
         };
         let line_number = line_end_index + 1;
-        let column_offset = start - line_start_pos;
+        let column_offset = offset - line_start_pos;
         (line_number, column_offset)
     }
+}
+
+pub struct Tokens<'a> {
+    source: &'a str,
+    chars: std::str::CharIndices<'a>,
+    next_char: Option<(usize, char)>,
+    lines: Lines,
+}
+
+impl<'a> Tokens<'a> {
+    pub fn new(source: &'a str) -> Self {
+        let mut result = Tokens {
+            source,
+            chars: source.char_indices(),
+            next_char: None,
+            lines: Lines::new(source.len()),
+        };
+        result.advance(); // Prime the pump
+        result
+    }
+
+    pub fn lines(self) -> Lines {
+        self.lines
+    }
+
+    /// Return the position of the given token as a (line, column) pair. See
+    /// `Lines::token_position` for more information about the range, etc.
+    pub fn token_position(&self, token: &Option<Token>) -> (usize, usize) {
+        self.lines.token_position(token)
+    }
 
     fn token(&self, start: usize, kind: TokenKind) -> Token<'a> {
         let value = &self.source[start..self.pos()];
@@ -363,7 +402,7 @@ impl<'a> Tokens<'a> {
     fn skip_whitespace(&mut self) {
         while let Some((pos, ch)) = self.next_char {
             if ch == '\n' {
-                self.newlines.push(pos);
+                self.lines.add_line(pos);
             } else if !ch.is_whitespace() {
                 break;
             }