[fine] Bring back some test coverage

fine/src/tokens.rs

@@ -547,8 +547,9 @@ mod tests {
     use super::*;
     use pretty_assertions::assert_eq;
 
-    fn test_tokens_impl(input: &str, expected: Vec<Token>) {
+    fn test_tokens_impl(input: &str, expected: Vec<Token>, expected_text: Vec<String>) {
         let mut result = Vec::new();
+        let mut result_text = Vec::new();
         let mut tokens = Tokens::new(input);
         let mut is_eof = false;
         while !is_eof {
@@ -557,10 +558,12 @@ mod tests {
             if token.kind == TokenKind::Whitespace {
                 continue;
             }
+            result_text.push(token.as_str(input).to_string());
             result.push(token);
         }
 
         assert_eq!(expected, result);
+        assert_eq!(expected_text, result_text);
     }
 
     macro_rules! test_tokens {
@@ -575,7 +578,13 @@ mod tests {
                     .collect();
                 expected.push(Token::new(TokenKind::EOF, $input.len(), $input.len()));
 
-                test_tokens_impl($input, expected);
+                let mut expected_text: Vec<_> = (vec![$($s),*])
+                    .into_iter()
+                    .map(|t| t.2.to_string())
+                    .collect();
+                expected_text.push("".to_string());
+
+                test_tokens_impl($input, expected, expected_text);
             }
         }
     }