1use std::ops::Range;
2
3#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
4pub enum Kind {
5 Key,
6 Header,
7 String,
8 Number,
9 Boolean,
10 Date,
11 Comment,
12 Punctuation,
13 Plain,
14 Invalid,
15}
16
17#[derive(Clone, Debug, PartialEq, Eq, Hash)]
18pub struct Token {
19 pub kind: Kind,
20 pub range: Range<usize>,
21}
22
23#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
24enum Open {
25 #[default]
26 None,
27 Basic,
28 Literal,
29}
30
31#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
32enum Expect {
33 #[default]
34 Key,
35 Value,
36 After,
37}
38
39#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
40pub struct State {
41 open: Open,
42 expect: Expect,
43 depth: u8,
44 tables: u64,
45}
46
47impl State {
48 pub fn in_string(&self) -> bool {
49 self.open != Open::None
50 }
51
52 pub fn depth(&self) -> usize {
53 usize::from(self.depth)
54 }
55
56 fn in_table(&self) -> bool {
57 self.depth > 0 && self.depth <= 64 && self.tables & (1 << (self.depth - 1)) != 0
58 }
59
60 fn push(&mut self, table: bool) {
61 if self.depth == u8::MAX {
62 return;
63 }
64 if self.depth < 64 {
65 let bit = 1 << self.depth;
66 if table {
67 self.tables |= bit;
68 } else {
69 self.tables &= !bit;
70 }
71 }
72 self.depth += 1;
73 }
74
75 fn pop(&mut self) {
76 self.depth -= 1;
77 if self.depth < 64 {
78 self.tables &= !(1 << self.depth);
79 }
80 self.expect = Expect::After;
81 }
82}
83
84struct Lexer<'a> {
85 bytes: &'a [u8],
86 at: usize,
87 state: State,
88 out: Vec<Token>,
89}
90
91fn bare(byte: u8) -> bool {
92 byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-'
93}
94
95fn word(byte: u8) -> bool {
96 bare(byte) || matches!(byte, b'+' | b'.' | b':')
97}
98
99fn char_len(byte: u8) -> usize {
100 match byte {
101 0xF0.. => 4,
102 0xE0.. => 3,
103 0xC0.. => 2,
104 _ => 1,
105 }
106}
107
108fn digits(bytes: &[u8], count: usize) -> bool {
109 bytes.len() >= count && bytes[..count].iter().all(u8::is_ascii_digit)
110}
111
112fn date_like(bytes: &[u8]) -> bool {
113 (digits(bytes, 4) && bytes.get(4) == Some(&b'-'))
114 || (digits(bytes, 2) && bytes.get(2) == Some(&b':'))
115}
116
117fn number_like(bytes: &[u8]) -> bool {
118 let body = match bytes.first() {
119 Some(b'+' | b'-') => &bytes[1..],
120 _ => bytes,
121 };
122 if body == b"inf" || body == b"nan" {
123 return true;
124 }
125 if let [b'0', b'x' | b'o' | b'b', rest @ ..] = body {
126 return !rest.is_empty() && rest.iter().all(|b| b.is_ascii_hexdigit() || *b == b'_');
127 }
128 body.first().is_some_and(u8::is_ascii_digit)
129 && body
130 .iter()
131 .all(|b| b.is_ascii_digit() || matches!(b, b'_' | b'.' | b'e' | b'E' | b'+' | b'-'))
132}
133
134impl Lexer<'_> {
135 fn peek(&self, offset: usize) -> Option<u8> {
136 self.bytes.get(self.at + offset).copied()
137 }
138
139 fn emit(&mut self, kind: Kind, end: usize) {
140 let start = self.at;
141 self.at = end;
142 if start == end {
143 return;
144 }
145 if let Some(last) = self.out.last_mut()
146 && last.kind == kind
147 && last.range.end == start
148 {
149 last.range.end = end;
150 return;
151 }
152 self.out.push(Token {
153 kind,
154 range: start..end,
155 });
156 }
157
158 fn line_end(&self) -> usize {
159 self.bytes[self.at..]
160 .iter()
161 .position(|b| *b == b'\n')
162 .map_or(self.bytes.len(), |offset| self.at + offset)
163 }
164
165 fn invalid_char(&mut self) {
166 let end = (self.at + char_len(self.bytes[self.at])).min(self.bytes.len());
167 self.emit(Kind::Invalid, end);
168 }
169
170 fn run(&self, keep: impl Fn(u8) -> bool) -> usize {
171 self.bytes[self.at..]
172 .iter()
173 .position(|b| !keep(*b))
174 .map_or(self.bytes.len(), |offset| self.at + offset)
175 }
176
177 fn line(mut self) -> (Vec<Token>, State) {
178 while self.at < self.bytes.len() {
179 if self.state.in_string() {
180 self.open_string();
181 continue;
182 }
183 match self.bytes[self.at] {
184 b'\n' => {
185 self.emit(Kind::Plain, self.at + 1);
186 if self.state.depth == 0 {
187 self.state.expect = Expect::Key;
188 }
189 }
190 b' ' | b'\t' | b'\r' => {
191 let end = self.run(|b| matches!(b, b' ' | b'\t' | b'\r'));
192 self.emit(Kind::Plain, end);
193 }
194 b'#' => {
195 let end = self.line_end();
196 self.emit(Kind::Comment, end);
197 }
198 _ => match self.state.expect {
199 Expect::Key => self.key(),
200 Expect::Value => self.value(),
201 Expect::After => self.after(),
202 },
203 }
204 }
205 (self.out, self.state)
206 }
207
208 fn quoted(&self, quote: u8) -> Option<usize> {
209 let mut at = self.at + 1;
210 while let Some(&byte) = self.bytes.get(at) {
211 match byte {
212 b'\n' => return None,
213 b'\\' if quote == b'"' => {
214 at += if self.bytes.get(at + 1) == Some(&b'\n') {
215 1
216 } else {
217 2
218 }
219 }
220 _ if byte == quote => return Some(at + 1),
221 _ => at += 1,
222 }
223 }
224 None
225 }
226
227 fn string(&mut self, kind: Kind) {
228 let quote = self.bytes[self.at];
229 match self.quoted(quote) {
230 Some(end) => self.emit(kind, end),
231 None => {
232 let end = self.line_end();
233 self.emit(Kind::Invalid, end);
234 }
235 }
236 }
237
238 fn key(&mut self) {
239 let byte = self.bytes[self.at];
240 match byte {
241 b'[' if self.state.depth == 0 => self.header(),
242 b'"' | b'\'' => self.string(Kind::Key),
243 b'.' => self.emit(Kind::Punctuation, self.at + 1),
244 b'=' => {
245 self.emit(Kind::Punctuation, self.at + 1);
246 self.state.expect = Expect::Value;
247 }
248 b'}' | b',' if self.state.in_table() => self.after(),
249 _ if bare(byte) => {
250 let end = self.run(bare);
251 self.emit(Kind::Key, end);
252 }
253 _ => self.invalid_char(),
254 }
255 }
256
257 fn header(&mut self) {
258 let double = self.peek(1) == Some(b'[');
259 let mut at = self.at + if double { 2 } else { 1 };
260 let close: &[u8] = if double { b"]]" } else { b"]" };
261 while let Some(&byte) = self.bytes.get(at) {
262 match byte {
263 b'\n' | b'#' => break,
264 b'"' | b'\'' => {
265 let from = self.at;
266 self.at = at;
267 let end = self.quoted(byte);
268 self.at = from;
269 match end {
270 Some(end) => at = end,
271 None => break,
272 }
273 }
274 b']' if self.bytes[at..].starts_with(close) => {
275 self.emit(Kind::Header, at + close.len());
276 self.state.expect = Expect::After;
277 return;
278 }
279 _ => at += 1,
280 }
281 }
282 let end = self.line_end();
283 let end = self.bytes[self.at..end]
284 .iter()
285 .position(|b| *b == b'#')
286 .map_or(end, |offset| self.at + offset);
287 self.emit(Kind::Invalid, end);
288 self.state.expect = Expect::After;
289 }
290
291 fn value(&mut self) {
292 let byte = self.bytes[self.at];
293 match byte {
294 b'"' | b'\'' => {
295 if self.bytes[self.at..].starts_with(&[byte; 3]) {
296 self.emit(Kind::String, self.at + 3);
297 self.state.open = if byte == b'"' {
298 Open::Basic
299 } else {
300 Open::Literal
301 };
302 } else {
303 self.string(Kind::String);
304 self.state.expect = Expect::After;
305 }
306 }
307 b'[' => {
308 self.emit(Kind::Punctuation, self.at + 1);
309 self.state.push(false);
310 self.state.expect = Expect::Value;
311 }
312 b'{' => {
313 self.emit(Kind::Punctuation, self.at + 1);
314 self.state.push(true);
315 self.state.expect = Expect::Key;
316 }
317 b']' | b'}' => self.after(),
318 _ if word(byte) => self.scalar(),
319 _ => self.invalid_char(),
320 }
321 }
322
323 fn scalar(&mut self) {
324 let mut end = self.run(word);
325 let text = &self.bytes[self.at..end];
326 let kind = if text == b"true" || text == b"false" {
327 Kind::Boolean
328 } else if date_like(text) {
329 if text.len() == 10
330 && self.bytes.get(end) == Some(&b' ')
331 && date_like(&self.bytes[end + 1..])
332 {
333 let from = self.at;
334 self.at = end + 1;
335 end = self.run(word);
336 self.at = from;
337 }
338 Kind::Date
339 } else if number_like(text) {
340 Kind::Number
341 } else {
342 Kind::Invalid
343 };
344 self.emit(kind, end);
345 self.state.expect = Expect::After;
346 }
347
348 fn after(&mut self) {
349 let byte = self.bytes[self.at];
350 let depth = self.state.depth;
351 match byte {
352 b',' if depth > 0 => {
353 self.emit(Kind::Punctuation, self.at + 1);
354 self.state.expect = if self.state.in_table() {
355 Expect::Key
356 } else {
357 Expect::Value
358 };
359 }
360 b']' if depth > 0 && !self.state.in_table() => {
361 self.emit(Kind::Punctuation, self.at + 1);
362 self.state.pop();
363 }
364 b'}' if self.state.in_table() => {
365 self.emit(Kind::Punctuation, self.at + 1);
366 self.state.pop();
367 }
368 _ => self.invalid_char(),
369 }
370 }
371
372 fn open_string(&mut self) {
373 let quote = if self.state.open == Open::Basic {
374 b'"'
375 } else {
376 b'\''
377 };
378 let mut at = self.at;
379 while let Some(&byte) = self.bytes.get(at) {
380 match byte {
381 b'\n' => {
382 self.emit(Kind::String, at + 1);
383 return;
384 }
385 b'\\' if quote == b'"' => {
386 at += if self.bytes.get(at + 1) == Some(&b'\n') {
387 1
388 } else {
389 2
390 }
391 }
392 _ if self.bytes[at..].starts_with(&[quote; 3]) => {
393 let extra = self.bytes[at + 3..]
394 .iter()
395 .take(2)
396 .take_while(|b| **b == quote)
397 .count();
398 self.emit(Kind::String, at + 3 + extra);
399 self.state.open = Open::None;
400 self.state.expect = Expect::After;
401 return;
402 }
403 _ => at += 1,
404 }
405 }
406 self.emit(Kind::String, self.bytes.len());
407 }
408}
409
410pub fn lex_line(line: &str, state: State) -> (Vec<Token>, State) {
411 Lexer {
412 bytes: line.as_bytes(),
413 at: 0,
414 state,
415 out: Vec::new(),
416 }
417 .line()
418}
419
420#[derive(Clone, Debug, PartialEq, Eq)]
421pub struct Line {
422 pub start: usize,
423 pub state: State,
424 pub tokens: Vec<Token>,
425}
426
427#[derive(Clone, Debug, PartialEq, Eq)]
428pub struct Lexed {
429 text: String,
430 lines: Vec<Line>,
431}
432
433fn line_ends(text: &str, from: usize) -> impl Iterator<Item = usize> + '_ {
434 let bytes = text.as_bytes();
435 let mut at = Some(from);
436 std::iter::from_fn(move || {
437 let start = at?;
438 match bytes[start..].iter().position(|b| *b == b'\n') {
439 Some(offset) => at = Some(start + offset + 1),
440 None => at = None,
441 }
442 Some(at.unwrap_or(bytes.len()))
443 })
444}
445
446impl Default for Lexed {
447 fn default() -> Lexed {
448 Lexed::new("")
449 }
450}
451
452impl Lexed {
453 pub fn new(text: &str) -> Lexed {
454 let mut lexed = Lexed {
455 text: String::new(),
456 lines: Vec::new(),
457 };
458 lexed.relex(text, 0, State::default(), None);
459 lexed
460 }
461
462 pub fn text(&self) -> &str {
463 &self.text
464 }
465
466 pub fn lines(&self) -> &[Line] {
467 &self.lines
468 }
469
470 pub fn tokens(&self) -> impl Iterator<Item = Token> + '_ {
471 self.lines.iter().flat_map(|line| {
472 line.tokens.iter().map(|token| Token {
473 kind: token.kind,
474 range: token.range.start + line.start..token.range.end + line.start,
475 })
476 })
477 }
478
479 pub fn line_of(&self, byte: usize) -> usize {
480 self.lines.partition_point(|line| line.start <= byte).max(1) - 1
481 }
482
483 pub fn update(&mut self, text: &str) -> usize {
484 if text == self.text {
485 return 0;
486 }
487 let old = self.text.as_bytes();
488 let new = text.as_bytes();
489 let prefix = old.iter().zip(new).take_while(|(a, b)| a == b).count();
490 let room = old.len().min(new.len()) - prefix;
491 let suffix = old
492 .iter()
493 .rev()
494 .zip(new.iter().rev())
495 .take(room)
496 .take_while(|(a, b)| a == b)
497 .count();
498 let first = self.line_of(prefix);
499 let start = self.lines[first].start;
500 let state = self.lines[first].state;
501 let tail = Tail {
502 lines: self.lines.split_off(first),
503 suffix,
504 };
505 self.relex(text, start, state, Some(tail))
506 }
507
508 fn relex(&mut self, text: &str, start: usize, mut state: State, tail: Option<Tail>) -> usize {
509 let delta = text.len() as isize - self.text.len() as isize;
510 let unchanged_from = tail.as_ref().map(|tail| text.len() - tail.suffix);
511 let mut relexed = 0;
512 let mut line_start = start;
513 let mut ends = line_ends(text, start).peekable();
514 while let Some(end) = ends.next() {
515 let (tokens, next) = lex_line(&text[line_start..end], state);
516 self.lines.push(Line {
517 start: line_start,
518 state,
519 tokens,
520 });
521 relexed += 1;
522 state = next;
523 line_start = end;
524 if ends.peek().is_none() {
525 break;
526 }
527 if let (Some(tail), Some(from)) = (&tail, unchanged_from)
528 && line_start >= from
529 {
530 let old_start = (line_start as isize - delta) as usize;
531 let at = tail.lines.partition_point(|line| line.start < old_start);
532 if let Some(line) = tail.lines.get(at)
533 && line.start == old_start
534 && line.state == state
535 {
536 self.lines.extend(tail.lines[at..].iter().map(|line| Line {
537 start: (line.start as isize + delta) as usize,
538 state: line.state,
539 tokens: line.tokens.clone(),
540 }));
541 break;
542 }
543 }
544 }
545 self.text.clear();
546 self.text.push_str(text);
547 relexed
548 }
549}
550
551struct Tail {
552 lines: Vec<Line>,
553 suffix: usize,
554}
555
556pub fn lex(text: &str) -> Vec<Token> {
557 Lexed::new(text).tokens().collect()
558}
559
560#[cfg(test)]
561mod tests {
562 use super::*;
563
564 fn kinds(text: &str) -> Vec<(Kind, &str)> {
565 let mut merged: Vec<Token> = Vec::new();
566 for token in lex(text) {
567 match merged.last_mut() {
568 Some(last) if last.kind == token.kind && last.range.end == token.range.start => {
569 last.range.end = token.range.end
570 }
571 _ => merged.push(token),
572 }
573 }
574 merged
575 .into_iter()
576 .filter(|token| token.kind != Kind::Plain)
577 .map(|token| (token.kind, &text[token.range]))
578 .collect()
579 }
580
581 fn covers(text: &str) {
582 let tokens = lex(text);
583 let mut at = 0;
584 for token in &tokens {
585 assert_eq!(token.range.start, at, "{text:?}: {tokens:?}");
586 assert!(token.range.end > token.range.start, "{text:?}");
587 assert!(text.is_char_boundary(token.range.end), "{text:?}");
588 at = token.range.end;
589 }
590 assert_eq!(at, text.len(), "{text:?}");
591 }
592
593 #[test]
594 fn every_kind_lexes_from_valid_toml() {
595 let text = "# scene\n[object.\"lamp a\"]\nname = 'lamp'\nlit = true\nsize = 1.5e3\nhex = 0xFF_00\nwhen = 1979-05-27 07:32:00Z\nat = 07:32:00\nlist = [1, -2, +inf]\n[[light]]\ncolor = { r = 1, g = 0 }\n";
596 assert_eq!(
597 kinds(text),
598 vec![
599 (Kind::Comment, "# scene"),
600 (Kind::Header, "[object.\"lamp a\"]"),
601 (Kind::Key, "name"),
602 (Kind::Punctuation, "="),
603 (Kind::String, "'lamp'"),
604 (Kind::Key, "lit"),
605 (Kind::Punctuation, "="),
606 (Kind::Boolean, "true"),
607 (Kind::Key, "size"),
608 (Kind::Punctuation, "="),
609 (Kind::Number, "1.5e3"),
610 (Kind::Key, "hex"),
611 (Kind::Punctuation, "="),
612 (Kind::Number, "0xFF_00"),
613 (Kind::Key, "when"),
614 (Kind::Punctuation, "="),
615 (Kind::Date, "1979-05-27 07:32:00Z"),
616 (Kind::Key, "at"),
617 (Kind::Punctuation, "="),
618 (Kind::Date, "07:32:00"),
619 (Kind::Key, "list"),
620 (Kind::Punctuation, "="),
621 (Kind::Punctuation, "["),
622 (Kind::Number, "1"),
623 (Kind::Punctuation, ","),
624 (Kind::Number, "-2"),
625 (Kind::Punctuation, ","),
626 (Kind::Number, "+inf"),
627 (Kind::Punctuation, "]"),
628 (Kind::Header, "[[light]]"),
629 (Kind::Key, "color"),
630 (Kind::Punctuation, "="),
631 (Kind::Punctuation, "{"),
632 (Kind::Key, "r"),
633 (Kind::Punctuation, "="),
634 (Kind::Number, "1"),
635 (Kind::Punctuation, ","),
636 (Kind::Key, "g"),
637 (Kind::Punctuation, "="),
638 (Kind::Number, "0"),
639 (Kind::Punctuation, "}"),
640 ]
641 );
642 covers(text);
643 }
644
645 #[test]
646 fn dotted_and_quoted_keys_are_keys() {
647 assert_eq!(
648 kinds("a.\"b c\".'d' = \"x\\\"y\""),
649 vec![
650 (Kind::Key, "a"),
651 (Kind::Punctuation, "."),
652 (Kind::Key, "\"b c\""),
653 (Kind::Punctuation, "."),
654 (Kind::Key, "'d'"),
655 (Kind::Punctuation, "="),
656 (Kind::String, "\"x\\\"y\""),
657 ]
658 );
659 }
660
661 #[test]
662 fn multi_line_strings_and_arrays_carry_across_lines() {
663 let text = "doc = \"\"\"\none \\\"\"\" still\ntwo\"\"\"\"\nraw = '''\n'''\nlist = [\n 1, # one\n \"two\",\n]\nnext = 1\n";
664 assert_eq!(
665 kinds(text),
666 vec![
667 (Kind::Key, "doc"),
668 (Kind::Punctuation, "="),
669 (Kind::String, "\"\"\"\none \\\"\"\" still\ntwo\"\"\"\""),
670 (Kind::Key, "raw"),
671 (Kind::Punctuation, "="),
672 (Kind::String, "'''\n'''"),
673 (Kind::Key, "list"),
674 (Kind::Punctuation, "="),
675 (Kind::Punctuation, "["),
676 (Kind::Number, "1"),
677 (Kind::Punctuation, ","),
678 (Kind::Comment, "# one"),
679 (Kind::String, "\"two\""),
680 (Kind::Punctuation, ","),
681 (Kind::Punctuation, "]"),
682 (Kind::Key, "next"),
683 (Kind::Punctuation, "="),
684 (Kind::Number, "1"),
685 ]
686 );
687 covers(text);
688 }
689
690 #[test]
691 fn broken_input_lexes_to_invalid_and_covers_every_byte() {
692 for text in [
693 "a = \"open\nb = 1\n",
694 "[\nkey = 1\n",
695 "[table\n",
696 "[[double]\n",
697 "a = 'open",
698 "\u{1}\u{7f}é = ✓ 1\n",
699 "a = \"\"\"never closed\nstill\n",
700 "a = '''never\n",
701 "a = 1 2 ]\n",
702 "a = [1,, }\n",
703 "a = {b = 1\n",
704 "] } , = .\n",
705 "a = tru\nb = 1x\n",
706 "\"",
707 "\\",
708 "a = \"\\",
709 "",
710 "\n\n\r\n",
711 ] {
712 covers(text);
713 }
714 assert_eq!(
715 kinds("a = \"open\nb = 1"),
716 vec![
717 (Kind::Key, "a"),
718 (Kind::Punctuation, "="),
719 (Kind::Invalid, "\"open"),
720 (Kind::Key, "b"),
721 (Kind::Punctuation, "="),
722 (Kind::Number, "1"),
723 ]
724 );
725 assert_eq!(kinds("[\nk = 1")[0], (Kind::Invalid, "["));
726 assert_eq!(kinds("[\nk = 1")[1], (Kind::Key, "k"));
727 assert_eq!(kinds("é = 1")[0], (Kind::Invalid, "é"));
728 assert_eq!(kinds("a = tru")[2], (Kind::Invalid, "tru"));
729 let open = kinds("a = \"\"\"never\nb = 1\n");
730 assert_eq!(open[2], (Kind::String, "\"\"\"never\nb = 1\n"));
731 }
732
733 #[test]
734 fn every_byte_is_covered_for_every_prefix() {
735 let text = "[a.\"b\"]\nx = \"\"\"m\n\"\"\"\ny = [1, {z = 'q'}]\n# end ✓\n";
736 for end in 0..=text.len() {
737 if text.is_char_boundary(end) {
738 covers(&text[..end]);
739 }
740 }
741 }
742
743 struct Seeded(u64);
744
745 impl Seeded {
746 fn next(&mut self) -> u64 {
747 self.0 = self
748 .0
749 .wrapping_mul(6364136223846793005)
750 .wrapping_add(1442695040888963407);
751 self.0 >> 33
752 }
753
754 fn below(&mut self, count: usize) -> usize {
755 (self.next() % count as u64) as usize
756 }
757 }
758
759 const SNIPPETS: &[&str] = &[
760 "key = 1",
761 "doc = \"\"\"",
762 "\"\"\"",
763 "raw = '''",
764 "'''",
765 "list = [",
766 "]",
767 " 2,",
768 "t = { a = 1",
769 "}",
770 "[table]",
771 "[",
772 "# comment",
773 "",
774 "s = \"open",
775 "é ✓",
776 "when = 1979-05-27 07:32:00",
777 ];
778
779 fn sample(lines: usize) -> String {
780 let mut text = String::new();
781 for index in 0..lines {
782 text.push_str(match index % 9 {
783 0 => "[object.n]",
784 1 => "name = \"lamp\"",
785 2 => "at = [0.0, 1.5, -2]",
786 3 => "doc = \"\"\"",
787 4 => "inside",
788 5 => "\"\"\"",
789 6 => "list = [",
790 7 => " 1, 2,",
791 _ => "]",
792 });
793 text.push('\n');
794 }
795 text
796 }
797
798 #[test]
799 fn incremental_relex_equals_a_full_lex_after_random_line_edits() {
800 for seed in [1_u64, 7, 42, 1234, 99991] {
801 let mut random = Seeded(seed);
802 let mut text = sample(40);
803 let mut lexed = Lexed::new(&text);
804 for _ in 0..200 {
805 let lines: Vec<&str> = text.split_inclusive('\n').collect();
806 let at = random.below(lines.len() + 1);
807 let snippet = SNIPPETS[random.below(SNIPPETS.len())];
808 let mut next = String::new();
809 for (index, line) in lines.iter().enumerate() {
810 if index == at {
811 match random.below(3) {
812 0 => {
813 next.push_str(snippet);
814 next.push('\n');
815 }
816 1 => {
817 next.push_str(snippet);
818 next.push('\n');
819 next.push_str(line);
820 }
821 _ => {}
822 }
823 } else {
824 next.push_str(line);
825 }
826 }
827 if at == lines.len() {
828 next.push_str(snippet);
829 }
830 text = next;
831 lexed.update(&text);
832 assert_eq!(lexed, Lexed::new(&text), "seed {seed}: {text:?}");
833 }
834 }
835 }
836
837 #[test]
838 fn incremental_relex_equals_a_full_lex_after_random_byte_edits() {
839 let mut random = Seeded(5);
840 let mut text = sample(30);
841 let mut lexed = Lexed::new(&text);
842 for _ in 0..500 {
843 let mut at = random.below(text.len() + 1);
844 while !text.is_char_boundary(at) {
845 at -= 1;
846 }
847 if random.below(2) == 0 && at < text.len() {
848 let end = at + text[at..].chars().next().unwrap().len_utf8();
849 text.replace_range(at..end, "");
850 } else {
851 let inserted = ["\"", "'", "[", "]", "\n", "x", "{", "}", "#", "="];
852 text.insert_str(at, inserted[random.below(inserted.len())]);
853 }
854 lexed.update(&text);
855 assert_eq!(lexed, Lexed::new(&text), "{text:?}");
856 }
857 }
858
859 #[test]
860 fn an_edit_relexes_only_until_the_state_matches_again() {
861 let text = sample(10_000);
862 let mut lexed = Lexed::new(&text);
863 assert_eq!(lexed.lines().len(), 10_001);
864 let at = text.find("name = \"lamp\"").unwrap() + 8;
865 let mut edited = text.clone();
866 edited.insert(at, 'x');
867 assert_eq!(lexed.update(&edited), 1);
868 assert_eq!(lexed, Lexed::new(&edited));
869 let line = edited.find("[object.n]").unwrap() + "[object.n]\n".len();
870 edited.insert_str(line, "s = '''\n");
871 assert!(lexed.update(&edited) > 9_000);
872 assert_eq!(lexed, Lexed::new(&edited));
873 assert_eq!(lexed.update(&edited), 0);
874 }
875}