1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
use crate::model::buffer::TextBuffer;
/// Iterator over lines in a TextBuffer with bidirectional support
/// Uses piece iterator for efficient sequential scanning (ONE O(log n) initialization)
///
/// # Performance Characteristics
///
/// Line tracking is now always computed when chunks are loaded:
/// - **All loaded chunks**: `line_starts = Vec<usize>` → exact line metadata available
/// - **Unloaded chunks**: Only metadata unavailable until first access
///
/// ## Current Performance:
/// - **Forward iteration (`next()`)**: ✅ Efficient O(1) amortized per line using piece iterator
/// - **Backward iteration (`prev()`)**: ✅ O(log n) using piece tree line indexing
/// - **Initialization (`new()`)**: ✅ O(log n) using offset_to_position
///
/// ## Design:
/// - Loaded chunks are always indexed (10% memory overhead per chunk)
/// - Cursor vicinity is always loaded and indexed → 100% accurate navigation
/// - Forward scanning with lazy loading handles long lines efficiently
/// - Backward navigation uses piece tree's line_range() lookup
///
/// The `estimated_line_length` parameter is still used for forward scanning to estimate
/// initial chunk sizes, but line boundaries are always accurate after data is loaded.
pub struct LineIterator<'a> {
buffer: &'a mut TextBuffer,
/// Current byte position in the document (points to start of current line)
current_pos: usize,
buffer_len: usize,
/// Estimated average line length in bytes (for large file estimation)
estimated_line_length: usize,
}
impl<'a> LineIterator<'a> {
/// Scan backward from byte_pos to find the start of the line
/// chunk_size: suggested chunk size for loading (used as performance hint only)
fn find_line_start_backward(
buffer: &mut TextBuffer,
byte_pos: usize,
chunk_size: usize,
) -> usize {
if byte_pos == 0 {
return 0;
}
// Scan backward in chunks until we find a newline or reach position 0
// The chunk_size is just a hint for performance - we MUST find the actual line start
let mut search_end = byte_pos;
loop {
let scan_start = search_end.saturating_sub(chunk_size);
let scan_len = search_end - scan_start;
// Load the chunk we need to scan
if let Ok(chunk) = buffer.get_text_range_mut(scan_start, scan_len) {
// Scan backward through the chunk to find the last newline
for i in (0..chunk.len()).rev() {
if chunk[i] == b'\n' {
// Found newline - line starts at the next byte
return scan_start + i + 1;
}
}
}
// No newline found in this chunk
if scan_start == 0 {
// Reached the start of the buffer - line starts at 0
return 0;
}
// Continue searching from earlier position
search_end = scan_start;
}
}
pub(crate) fn new(
buffer: &'a mut TextBuffer,
byte_pos: usize,
estimated_line_length: usize,
) -> Self {
let buffer_len = buffer.len();
let byte_pos = byte_pos.min(buffer_len);
// Find the start of the line containing byte_pos
let line_start = if byte_pos == 0 {
0
} else {
// CRITICAL: Pre-load the chunk containing byte_pos to ensure offset_to_position works
// Handle EOF case where byte_pos might equal buffer_len
let pos_to_load = if byte_pos >= buffer_len {
buffer_len.saturating_sub(1)
} else {
byte_pos
};
if pos_to_load < buffer_len {
let _ = buffer.get_text_range_mut(pos_to_load, 1);
}
// Scan backward from byte_pos to find the start of the line
// We scan backward looking for a newline character
// NOTE: We previously tried to use offset_to_position() but it has bugs with column calculation
Self::find_line_start_backward(buffer, byte_pos, estimated_line_length)
};
LineIterator {
buffer,
current_pos: line_start,
buffer_len,
estimated_line_length,
}
}
/// Get the next line (moving forward)
/// Uses lazy loading to handle unloaded buffers transparently
pub fn next(&mut self) -> Option<(usize, String)> {
if self.current_pos >= self.buffer_len {
return None;
}
let line_start = self.current_pos;
// Estimate line length for chunk loading (typically lines are < 200 bytes)
// We load more than average to handle long lines without multiple loads
let estimated_max_line_length = self.estimated_line_length * 3;
let bytes_to_scan = estimated_max_line_length.min(self.buffer_len - self.current_pos);
// Use get_text_range_mut() which handles lazy loading automatically
// This never scans the entire file - only loads the chunk needed for this line
let chunk = match self
.buffer
.get_text_range_mut(self.current_pos, bytes_to_scan)
{
Ok(data) => data,
Err(e) => {
tracing::error!(
"LineIterator: Failed to load chunk at offset {}: {}",
self.current_pos,
e
);
return None;
}
};
// Scan for newline in the loaded chunk
let mut line_len = 0;
let mut found_newline = false;
for &byte in chunk.iter() {
line_len += 1;
if byte == b'\n' {
found_newline = true;
break;
}
}
// If we didn't find a newline and didn't reach EOF, the line is longer than our estimate
// Load more data iteratively (rare case for very long lines)
if !found_newline && self.current_pos + line_len < self.buffer_len {
// Line is longer than expected, keep loading until we find newline or EOF
let mut extended_chunk = chunk;
while !found_newline && self.current_pos + extended_chunk.len() < self.buffer_len {
let additional_bytes = estimated_max_line_length
.min(self.buffer_len - self.current_pos - extended_chunk.len());
match self
.buffer
.get_text_range_mut(self.current_pos + extended_chunk.len(), additional_bytes)
{
Ok(mut more_data) => {
let start_len = extended_chunk.len();
extended_chunk.append(&mut more_data);
// Scan the newly added portion
for &byte in extended_chunk[start_len..].iter() {
line_len += 1;
if byte == b'\n' {
found_newline = true;
break;
}
}
}
Err(e) => {
tracing::error!("LineIterator: Failed to extend chunk: {}", e);
break;
}
}
}
// Use the extended chunk
let line_bytes = &extended_chunk[..line_len];
self.current_pos += line_len;
let line_string = String::from_utf8_lossy(line_bytes).into_owned();
return Some((line_start, line_string));
}
// Normal case: found newline or reached EOF within initial chunk
let line_bytes = &chunk[..line_len];
self.current_pos += line_len;
let line_string = String::from_utf8_lossy(line_bytes).into_owned();
Some((line_start, line_string))
}
/// Get the previous line (moving backward)
/// Uses direct byte scanning which works even with unloaded chunks
pub fn prev(&mut self) -> Option<(usize, String)> {
if self.current_pos == 0 {
return None;
}
// current_pos is the start of the current line
// Scan backward from current_pos-1 to find the end of the previous line
if self.current_pos == 0 {
return None;
}
// Load a reasonable chunk backward for scanning
let scan_distance = self.estimated_line_length * 3;
let scan_start = self.current_pos.saturating_sub(scan_distance);
let scan_len = self.current_pos - scan_start;
// Load the data we need to scan
let chunk = match self.buffer.get_text_range_mut(scan_start, scan_len) {
Ok(data) => data,
Err(e) => {
tracing::error!(
"LineIterator::prev(): Failed to load chunk at {}: {}",
scan_start,
e
);
return None;
}
};
// Scan backward to find the last newline (end of previous line)
let mut prev_line_end = None;
for i in (0..chunk.len()).rev() {
if chunk[i] == b'\n' {
prev_line_end = Some(scan_start + i);
break;
}
}
let prev_line_end = prev_line_end?;
// Now find the start of the previous line by scanning backward from prev_line_end
let prev_line_start = if prev_line_end == 0 {
0
} else {
Self::find_line_start_backward(self.buffer, prev_line_end, scan_distance)
};
// Load the previous line content
let prev_line_len = prev_line_end - prev_line_start + 1; // +1 to include the newline
let line_bytes = match self
.buffer
.get_text_range_mut(prev_line_start, prev_line_len)
{
Ok(data) => data,
Err(e) => {
tracing::error!(
"LineIterator::prev(): Failed to load line at {}: {}",
prev_line_start,
e
);
return None;
}
};
let line_string = String::from_utf8_lossy(&line_bytes).into_owned();
self.current_pos = prev_line_start;
Some((prev_line_start, line_string))
}
/// Get the current position in the buffer (byte offset of current line start)
pub fn current_position(&self) -> usize {
self.current_pos
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_line_iterator_new_at_line_start() {
let mut buffer = TextBuffer::from_bytes(b"Hello\nWorld\nTest".to_vec());
// Test iterator at position 0 (start of line 0)
let iter = buffer.line_iterator(0, 80);
assert_eq!(iter.current_position(), 0, "Should be at start of line 0");
// Test iterator at position 6 (start of line 1, after \n)
let iter = buffer.line_iterator(6, 80);
assert_eq!(iter.current_position(), 6, "Should be at start of line 1");
// Test iterator at position 12 (start of line 2, after second \n)
let iter = buffer.line_iterator(12, 80);
assert_eq!(iter.current_position(), 12, "Should be at start of line 2");
}
#[test]
fn test_line_iterator_new_in_middle_of_line() {
let mut buffer = TextBuffer::from_bytes(b"Hello\nWorld\nTest".to_vec());
// Test iterator at position 3 (middle of "Hello")
let iter = buffer.line_iterator(3, 80);
assert_eq!(iter.current_position(), 0, "Should find start of line 0");
// Test iterator at position 9 (middle of "World")
let iter = buffer.line_iterator(9, 80);
assert_eq!(iter.current_position(), 6, "Should find start of line 1");
// Test iterator at position 14 (middle of "Test")
let iter = buffer.line_iterator(14, 80);
assert_eq!(iter.current_position(), 12, "Should find start of line 2");
}
#[test]
fn test_line_iterator_next() {
let mut buffer = TextBuffer::from_bytes(b"Hello\nWorld\nTest".to_vec());
let mut iter = buffer.line_iterator(0, 80);
// First line
let (pos, content) = iter.next().expect("Should have first line");
assert_eq!(pos, 0);
assert_eq!(content, "Hello\n");
// Second line
let (pos, content) = iter.next().expect("Should have second line");
assert_eq!(pos, 6);
assert_eq!(content, "World\n");
// Third line
let (pos, content) = iter.next().expect("Should have third line");
assert_eq!(pos, 12);
assert_eq!(content, "Test");
// No more lines
assert!(iter.next().is_none());
}
#[test]
fn test_line_iterator_from_middle_position() {
let mut buffer = TextBuffer::from_bytes(b"Hello\nWorld\nTest".to_vec());
// Start from position 9 (middle of "World")
let mut iter = buffer.line_iterator(9, 80);
assert_eq!(
iter.current_position(),
6,
"Should be at start of line containing position 9"
);
// First next() should return current line
let (pos, content) = iter.next().expect("Should have current line");
assert_eq!(pos, 6);
assert_eq!(content, "World\n");
// Second next() should return next line
let (pos, content) = iter.next().expect("Should have next line");
assert_eq!(pos, 12);
assert_eq!(content, "Test");
}
#[test]
fn test_line_iterator_offset_to_position_consistency() {
let mut buffer = TextBuffer::from_bytes(b"Hello\nWorld".to_vec());
// For each position, verify that offset_to_position returns correct values
let expected = vec![
(0, 0, 0), // H
(1, 0, 1), // e
(2, 0, 2), // l
(3, 0, 3), // l
(4, 0, 4), // o
(5, 0, 5), // \n
(6, 1, 0), // W
(7, 1, 1), // o
(8, 1, 2), // r
(9, 1, 3), // l
(10, 1, 4), // d
];
for (offset, expected_line, expected_col) in expected {
let pos = buffer
.offset_to_position(offset)
.expect(&format!("Should have position for offset {}", offset));
assert_eq!(pos.line, expected_line, "Wrong line for offset {}", offset);
assert_eq!(
pos.column, expected_col,
"Wrong column for offset {}",
offset
);
// Verify LineIterator uses this correctly
let iter = buffer.line_iterator(offset, 80);
let expected_line_start = if expected_line == 0 { 0 } else { 6 };
assert_eq!(
iter.current_position(),
expected_line_start,
"LineIterator at offset {} should be at line start {}",
offset,
expected_line_start
);
}
}
#[test]
fn test_line_iterator_prev() {
let mut buffer = TextBuffer::from_bytes(b"Line1\nLine2\nLine3".to_vec());
// Start at line 2
let mut iter = buffer.line_iterator(12, 80);
// Go back to line 1
let (pos, content) = iter.prev().expect("Should have previous line");
assert_eq!(pos, 6);
assert_eq!(content, "Line2\n");
// Go back to line 0
let (pos, content) = iter.prev().expect("Should have previous line");
assert_eq!(pos, 0);
assert_eq!(content, "Line1\n");
// No more previous lines
assert!(iter.prev().is_none());
}
#[test]
fn test_line_iterator_single_line() {
let mut buffer = TextBuffer::from_bytes(b"Only one line".to_vec());
let mut iter = buffer.line_iterator(0, 80);
let (pos, content) = iter.next().expect("Should have the line");
assert_eq!(pos, 0);
assert_eq!(content, "Only one line");
assert!(iter.next().is_none());
assert!(iter.prev().is_none());
}
#[test]
fn test_line_iterator_empty_lines() {
let mut buffer = TextBuffer::from_bytes(b"Line1\n\nLine3".to_vec());
let mut iter = buffer.line_iterator(0, 80);
let (pos, content) = iter.next().expect("First line");
assert_eq!(pos, 0);
assert_eq!(content, "Line1\n");
let (pos, content) = iter.next().expect("Empty line");
assert_eq!(pos, 6);
assert_eq!(content, "\n");
let (pos, content) = iter.next().expect("Third line");
assert_eq!(pos, 7);
assert_eq!(content, "Line3");
}
/// BUG REPRODUCTION: Line longer than estimated_line_length
/// When a line is longer than the estimated_line_length passed to line_iterator(),
/// the LineIterator::new() constructor fails to find the actual line start.
///
/// This causes Home/End key navigation to fail on long lines.
#[test]
fn test_line_iterator_long_line_exceeds_estimate() {
// Create a line that's 200 bytes long (much longer than typical estimate)
let long_line = "x".repeat(200);
let content = format!("{}\n", long_line);
let mut buffer = TextBuffer::from_bytes(content.as_bytes().to_vec());
// Use a small estimated_line_length (50 bytes) - smaller than actual line
let estimated_line_length = 50;
// Position cursor at the END of the long line (position 200, before the \n)
let cursor_at_end = 200;
// Create iterator from end of line - this should find position 0 as line start
let iter = buffer.line_iterator(cursor_at_end, estimated_line_length);
// BUG: iter.current_position() returns 150 (200 - 50) instead of 0
// because find_line_start_backward only scans back 50 bytes
assert_eq!(
iter.current_position(),
0,
"LineIterator should find actual line start (0), not estimation boundary ({})",
cursor_at_end - estimated_line_length
);
// Test with cursor in the middle too
let cursor_in_middle = 100;
let iter = buffer.line_iterator(cursor_in_middle, estimated_line_length);
assert_eq!(
iter.current_position(),
0,
"LineIterator should find line start regardless of cursor position"
);
}
/// BUG REPRODUCTION: Multiple lines where one exceeds estimate
/// Tests that line iteration works correctly even when one line is very long
#[test]
fn test_line_iterator_mixed_line_lengths() {
// Short line, very long line, short line
let long_line = "L".repeat(300);
let content = format!("Short1\n{}\nShort2\n", long_line);
let mut buffer = TextBuffer::from_bytes(content.as_bytes().to_vec());
let estimated_line_length = 50;
// Position cursor at end of long line (position 7 + 300 = 307)
let cursor_pos = 307;
let iter = buffer.line_iterator(cursor_pos, estimated_line_length);
// Should find position 7 (start of long line), not 257 (307 - 50)
assert_eq!(
iter.current_position(),
7,
"Should find start of long line at position 7, not estimation boundary"
);
}
}