From 1ac263b678098fab321d5494bad050b140ce649c Mon Sep 17 00:00:00 2001
From: emil28092005 <emil28092005@gmail.com>
Date: Wed, 16 Sep 2026 04:02:10 +0300
Subject: [PATCH] fix: verify raw source bytes and bound reads after file
 changes

---
 src/micro_scout/index.py   |  6 +++---
 src/micro_scout/scout.py   |  4 ++--
 src/micro_scout/symbols.py | 12 +++++++++++-
 tests/test_retrieval.py    | 26 ++++++++++++++++++++++++++
 4 files changed, 42 insertions(+), 6 deletions(-)

diff --git a/src/micro_scout/index.py b/src/micro_scout/index.py
index 798ace9..852bc93 100644
--- a/src/micro_scout/index.py
+++ b/src/micro_scout/index.py
@@ -12,7 +12,7 @@ from typing import TYPE_CHECKING
 
 import numpy as np
 
-from micro_scout.symbols import Symbol, build_edges, parse_source, source_paths
+from micro_scout.symbols import Symbol, build_edges, parse_source, read_source, source_paths
 from micro_scout.text import digest
 
 if TYPE_CHECKING:
@@ -68,8 +68,8 @@ def build_index(
     for path in source_paths(root):
         relative = path.relative_to(root).as_posix()
         try:
-            text = path.read_text(encoding="utf-8")
-        except (UnicodeError, OSError) as exc:
+            text = read_source(path)
+        except (ValueError, OSError) as exc:
             warnings.append({"path": relative, "reason": type(exc).__name__})
             continue
         files[relative] = digest(text)
diff --git a/src/micro_scout/scout.py b/src/micro_scout/scout.py
index 445b271..3cd4a5c 100644
--- a/src/micro_scout/scout.py
+++ b/src/micro_scout/scout.py
@@ -13,7 +13,7 @@ import numpy as np
 
 from micro_scout.index import Index
 from micro_scout.lexical import BM25, reciprocal_rank_fusion, top_indices
-from micro_scout.symbols import EXTENSIONS, Symbol
+from micro_scout.symbols import EXTENSIONS, Symbol, read_source
 from micro_scout.text import digest
 
 if TYPE_CHECKING:
@@ -55,7 +55,7 @@ class Scout:
                     )
                 ):
                     raise ValueError("Source path escapes repository or uses a symlink")
-                text = path.read_text(encoding="utf-8")
+                text = read_source(path)
             except (OSError, UnicodeError) as exc:
                 raise StaleReferenceError(
                     f"Source unavailable: {symbol.path}; rebuild index"
diff --git a/src/micro_scout/symbols.py b/src/micro_scout/symbols.py
index 693c96c..e2e90c8 100644
--- a/src/micro_scout/symbols.py
+++ b/src/micro_scout/symbols.py
@@ -44,6 +44,16 @@ EXCLUDED = {
     ".mypy_cache",
     ".pytest_cache",
 }
+MAX_FILE_BYTES = 1_000_000
+
+
+def read_source(path: Path, max_bytes: int = MAX_FILE_BYTES) -> str:
+    """Read bounded UTF-8 without newline translation, so SHA-256 matches file bytes."""
+    with path.open("rb") as stream:
+        raw = stream.read(max_bytes + 1)
+    if len(raw) > max_bytes:
+        raise ValueError("Source exceeds the file-size limit")
+    return raw.decode("utf-8")
 
 
 @dataclass(frozen=True)
@@ -71,7 +81,7 @@ class Symbol:
         return asdict(self)
 
 
-def source_paths(root: Path, max_file_bytes: int = 1_000_000) -> list[Path]:
+def source_paths(root: Path, max_file_bytes: int = MAX_FILE_BYTES) -> list[Path]:
     """Honor gitignore when available; never traverse symlinks or hidden trees."""
     root = root.resolve(strict=True)
     if not root.is_dir():
diff --git a/tests/test_retrieval.py b/tests/test_retrieval.py
index 3f6a683..42575a3 100644
--- a/tests/test_retrieval.py
+++ b/tests/test_retrieval.py
@@ -237,3 +237,29 @@ def test_repository_cluster_bootstrap_retains_group_correlation():
     assert result["delta"] == pytest.approx(0.275)
     assert result["ci95"] == pytest.approx([-0.4, 0.5])
     assert result["repository_clusters"] == 2
+
+
+def test_file_hash_matches_raw_crlf_bytes_and_detects_line_ending_changes(repository, tmp_path):
+    import hashlib
+
+    raw = b"def crlf():\r\n    return 42\r\n"
+    source = repository / "crlf.py"
+    source.write_bytes(raw)
+    path = tmp_path / "index.sqlite"
+    build_index(repository, path)
+    scout = Scout(Index(path))
+    symbol = next(s for s in scout.index.symbols if s.name == "crlf")
+    assert scout.read(symbol.id)["file_sha256"] == hashlib.sha256(raw).hexdigest()
+    source.write_bytes(raw.replace(b"\r\n", b"\n"))
+    with pytest.raises(StaleReferenceError):
+        scout.read(symbol.id)
+
+
+def test_source_that_grows_after_indexing_is_bounded(repository, tmp_path):
+    path = tmp_path / "index.sqlite"
+    build_index(repository, path)
+    scout = Scout(Index(path))
+    symbol = next(s for s in scout.index.symbols if s.name == "add_numbers")
+    (repository / "numbers.py").write_text("x" * 1_000_001)
+    with pytest.raises(ValueError, match="file-size limit"):
+        scout.read(symbol.id)