From fe5642e0f29708b223e989a055edb2254eccfa2b Mon Sep 17 00:00:00 2001 From: emil Date: Sat, 16 May 2026 14:52:43 +0300 Subject: [PATCH] =?UTF-8?q?v0.2:=20full=20repurpose=20=E2=80=94=20codebase?= =?UTF-8?q?=20exploration=20with=20per-directory=20cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BREAKING. memwalk is no longer a personal-activity recorder; it's an AI tool for asking questions about any codebase, with cached SSM state per directory. What's gone ----------- - sources/git.py, sources/bash.py — personal activity collectors - snapshot.py — daily snapshot rotation - ingest.py — orchestration tied to git+bash use case - standup / update CLI commands - All v0.1 config keys (scan_paths, bash settings, bootstrap_days) What's new ---------- - corpus.py — walk a codebase, filter source files, build a single ingest-ready text block with a stable manifest hash for cache invalidation. - cache.py — per-directory cached state + sidecar metadata JSON. Cache key = sha256(abs_path)[:16]; freshness check = manifest hash over (rel_path, size, mtime_ns). - engine.py — shared digest/ask orchestration used by both CLI and MCP server. - cli.py — init, digest, ask, list, drop, status, mcp. - mcp_server.py — tools: digest, ask, list_caches, drop_cache, status. - config.py — drastically simplified (just model+inference defaults). The MCP server still ships as `memwalk mcp` and works the same way with Claude Code / opencode. Validated on memwalk's own source: digest in ~4s, ask answers in ~3s (model+state load) including "list CLI commands", "where is cache stored, what filename pattern", "how does cache invalidation work" — all accurate down to specific details (sha256 length, file extensions, metadata field semantics). memba dep installed via git URL until both packages reach PyPI. Co-Authored-By: Claude Opus 4.7 --- README.md | 151 +++++++++++++---------- memwalk/__init__.py | 4 +- memwalk/cache.py | 155 +++++++++++++++++++++++ memwalk/cli.py | 239 +++++++++++++++++++----------------- memwalk/config.py | 120 +++--------------- memwalk/corpus.py | 124 +++++++++++++++++++ memwalk/engine.py | 151 +++++++++++++++++++++++ memwalk/ingest.py | 118 ------------------ memwalk/mcp_server.py | 228 +++++++++++++++++++--------------- memwalk/snapshot.py | 36 ------ memwalk/sources/__init__.py | 13 -- memwalk/sources/bash.py | 153 ----------------------- memwalk/sources/git.py | 92 -------------- pyproject.toml | 11 +- 14 files changed, 789 insertions(+), 806 deletions(-) create mode 100644 memwalk/cache.py create mode 100644 memwalk/corpus.py create mode 100644 memwalk/engine.py delete mode 100644 memwalk/ingest.py delete mode 100644 memwalk/snapshot.py delete mode 100644 memwalk/sources/__init__.py delete mode 100644 memwalk/sources/bash.py delete mode 100644 memwalk/sources/git.py diff --git a/README.md b/README.md index 237aa55..3f5aad5 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,52 @@ # memwalk -> Walk through your work memory. +> Ask AI about any codebase — local, cached, SSM-state-backed. -A local-first CLI that watches your git activity (and optionally your shell -history) and feeds it into a Mamba-based LLM via persistent state. You can -then ask in plain English what you were doing last week, why you started that -branch, or generate a standup from yesterday's commits — without your data -ever leaving the machine. +`memwalk` reads an entire codebase into a Mamba-based LLM via persistent +state, so subsequent questions answer in <1 s without re-reading anything. +The state is byte-portable (via [memba](https://github.com/emil28092005/Memba)) +and cached per-directory by file manifest hash, so re-asking is free until +the source changes. -Built on **[memba](https://github.com/emil28092005/Memba)** for state -persistence and **[NVIDIA Nemotron-3-Nano-4B](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-GGUF)** -(or any other GGUF SSM/hybrid model) for inference. +Built on **memba** for state persistence and +**[NVIDIA Nemotron-3-Nano-4B](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-GGUF)** +(hybrid Mamba-2 + Transformer, 1M training context) for inference. + +## What makes this different from Cursor / Cody / Aider + +| Tool | Approach | Whole-repo question | +|--------------|--------------------------------|--------------------------| +| Cursor | Embed chunks, retrieve at Q | Fragmented context | +| Cody | BM25 + dense embeddings (RAG) | Pre-indexed, retrieved | +| Aider | Symbol-level repo map | Signatures only | +| **memwalk** | **Read everything once, cache the SSM state** | Holistic answer; <1s re-asks | + +SSM state is **fixed-size** (Mamba's defining property), so even a 1M-token +codebase compresses into a constant-size file (~85 MB at our settings). +Reload is millisecond-scale — re-asking a freshly-cached repo costs no +model inference until you ask the next question. ## Status -v0.1 — alpha. Works end-to-end on Linux for the maintainer; APIs and on-disk -format may change. +v0.2 — alpha. Works for the maintainer end-to-end; APIs and on-disk format +may shift. ## Install +memba is not on PyPI yet, so install via git: + ```bash -# Until memba is on PyPI, install both editable from local clones: -pip install -e /path/to/Memba -pip install -e /path/to/memwalk +pip install git+https://github.com/emil28092005/memwalk.git +# (pulls memba @ main as a transitive git dep) ``` -Make sure you have a GGUF Mamba-2 or hybrid model. Recommended: +Or from a local clone: + +```bash +pip install -e ~/Desktop/Coding/memwalk +``` + +Make sure you have a GGUF Mamba-2 / hybrid model. Recommended: ```bash hf download nvidia/NVIDIA-Nemotron-3-Nano-4B-GGUF \ @@ -36,63 +57,35 @@ hf download nvidia/NVIDIA-Nemotron-3-Nano-4B-GGUF \ ## Quickstart ```bash -memwalk init # interactive setup -memwalk update # ingest last 30 days of git+bash -memwalk standup # auto-generate daily standup -memwalk ask "What was I working on last week?" -memwalk status +memwalk init # one-time: set model path +memwalk digest ~/Desktop/Coding/myrepo # first time: read everything (~10s) +memwalk ask ~/Desktop/Coding/myrepo "How does auth work?" # <1s +memwalk ask ~/Desktop/Coding/myrepo "Which file owns the migration logic?" +memwalk list # show all cached codebases +memwalk drop ~/Desktop/Coding/myrepo # invalidate cache +memwalk status # config + cache summary ``` -## What it actually does +`memwalk ask` auto-digests on first use, so the explicit `digest` step is +optional. The cache is invalidated automatically when any source file +changes (mtime / size). -`memwalk update` walks your configured git repos and (optionally) your bash -history, formats new events into a readable activity block, and feeds that -into the SSM model. The model's hidden state — a fixed ~85 MB blob — is -saved to `~/.memwalk/current.memb` via memba. - -`memwalk ask` and `memwalk standup` load that state and query it. The model -recalls themes, projects, and trajectory across processes and reboots. - -## Use from an agent (MCP) - -memwalk ships an MCP server so Claude Code / opencode / any MCP-aware -agent can query your memory as native tools. +## Use from an AI agent (MCP) ```bash memwalk mcp # starts a stdio MCP server ``` -Tools exposed: `ask(question)`, `standup()`, `status()`, `update()`. -The Session loads lazily on the first call that needs it, then stays in -memory — first call ~2 s, subsequent calls <500 ms. +Tools: `digest(path)`, `ask(path, question)`, `list_caches()`, +`drop_cache(path)`, `status()`. -### Configure Claude Code - -Easiest way (Claude Code CLI): +### Claude Code ```bash claude mcp add memwalk -- memwalk mcp ``` -Or by hand, in `~/.claude/mcp_servers.json` (path may vary by version): - -```json -{ - "mcpServers": { - "memwalk": { - "command": "memwalk", - "args": ["mcp"] - } - } -} -``` - -Restart Claude Code. Tools appear as `mcp__memwalk__ask`, -`mcp__memwalk__standup`, etc. - -### Configure opencode - -opencode uses its own MCP block in `opencode.json`: +Or by hand in your MCP config: ```json { @@ -102,16 +95,40 @@ opencode uses its own MCP block in `opencode.json`: } ``` -## Layout +### opencode / Hermes / other MCP clients -``` -~/.memwalk/ -├── config.toml -├── current.memb ← rolling state -├── last_update.txt -└── snapshots/ - └── 2026-05-16.memb ← daily snapshot before each update -``` +Same shape — they all consume `{"command": "memwalk", "args": ["mcp"]}`. + +After the agent connects it sees `mcp__memwalk__digest`, +`mcp__memwalk__ask`, etc. Typical flow: + +> User: *"What changed in the migrations folder of my CU\_Points repo this month?"* +> +> Agent: calls `mcp__memwalk__ask(path="~/Desktop/Coding/AI/CU_Points", +> question="...")`. memwalk auto-digests if needed, returns answer. + +## What does it actually do well? + +Validated on memba's own codebase (13 files, ~63 K chars): + +- Listed every header field of the state file format **in order** +- Explained the architectural reason for the `eval+sample` rewrite +- Identified which side of the C/Python boundary writes the MEMB trailer +- Listed all CLI subcommands accurately +- Suggested correct file path + approach for adding a new command + +Recall is **descriptive-strong** — facts that are in the source. It is not +a substitute for a real debugger or a code generator. For complex +reasoning over small snippets, a bigger code-tuned model is still better. + +## Limits + +- **Single-shot context, not chunked retrieval.** Whole corpus must fit in + `n_ctx` (default 32 K tokens ≈ ~120 K chars). Bigger repos: bump `n_ctx`, + use a beefier GPU, or filter `INCLUDE_SUFFIXES` in `corpus.py`. +- **No code-aware filtering yet** — every text file under the root is + read. Use `.gitignore`-style filtering in v0.3. +- **No GPU-less mode tested** — should work on CPU but slow. ## License diff --git a/memwalk/__init__.py b/memwalk/__init__.py index b545bb2..a80898a 100644 --- a/memwalk/__init__.py +++ b/memwalk/__init__.py @@ -1,3 +1,3 @@ -"""memwalk — semantic recall of your work via local SSM models.""" +"""memwalk — ask AI about any codebase via cached SSM state.""" -__version__ = "0.1.0" +__version__ = "0.2.0" diff --git a/memwalk/cache.py b/memwalk/cache.py new file mode 100644 index 0000000..3960ce5 --- /dev/null +++ b/memwalk/cache.py @@ -0,0 +1,155 @@ +"""Per-directory cached SSM state. + +Each cached codebase has two sidecar files under CACHE_DIR: + + cache/.memb — the memba state file (binary) + cache/.json — metadata (source path, manifest hash, stats…) + +`` is derived from the absolute source path so the same directory +always maps to the same files. The manifest hash inside the metadata +detects whether the source has changed since the cache was built. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from dataclasses import dataclass, asdict +from datetime import datetime +from pathlib import Path + +CACHE_DIR = Path.home() / ".memwalk/cache" + + +# ── Metadata model ─────────────────────────────────────────────── + +@dataclass(slots=True) +class CacheMeta: + key: str + source_path: str # absolute path to the source dir + manifest_hash: str # current files' fingerprint at digest time + n_files: int + n_chars: int + n_ctx: int + model_path: str + created_iso: str + last_used_iso: str + + @property + def state_path(self) -> Path: + return CACHE_DIR / f"{self.key}.memb" + + @property + def meta_path(self) -> Path: + return CACHE_DIR / f"{self.key}.json" + + def touch(self) -> None: + self.last_used_iso = datetime.now().isoformat(timespec="seconds") + self.save() + + def save(self) -> None: + CACHE_DIR.mkdir(parents=True, exist_ok=True) + self.meta_path.write_text(json.dumps(asdict(self), indent=2)) + + +# ── Key derivation ─────────────────────────────────────────────── + +def cache_key(source_path: Path) -> str: + """Short, path-stable cache key (16 hex chars). Same dir → same key.""" + abs_path = str(source_path.resolve()) + return hashlib.sha256(abs_path.encode("utf-8")).hexdigest()[:16] + + +# ── Read ───────────────────────────────────────────────────────── + +def load_meta(source_path: Path) -> CacheMeta | None: + """Return the cache meta for @source_path, or None if not cached.""" + key = cache_key(source_path) + meta_file = CACHE_DIR / f"{key}.json" + if not meta_file.exists(): + return None + try: + data = json.loads(meta_file.read_text()) + return CacheMeta(**data) + except (json.JSONDecodeError, TypeError): + return None + + +def is_fresh(meta: CacheMeta, current_manifest: str) -> bool: + """True iff cache file is intact and manifest hash unchanged.""" + return (meta.state_path.exists() + and meta.manifest_hash == current_manifest) + + +def list_all() -> list[CacheMeta]: + """Return every cached entry on disk, newest-used first.""" + if not CACHE_DIR.exists(): + return [] + entries: list[CacheMeta] = [] + for meta_file in CACHE_DIR.glob("*.json"): + try: + data = json.loads(meta_file.read_text()) + entries.append(CacheMeta(**data)) + except (json.JSONDecodeError, TypeError): + continue + entries.sort(key=lambda m: m.last_used_iso, reverse=True) + return entries + + +# ── Write / delete ─────────────────────────────────────────────── + +def write_meta( + source_path: Path, + *, + manifest_hash: str, + n_files: int, + n_chars: int, + n_ctx: int, + model_path: str, +) -> CacheMeta: + key = cache_key(source_path) + now = datetime.now().isoformat(timespec="seconds") + meta = CacheMeta( + key=key, + source_path=str(source_path.resolve()), + manifest_hash=manifest_hash, + n_files=n_files, + n_chars=n_chars, + n_ctx=n_ctx, + model_path=str(model_path), + created_iso=now, + last_used_iso=now, + ) + meta.save() + return meta + + +def drop(source_path: Path) -> bool: + """Remove cache for @source_path. Returns True if anything was deleted.""" + key = cache_key(source_path) + deleted = False + for ext in (".memb", ".json"): + p = CACHE_DIR / f"{key}{ext}" + if p.exists(): + p.unlink() + deleted = True + return deleted + + +# ── memba session naming convention ────────────────────────────── +# +# We want memba's `Session` to write to / read from `cache/.memb`. +# memba builds its filename as `/.memb`, so: +# +# Session(session_id=key, state_dir=CACHE_DIR, ...) +# +# already gives us the right path. Helpers below just compute key. + +def session_id_for(source_path: Path) -> str: + return cache_key(source_path) + + +def state_dir() -> Path: + CACHE_DIR.mkdir(parents=True, exist_ok=True) + return CACHE_DIR diff --git a/memwalk/cli.py b/memwalk/cli.py index 9383ea1..fe18dbf 100644 --- a/memwalk/cli.py +++ b/memwalk/cli.py @@ -1,8 +1,8 @@ -"""memwalk CLI — typer entry point.""" +"""memwalk CLI v0.2 — codebase exploration via cached SSM state.""" from __future__ import annotations -from datetime import datetime, timedelta +from datetime import datetime from pathlib import Path import typer @@ -10,18 +10,19 @@ from rich.console import Console from rich.prompt import Prompt from rich.table import Table -from . import __version__ +from . import __version__, cache from .config import ( - DEFAULT_MODEL_HINT, HOME_DIR, BashConfig, Config, GitConfig, - load_config, read_last_update, write_config, + CONFIG_PATH, DEFAULT_MODEL_HINT, Config, + load_config, write_config, ) -from .ingest import open_session, query, update -from .snapshot import prune_old -from .sources import bash as bash_src -from .sources import git as git_src +from .engine import ask as engine_ask +from .engine import digest as engine_digest -cli = typer.Typer(name="memwalk", help="Walk through your work memory.", - add_completion=False) +cli = typer.Typer( + name="memwalk", + help="Ask AI about any codebase — local, cached, SSM-state-backed.", + add_completion=False, +) console = Console() @@ -29,172 +30,178 @@ console = Console() @cli.command() def init( - model: str = typer.Option(None, "--model", help="Path to GGUF model"), - scan_path: list[str] = typer.Option(None, "--scan", - help="Directory to scan for git repos (repeatable)"), - no_bash: bool = typer.Option(False, "--no-bash", help="Disable bash history"), - force: bool = typer.Option(False, "--force", "-f", - help="Overwrite existing config"), + model: str = typer.Option(None, "--model", help="Path to GGUF model"), + n_ctx: int = typer.Option(32768, "--n-ctx", help="Inference context window"), + gpu_layers: int = typer.Option(-1, "--gpu-layers", "-g"), + force: bool = typer.Option(False, "--force", "-f"), ) -> None: - """Interactive (or flag-driven) setup. Writes ~/.memwalk/config.toml.""" - config_path = HOME_DIR / "config.toml" - if config_path.exists() and not force: - console.print(f"[yellow]Config already exists at {config_path}[/yellow]") - console.print("Use --force to overwrite, or edit the file by hand.") + """One-time setup. Writes ~/.memwalk/config.toml.""" + if CONFIG_PATH.exists() and not force: + console.print(f"[yellow]Config already exists at {CONFIG_PATH}[/yellow]") + console.print("Use --force to overwrite, or edit by hand.") raise typer.Exit(1) console.print(f"[bold cyan]memwalk init v{__version__}[/bold cyan]\n") - - # Model if model is None: console.print(DEFAULT_MODEL_HINT + "\n") model = Prompt.ask("Path to GGUF model") - model_p = Path(model).expanduser() - if not model_p.exists(): - console.print(f"[yellow]warning: {model_p} doesn't exist yet[/yellow]") - - # Scan paths - if scan_path: - scan_paths = [Path(p).expanduser() for p in scan_path] - else: - default = str(Path.home() / "Desktop/Coding") - raw = Prompt.ask( - "Directories to scan for git repos (comma-separated)", - default=default, - ) - scan_paths = [Path(p.strip()).expanduser() for p in raw.split(",") if p.strip()] cfg = Config( - model_path=model_p, - git=GitConfig(scan_paths=scan_paths), - bash=BashConfig(enabled=not no_bash), + model_path=Path(model).expanduser(), + n_gpu_layers=gpu_layers, + n_ctx=n_ctx, ) write_config(cfg) - - console.print(f"\n[green]✓[/green] Wrote {cfg.config_path}") - console.print(f"[green]✓[/green] State dir: {cfg.state_dir}") - console.print(f"\nNext: [bold]memwalk update[/bold] to ingest the last 30 days") + console.print(f"\n[green]✓[/green] {CONFIG_PATH}") + console.print( + f"\nNext: [bold]memwalk digest /path/to/repo[/bold] to ingest a codebase,\n" + f"then [bold]memwalk ask /path/to/repo \"...\"[/bold] to query." + ) -# ── update ──────────────────────────────────────────────────────── +# ── digest ──────────────────────────────────────────────────────── -@cli.command(name="update") -def update_cmd( +@cli.command() +def digest( + path: str = typer.Argument(..., help="Codebase root to ingest"), + n_ctx: int = typer.Option(None, "--n-ctx", + help="Override config n_ctx for this digest"), + force: bool = typer.Option(False, "--force", "-f", + help="Re-ingest even if a fresh cache exists"), verbose: bool = typer.Option(False, "--verbose", "-v"), ) -> None: - """Ingest new git+bash events since last update.""" + """Read all source files under PATH, build cached SSM state.""" cfg = load_config() - with console.status("Ingesting…"): - result = update(cfg, verbose=verbose) - - if result["ingested"] == 0: - console.print(f"[dim]No new activity since {result['since'].strftime('%Y-%m-%d %H:%M')}[/dim]") + source = Path(path).expanduser().resolve() + with console.status(f"Digesting {source}…"): + result = engine_digest(cfg, source, n_ctx=n_ctx, force=force, + verbose=verbose) + m = result.meta + if result.elapsed_s == 0.0: + console.print(f"[dim]Cache hit — already fresh ({m.n_files} files, " + f"{m.n_chars:,} chars).[/dim]") return - console.print( - f"[green]✓[/green] Ingested {result['ingested']} events " - f"([cyan]{result['git']}[/cyan] commits + " - f"[cyan]{result['bash']}[/cyan] shell sessions) " - f"in {result['elapsed_s']:.1f}s" + f"[green]✓[/green] Digested {m.n_files} files, {m.n_chars:,} chars " + f"in {result.elapsed_s:.1f}s ({result.char_rate:,.0f} char/s)" ) - console.print(f" Window: {result['since'].strftime('%Y-%m-%d %H:%M')} → " - f"{result['until'].strftime('%Y-%m-%d %H:%M')}") - console.print(f" State : {result['state_size']:,} bytes" - + (" (daily snapshot taken)" if result["snapshotted"] else "")) + console.print(f" cache : [dim]{m.state_path}[/dim]") + ack = result.ack + console.print(f" model : {ack[:140]}{'…' if len(ack) > 140 else ''}") # ── ask ─────────────────────────────────────────────────────────── @cli.command() def ask( - question: str = typer.Argument(..., help="What to ask the model"), + path: str = typer.Argument(..., help="Codebase root (digest first or auto)"), + question: str = typer.Argument(..., help="Natural-language question"), max_tokens: int = typer.Option(400, "--max-tokens"), + no_auto_digest: bool = typer.Option(False, "--no-auto-digest", + help="Fail instead of digesting if cache missing"), verbose: bool = typer.Option(False, "--verbose", "-v"), ) -> None: - """Query the current state.""" + """Query the cached codebase. Auto-digests if no cache exists.""" cfg = load_config() + source = Path(path).expanduser().resolve() with console.status("Thinking…"): - answer = query(cfg, question, max_tokens=max_tokens, verbose=verbose) + answer, meta, just_digested = engine_ask( + cfg, source, question, + max_tokens=max_tokens, + auto_digest=not no_auto_digest, + verbose=verbose, + ) + if just_digested: + console.print(f"[dim](digested {meta.n_files} files / " + f"{meta.n_chars:,} chars on demand)[/dim]\n") console.print(answer) -# ── standup ─────────────────────────────────────────────────────── +# ── list ────────────────────────────────────────────────────────── + +@cli.command("list") +def list_caches() -> None: + """Show all cached codebases.""" + entries = cache.list_all() + if not entries: + console.print(f"[dim]No cached codebases yet. Try `memwalk digest `.[/dim]") + return + table = Table(title="Cached codebases", show_lines=False) + table.add_column("Source", style="cyan", overflow="fold") + table.add_column("Files", justify="right") + table.add_column("Chars", justify="right") + table.add_column("n_ctx", justify="right") + table.add_column("Last used") + for m in entries: + try: + ts = datetime.fromisoformat(m.last_used_iso).strftime("%Y-%m-%d %H:%M") + except ValueError: + ts = m.last_used_iso + table.add_row( + m.source_path, + f"{m.n_files}", + f"{m.n_chars:,}", + f"{m.n_ctx:,}", + ts, + ) + console.print(table) + + +# ── drop ────────────────────────────────────────────────────────── @cli.command() -def standup( - verbose: bool = typer.Option(False, "--verbose", "-v"), +def drop( + path: str = typer.Argument(..., help="Source dir whose cache to invalidate"), + yes: bool = typer.Option(False, "--yes", "-y"), ) -> None: - """Generate a brief 'what I did + what's next' summary from recent activity.""" - cfg = load_config() - q = ( - "Generate my daily standup notes. Cover: what I worked on yesterday " - "(grouped by project), what I plan today based on the trajectory, and " - "any blockers visible in the activity. Be concise — bullet points, " - "no preamble." - ) - with console.status("Thinking…"): - answer = query(cfg, q, max_tokens=500, verbose=verbose) - console.print("[bold cyan]Standup:[/bold cyan]\n") - console.print(answer) + """Invalidate cache for a codebase.""" + source = Path(path).expanduser().resolve() + meta = cache.load_meta(source) + if meta is None: + console.print(f"[dim]No cache for {source}[/dim]") + return + if not yes and not typer.confirm( + f"Drop cache for {meta.source_path} ({meta.n_files} files, " + f"{meta.n_chars:,} chars)?" + ): + raise typer.Abort() + deleted = cache.drop(source) + console.print(f"[dim]{'Dropped' if deleted else 'Nothing to drop'}: {source}[/dim]") # ── status ──────────────────────────────────────────────────────── @cli.command() def status() -> None: - """Show config and state info.""" + """Show config and cache summary.""" try: cfg = load_config() except FileNotFoundError as e: console.print(f"[red]{e}[/red]") raise typer.Exit(1) - last = read_last_update(cfg) + entries = cache.list_all() table = Table(show_header=False, box=None) - table.add_row("[bold]config[/bold]", str(cfg.config_path)) - table.add_row("[bold]model[/bold]", str(cfg.model_path)) - table.add_row("[bold]state[/bold]", - f"{cfg.state_path} ({cfg.state_path.stat().st_size:,} B)" - if cfg.state_path.exists() else f"{cfg.state_path} (none)") - table.add_row("[bold]scan paths[/bold]", ", ".join(str(p) for p in cfg.git.scan_paths)) - table.add_row("[bold]bash[/bold]", "on" if cfg.bash.enabled else "off") - table.add_row("[bold]last update[/bold]", - last.strftime("%Y-%m-%d %H:%M") if last else "never") - if cfg.snapshots_dir.exists(): - snaps = sorted(cfg.snapshots_dir.glob("*.memb")) - table.add_row("[bold]snapshots[/bold]", - f"{len(snaps)} " + (f"(latest {snaps[-1].stem})" if snaps else "")) + table.add_row("[bold]config[/bold]", str(CONFIG_PATH)) + table.add_row("[bold]model[/bold]", str(cfg.model_path)) + table.add_row("[bold]n_ctx[/bold]", f"{cfg.n_ctx:,}") + table.add_row("[bold]gpu_layers[/bold]", str(cfg.n_gpu_layers)) + table.add_row("[bold]caches[/bold]", f"{len(entries)} codebase(s)") console.print(table) + if entries: + console.print() + list_caches() # ── mcp ─────────────────────────────────────────────────────────── @cli.command() def mcp() -> None: - """Run as an MCP server (stdio) for Claude Code / opencode / Hermes / etc. - - The Session is loaded lazily on the first tool call that needs it, then - reused — so subsequent queries are fast. Configure your agent to launch - this command; for Claude Code add to ~/.claude/mcp_servers.json: - - {"mcpServers": {"memwalk": {"command": "memwalk", "args": ["mcp"]}}} - """ + """Run as an MCP server for Claude Code / opencode / Hermes / etc.""" from .mcp_server import main as mcp_main mcp_main() -# ── prune ───────────────────────────────────────────────────────── - -@cli.command() -def prune( - keep_days: int = typer.Option(90, "--keep-days", help="Snapshots older than this are deleted"), -) -> None: - """Delete old daily snapshots.""" - cfg = load_config() - n = prune_old(cfg, keep_days=keep_days) - console.print(f"[dim]Deleted {n} snapshots older than {keep_days} days[/dim]") - - def main() -> None: cli() diff --git a/memwalk/config.py b/memwalk/config.py index ddd800c..5f9a3bf 100644 --- a/memwalk/config.py +++ b/memwalk/config.py @@ -1,20 +1,13 @@ -"""Config & state-tracking for memwalk. +"""Minimal memwalk config — just model + inference defaults. -Layout: - ~/.memwalk/ - config.toml # user settings - last_update.txt # ISO timestamp of last successful update - current.memb # rolling state file - snapshots/ - 2026-05-16.memb - ... +v0.2 dropped scan_paths / bash settings; codebase paths are passed +per-command instead, so config has no per-corpus knobs. """ from __future__ import annotations import sys -from dataclasses import dataclass, field -from datetime import datetime +from dataclasses import dataclass from pathlib import Path if sys.version_info >= (3, 11): @@ -24,123 +17,44 @@ else: HOME_DIR = Path.home() / ".memwalk" +CONFIG_PATH = HOME_DIR / "config.toml" DEFAULT_MODEL_HINT = ( - "Recommended: NVIDIA Nemotron-3-Nano-4B (hybrid Mamba+Transformer).\n" - "Download with: hf download nvidia/NVIDIA-Nemotron-3-Nano-4B-GGUF " - "NVIDIA-Nemotron3-Nano-4B-Q4_K_M.gguf --local-dir ~/.memwalk/models" + "Recommended: NVIDIA Nemotron-3-Nano-4B-GGUF (hybrid Mamba-Transformer, " + "1M-token training context).\n" + "Download with: hf download nvidia/NVIDIA-Nemotron-3-Nano-4B-GGUF \\\n" + " NVIDIA-Nemotron3-Nano-4B-Q4_K_M.gguf \\\n" + " --local-dir ~/.memwalk/models" ) -@dataclass -class GitConfig: - scan_paths: list[Path] = field(default_factory=list) - bootstrap_days: int = 30 - - -@dataclass -class BashConfig: - enabled: bool = True - history_file: Path = Path.home() / ".bash_history" - session_gap_min: int = 30 - - @dataclass class Config: model_path: Path n_gpu_layers: int = -1 n_ctx: int = 32768 - state_dir: Path = HOME_DIR - git: GitConfig = field(default_factory=GitConfig) - bash: BashConfig = field(default_factory=BashConfig) - - @property - def config_path(self) -> Path: - return HOME_DIR / "config.toml" - - @property - def state_path(self) -> Path: - return self.state_dir / "current.memb" - - @property - def snapshots_dir(self) -> Path: - return self.state_dir / "snapshots" - - @property - def last_update_path(self) -> Path: - return self.state_dir / "last_update.txt" def load_config() -> Config: - """Load config.toml from ~/.memwalk/ or raise FileNotFoundError.""" - path = HOME_DIR / "config.toml" - if not path.exists(): + if not CONFIG_PATH.exists(): raise FileNotFoundError( - f"No config at {path}. Run `memwalk init` first." + f"No config at {CONFIG_PATH}. Run `memwalk init` first." ) - - with path.open("rb") as f: + with CONFIG_PATH.open("rb") as f: data = tomllib.load(f) - - model_path = Path(data["model_path"]).expanduser() - git_cfg = data.get("git", {}) - bash_cfg = data.get("bash", {}) - return Config( - model_path=model_path, + model_path=Path(data["model_path"]).expanduser(), n_gpu_layers=int(data.get("n_gpu_layers", -1)), - n_ctx=int(data.get("n_ctx", 8192)), - state_dir=Path(data.get("state_dir", HOME_DIR)).expanduser(), - git=GitConfig( - scan_paths=[Path(p).expanduser() for p in git_cfg.get("scan_paths", [])], - bootstrap_days=int(git_cfg.get("bootstrap_days", 30)), - ), - bash=BashConfig( - enabled=bool(bash_cfg.get("enabled", True)), - history_file=Path(bash_cfg.get("history_file", - Path.home() / ".bash_history")).expanduser(), - session_gap_min=int(bash_cfg.get("session_gap_min", 30)), - ), + n_ctx=int(data.get("n_ctx", 32768)), ) def write_config(cfg: Config) -> None: - """Persist config to ~/.memwalk/config.toml using a hand-rolled writer - (Python stdlib has no TOML writer until 3.13+).""" HOME_DIR.mkdir(parents=True, exist_ok=True) - cfg.state_dir.mkdir(parents=True, exist_ok=True) - cfg.snapshots_dir.mkdir(parents=True, exist_ok=True) - lines = [ - "# memwalk config — edit by hand or rerun `memwalk init`", + "# memwalk config — edit by hand or rerun `memwalk init --force`", f'model_path = "{cfg.model_path}"', f"n_gpu_layers = {cfg.n_gpu_layers}", f"n_ctx = {cfg.n_ctx}", - f'state_dir = "{cfg.state_dir}"', - "", - "[git]", - "scan_paths = [", - *[f' "{p}",' for p in cfg.git.scan_paths], - "]", - f"bootstrap_days = {cfg.git.bootstrap_days}", - "", - "[bash]", - f"enabled = {str(cfg.bash.enabled).lower()}", - f'history_file = "{cfg.bash.history_file}"', - f"session_gap_min = {cfg.bash.session_gap_min}", ] - cfg.config_path.write_text("\n".join(lines) + "\n") - - -def read_last_update(cfg: Config) -> datetime | None: - path = cfg.last_update_path - if not path.exists(): - return None - try: - return datetime.fromisoformat(path.read_text().strip()) - except ValueError: - return None - - -def write_last_update(cfg: Config, ts: datetime) -> None: - cfg.last_update_path.write_text(ts.isoformat()) + CONFIG_PATH.write_text("\n".join(lines) + "\n") diff --git a/memwalk/corpus.py b/memwalk/corpus.py new file mode 100644 index 0000000..15a23bf --- /dev/null +++ b/memwalk/corpus.py @@ -0,0 +1,124 @@ +"""Codebase walker — produces a single ingest-ready text block + a stable +manifest hash for cache invalidation.""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from pathlib import Path + +# Source-ish file extensions we read by default. Override with --extensions. +DEFAULT_INCLUDE_SUFFIXES: frozenset[str] = frozenset({ + ".py", ".pyi", + ".c", ".h", ".cpp", ".hpp", ".cc", ".cxx", + ".rs", ".go", ".java", ".kt", ".scala", ".swift", + ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".vue", ".svelte", + ".rb", ".php", ".cs", ".fs", ".ex", ".exs", ".erl", ".clj", ".cljs", + ".sh", ".bash", ".zsh", ".fish", ".ps1", + ".toml", ".yaml", ".yml", ".json", ".xml", ".ini", ".cfg", ".conf", + ".md", ".rst", ".txt", + ".sql", ".graphql", ".proto", + ".dockerfile", ".tf", ".hcl", +}) + +# Directories we never descend into. +DEFAULT_EXCLUDE_DIRS: frozenset[str] = frozenset({ + ".git", ".hg", ".svn", + "__pycache__", "node_modules", "vendor", "third_party", + ".venv", "venv", "env", ".env", + "build", "dist", "target", "out", "bin", "obj", + ".next", ".nuxt", ".cache", + ".pytest_cache", ".mypy_cache", ".ruff_cache", ".tox", + "coverage", ".coverage", "htmlcov", + ".idea", ".vscode", + "llama.cpp", # common vendored ML dep — too big +}) + +# Glob patterns for files we always skip. +DEFAULT_EXCLUDE_PATTERNS: tuple[str, ...] = ( + "*.gguf", "*.safetensors", "*.bin", "*.onnx", "*.pt", "*.pth", + "*.so", "*.so.*", "*.dylib", "*.dll", + "*.o", "*.a", "*.obj", "*.exe", + "*.pyc", "*.pyo", + "*.memb", + "package-lock.json", "yarn.lock", "Cargo.lock", "uv.lock", + "poetry.lock", "Pipfile.lock", "*.lock", +) + +DEFAULT_MAX_FILE_BYTES: int = 64 * 1024 + + +@dataclass(frozen=True, slots=True) +class CorpusFile: + rel_path: str # path relative to corpus root, forward slashes + bytes: int + mtime_ns: int + text: str # file content (UTF-8, replaced on errors) + + +def collect_files( + root: Path, + *, + include_suffixes: frozenset[str] = DEFAULT_INCLUDE_SUFFIXES, + exclude_dirs: frozenset[str] = DEFAULT_EXCLUDE_DIRS, + exclude_patterns: tuple[str, ...] = DEFAULT_EXCLUDE_PATTERNS, + max_file_bytes: int = DEFAULT_MAX_FILE_BYTES, +) -> list[CorpusFile]: + """Walk @root, return CorpusFile entries sorted by relative path.""" + out: list[CorpusFile] = [] + for p in root.rglob("*"): + if not p.is_file(): + continue + if any(part in exclude_dirs for part in p.parts): + continue + if p.suffix and p.suffix not in include_suffixes: + continue + if not p.suffix and p.name.lower() not in {"dockerfile", "makefile"}: + continue + if any(p.match(pat) for pat in exclude_patterns): + continue + try: + st = p.stat() + except OSError: + continue + if st.st_size > max_file_bytes: + continue + try: + text = p.read_text(encoding="utf-8") + except (UnicodeDecodeError, OSError): + continue + rel = p.relative_to(root).as_posix() + out.append(CorpusFile( + rel_path=rel, bytes=st.st_size, mtime_ns=st.st_mtime_ns, text=text, + )) + out.sort(key=lambda f: f.rel_path) + return out + + +def build_corpus(root: Path, files: list[CorpusFile]) -> str: + """Format files into a single text block with a manifest at the top.""" + if not files: + return "" + n_chars = sum(len(f.text) for f in files) + header = ( + f"=== CODEBASE: {root.name} ===\n" + f"{len(files)} files, {n_chars:,} characters total.\n\n" + f"File manifest:\n" + + "\n".join(f" {f.rel_path}" for f in files) + + "\n" + ) + bodies = "\n".join( + f"\n=== {f.rel_path} ({f.bytes} bytes) ===\n{f.text}" + for f in files + ) + return header + bodies + + +def manifest_hash(files: list[CorpusFile]) -> str: + """Stable SHA-256 over (rel_path, size, mtime_ns) tuples. + Changes whenever any included file is added, removed, or modified. + Returns 16 hex chars (enough for cache key uniqueness, easy to log).""" + h = hashlib.sha256() + for f in files: + h.update(f"{f.rel_path}\0{f.bytes}\0{f.mtime_ns}\n".encode("utf-8")) + return h.hexdigest()[:16] diff --git a/memwalk/engine.py b/memwalk/engine.py new file mode 100644 index 0000000..fad5388 --- /dev/null +++ b/memwalk/engine.py @@ -0,0 +1,151 @@ +"""Thin orchestration over corpus + cache + memba Session. +Shared by the CLI and the MCP server so they behave identically. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from pathlib import Path + +from memba import Session + +from . import cache, corpus +from .config import Config + +# Prompt that frames the ingest call so the assistant turn stored in state +# is *substantive* (not "noted") — avoids the contextual inertia bug we +# hit in v0.1. +_INGEST_PROMPT = ( + "Below is the entire source of a codebase. Read all files carefully — " + "I will ask specific questions about the code in later turns. After " + "reading, briefly state which 2-3 files seem most central and what the " + "project appears to do, in two short sentences." +) + +# Wrapping prefix on every query so the model switches out of any +# acknowledgement pattern and engages with the loaded codebase. +_QUERY_FRAMING = ( + "Drawing on the source code I shared with you earlier, please answer " + "this clearly and concretely:\n\n" +) + + +@dataclass(slots=True) +class DigestResult: + meta: cache.CacheMeta + elapsed_s: float + ack: str + char_rate: float + + +# ── digest ────────────────────────────────────────────────────── + +def digest( + cfg: Config, + source_path: Path, + *, + n_ctx: int | None = None, + force: bool = False, + verbose: bool = False, +) -> DigestResult: + """Ingest @source_path into a cached SSM state. + + If a fresh cache already exists and force is False, returns it + without touching the model. + """ + if not source_path.exists() or not source_path.is_dir(): + raise NotADirectoryError(source_path) + + n_ctx = n_ctx or cfg.n_ctx + files = corpus.collect_files(source_path) + if not files: + raise RuntimeError(f"No source files found under {source_path}") + + mh = corpus.manifest_hash(files) + existing = cache.load_meta(source_path) + if existing and not force and cache.is_fresh(existing, mh): + existing.touch() + return DigestResult(meta=existing, elapsed_s=0.0, ack="(cache hit)", + char_rate=0.0) + + # Wipe stale cache state file so memba Session doesn't auto-load it + if existing: + cache.drop(source_path) + + text = corpus.build_corpus(source_path, files) + n_chars = len(text) + + sess = Session( + model_path=str(cfg.model_path), + session_id=cache.session_id_for(source_path), + state_dir=str(cache.state_dir()), + n_gpu_layers=cfg.n_gpu_layers, + n_ctx=n_ctx, + chat_format="chatml", + verbose=verbose, + ) + + t0 = time.time() + ack = sess.chat(f"{_INGEST_PROMPT}\n\n{text}", max_tokens=160) + elapsed = time.time() - t0 + sess.save() + + meta = cache.write_meta( + source_path, + manifest_hash=mh, + n_files=len(files), + n_chars=n_chars, + n_ctx=n_ctx, + model_path=str(cfg.model_path), + ) + return DigestResult(meta=meta, elapsed_s=elapsed, ack=ack, + char_rate=n_chars / elapsed if elapsed > 0 else 0.0) + + +# ── ask ───────────────────────────────────────────────────────── + +def ask( + cfg: Config, + source_path: Path, + question: str, + *, + max_tokens: int = 400, + auto_digest: bool = True, + verbose: bool = False, +) -> tuple[str, cache.CacheMeta, bool]: + """Load cached state for @source_path and ask a question. + + Returns (answer, meta, was_digested_now). + + If no fresh cache exists and auto_digest is True, runs digest first. + """ + files = corpus.collect_files(source_path) + if not files: + raise RuntimeError(f"No source files found under {source_path}") + mh = corpus.manifest_hash(files) + + meta = cache.load_meta(source_path) + was_digested_now = False + if meta is None or not cache.is_fresh(meta, mh): + if not auto_digest: + raise RuntimeError( + f"No fresh cache for {source_path}. " + "Run `memwalk digest` first, or pass auto_digest=True." + ) + result = digest(cfg, source_path, verbose=verbose) + meta = result.meta + was_digested_now = True + + sess = Session( + model_path=str(cfg.model_path), + session_id=meta.key, + state_dir=str(cache.state_dir()), + n_gpu_layers=cfg.n_gpu_layers, + n_ctx=meta.n_ctx, # MUST match the n_ctx the cache was built at + chat_format="chatml", + verbose=verbose, + ) + answer = sess.chat(_QUERY_FRAMING + question, max_tokens=max_tokens) + meta.touch() + return answer, meta, was_digested_now diff --git a/memwalk/ingest.py b/memwalk/ingest.py deleted file mode 100644 index 5824f31..0000000 --- a/memwalk/ingest.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Orchestrate: pull events from sources, format, feed to a memba Session.""" - -from __future__ import annotations - -import time -from datetime import datetime, timedelta -from pathlib import Path - -from memba import Session - -from .config import Config, read_last_update, write_last_update -from .snapshot import maybe_snapshot -from .sources import Event, bash as bash_src, git as git_src - - -def open_session(cfg: Config, *, verbose: bool = False) -> Session: - """Open a memba Session pointing at memwalk's rolling state file.""" - return Session( - model_path=str(cfg.model_path), - session_id="current", - state_dir=str(cfg.state_dir), - n_gpu_layers=cfg.n_gpu_layers, - n_ctx=cfg.n_ctx, - chat_format="chatml", - verbose=verbose, - ) - - -def collect_all(cfg: Config, since: datetime) -> dict[str, list[Event]]: - """Pull events from every enabled source.""" - out: dict[str, list[Event]] = {} - if cfg.git.scan_paths: - out["git"] = git_src.collect(cfg.git.scan_paths, since) - if cfg.bash.enabled: - out["bash"] = bash_src.collect( - cfg.bash.history_file, since, cfg.bash.session_gap_min - ) - return out - - -def format_block(events_by_source: dict[str, list[Event]], - since: datetime, until: datetime) -> str: - """Build a single human-readable text block for ingestion.""" - chunks: list[str] = [ - f"=== Activity from {since.date()} to {until.date()} ===" - ] - if events_by_source.get("git"): - chunks.append(git_src.format_block(events_by_source["git"])) - if events_by_source.get("bash"): - chunks.append(bash_src.format_block(events_by_source["bash"])) - return "\n\n".join(chunks) - - -def update(cfg: Config, *, verbose: bool = False) -> dict: - """Ingest new events into the state and return a small summary dict.""" - now = datetime.now() - since = read_last_update(cfg) - if since is None: - # First run: bootstrap window - since = now - timedelta(days=cfg.git.bootstrap_days) - - events = collect_all(cfg, since) - n_git = len(events.get("git", [])) - n_bash = len(events.get("bash", [])) - - if n_git + n_bash == 0: - return {"ingested": 0, "git": 0, "bash": 0, "since": since, "until": now} - - block = format_block(events, since, now) - - # Rotate daily snapshot BEFORE mutating state - snapshotted = maybe_snapshot(cfg) - - sess = open_session(cfg, verbose=verbose) - - prompt = ( - "Below is a record of my recent work activity. Read it carefully, " - "then in two short sentences describe (a) the dominant theme of " - "this period and (b) one or two standout projects. I will ask " - "specific follow-up questions in later turns.\n\n" + block - ) - - t0 = time.time() - ack = sess.chat(prompt, max_tokens=120) - elapsed = time.time() - t0 - - sess.save() - write_last_update(cfg, now) - - return { - "ingested": n_git + n_bash, - "git": n_git, - "bash": n_bash, - "since": since, - "until": now, - "ack": ack, - "elapsed_s": elapsed, - "snapshotted": snapshotted, - "state_size": sess.state_size, - } - - -def query(cfg: Config, question: str, max_tokens: int = 400, - verbose: bool = False) -> str: - """Load the current state and ask a question. - - The question is wrapped in a short framing prefix so the model - switches out of any acknowledgement pattern carried by prior ingest - turns and actually answers from the activity it absorbed. - """ - if not cfg.state_path.exists(): - raise FileNotFoundError("No state yet — run `memwalk update` first.") - sess = open_session(cfg, verbose=verbose) - framed = ( - "Drawing on the work activity I shared with you earlier, please " - f"answer this clearly and concretely:\n\n{question}" - ) - return sess.chat(framed, max_tokens=max_tokens) diff --git a/memwalk/mcp_server.py b/memwalk/mcp_server.py index 6f35fa9..493bfcd 100644 --- a/memwalk/mcp_server.py +++ b/memwalk/mcp_server.py @@ -1,56 +1,33 @@ """ -MCP server — exposes memwalk as tools for Claude Code / opencode / any -MCP-aware agent. Runs over stdio. +MCP server (stdio) — exposes memwalk as tools for Claude Code / opencode / +Hermes / any MCP-aware agent. Tools: - ask(question) — query the current memwalk state, returns the answer text - standup() — generate standup notes from accumulated activity - status() — config + state metadata (no model load required) - update() — refresh state from git/bash (slow, on demand) - -Lifetime model: the underlying memba Session is loaded lazily on the first -tool call that needs it, then reused for the rest of the process — so the -first query pays ~2s of model+state load, subsequent queries are <500ms. + digest(path) — ingest a codebase into cached SSM state + ask(path, question) — query a codebase (auto-digests if needed) + list_caches() — show all cached codebases + drop_cache(path) — invalidate a cache + status() — config + cache summary """ from __future__ import annotations import asyncio import json +from pathlib import Path from typing import Any from mcp.server import Server from mcp.server.stdio import stdio_server from mcp.types import TextContent, Tool -from . import __version__ -from .config import load_config, read_last_update -from .ingest import open_session, update as run_update +from . import __version__, cache +from .config import CONFIG_PATH, load_config +from .engine import ask as engine_ask +from .engine import digest as engine_digest _server = Server("memwalk") -# Singleton session — created on first call that needs it -_session = None -_QUERY_FRAMING = ( - "Drawing on the work activity I shared with you earlier, please answer " - "this clearly and concretely:\n\n" -) - - -def _get_session(): - """Lazy-load (or reload) the memba Session.""" - global _session - if _session is None: - cfg = load_config() - _session = open_session(cfg) - return _session - - -def _reset_session() -> None: - """Drop the cached session — used after `update` so next query sees fresh state.""" - global _session - _session = None - # ── Tool declarations ──────────────────────────────────────────── @@ -58,54 +35,81 @@ def _reset_session() -> None: async def list_tools() -> list[Tool]: return [ Tool( - name="ask", + name="digest", description=( - "Query the user's accumulated work memory. Returns the model's " - "natural-language answer based on git commits and shell activity " - "previously ingested by memwalk. Use for questions like " - "'what was I working on last week?', 'which project saw the most " - "activity?', 'when did I start branch X?'." + "Read all source files under the given directory and build a " + "cached SSM state that can be queried in subsequent ask() calls. " + "Slow first time (5-30s for medium repos, longer for big ones); " + "cache is reused on subsequent calls until source files change. " + "Use before ask() to control when ingestion happens, or just call " + "ask() directly which will auto-digest as needed." ), inputSchema={ "type": "object", "properties": { - "question": { + "path": { "type": "string", - "description": "Natural-language question about the user's recent work.", + "description": "Absolute path to the codebase root.", }, - "max_tokens": { + "force": { + "type": "boolean", + "description": "Re-ingest even if cache is fresh.", + "default": False, + }, + "n_ctx": { "type": "integer", - "description": "Maximum tokens to generate (default 400).", - "default": 400, + "description": "Override config n_ctx for this digest.", }, }, - "required": ["question"], + "required": ["path"], }, ), Tool( - name="standup", + name="ask", description=( - "Generate concise daily-standup notes from the user's recent " - "activity: what they did yesterday (grouped by project), planned " - "next steps, and any blockers visible in commit messages." + "Query a codebase using its cached SSM state. Returns the " + "model's answer based on the previously digested source. " + "Auto-digests if no fresh cache exists (first call may be " + "slow). Subsequent calls on the same codebase are fast " + "(<1s typical). Best for descriptive questions: 'what does " + "module X do', 'where is concept Y used', 'list all CLI " + "commands', 'how would I add feature Z'." + ), + inputSchema={ + "type": "object", + "properties": { + "path": {"type": "string"}, + "question": {"type": "string"}, + "max_tokens": {"type": "integer", "default": 400}, + }, + "required": ["path", "question"], + }, + ), + Tool( + name="list_caches", + description=( + "Return all cached codebases as JSON: source path, file count, " + "char count, n_ctx, and last-used timestamp. Cheap — does not " + "load the model." ), inputSchema={"type": "object", "properties": {}}, ), + Tool( + name="drop_cache", + description=( + "Invalidate the cached state for the given codebase path. Next " + "ask() on that path will trigger a fresh digest." + ), + inputSchema={ + "type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"], + }, + ), Tool( name="status", description=( - "Return memwalk configuration and state metadata as JSON. " - "Cheap — does not load the model. Useful for sanity-checking " - "whether memwalk has up-to-date data." - ), - inputSchema={"type": "object", "properties": {}}, - ), - Tool( - name="update", - description=( - "Ingest new git+bash activity into the state. Slow (a few seconds — " - "loads the model). Call only when the user explicitly asks for a " - "refresh, or when status() shows the last update is stale." + "Return memwalk config and cache summary as JSON. No model load." ), inputSchema={"type": "object", "properties": {}}, ), @@ -116,59 +120,79 @@ async def list_tools() -> list[Tool]: @_server.call_tool() async def call_tool(name: str, arguments: dict[str, Any]) -> list[TextContent]: + if name == "digest": + cfg = load_config() + source = Path(arguments["path"]).expanduser().resolve() + result = await asyncio.to_thread( + engine_digest, cfg, source, + n_ctx=arguments.get("n_ctx"), + force=arguments.get("force", False), + ) + payload = { + "source_path": result.meta.source_path, + "key": result.meta.key, + "n_files": result.meta.n_files, + "n_chars": result.meta.n_chars, + "n_ctx": result.meta.n_ctx, + "elapsed_s": result.elapsed_s, + "cache_hit": result.elapsed_s == 0.0, + "model_ack": result.ack, + } + return [TextContent(type="text", text=json.dumps(payload, indent=2))] + if name == "ask": + cfg = load_config() + source = Path(arguments["path"]).expanduser().resolve() question = arguments.get("question", "").strip() if not question: return [TextContent(type="text", text="error: question is required")] - sess = await asyncio.to_thread(_get_session) - answer = await asyncio.to_thread( - sess.chat, - _QUERY_FRAMING + question, - arguments.get("max_tokens", 400), + answer, meta, just_digested = await asyncio.to_thread( + engine_ask, cfg, source, question, + max_tokens=arguments.get("max_tokens", 400), + auto_digest=True, ) - return [TextContent(type="text", text=answer)] + prefix = "(digested on demand) " if just_digested else "" + return [TextContent(type="text", text=prefix + answer)] - if name == "standup": - sess = await asyncio.to_thread(_get_session) - standup_q = ( - "Generate my daily standup notes. Cover: what I worked on yesterday " - "(grouped by project), what I plan today based on the trajectory, and " - "any blockers visible in the activity. Be concise — bullet points, " - "no preamble." - ) - answer = await asyncio.to_thread( - sess.chat, _QUERY_FRAMING + standup_q, 500 - ) - return [TextContent(type="text", text=answer)] + if name == "list_caches": + entries = cache.list_all() + out = [ + { + "source_path": m.source_path, + "key": m.key, + "n_files": m.n_files, + "n_chars": m.n_chars, + "n_ctx": m.n_ctx, + "model_path": m.model_path, + "created": m.created_iso, + "last_used": m.last_used_iso, + } + for m in entries + ] + return [TextContent(type="text", text=json.dumps(out, indent=2))] + + if name == "drop_cache": + source = Path(arguments["path"]).expanduser().resolve() + deleted = await asyncio.to_thread(cache.drop, source) + return [TextContent(type="text", + text=f"{'dropped' if deleted else 'no cache for'}: {source}")] if name == "status": - cfg = load_config() - last = read_last_update(cfg) + try: + cfg = load_config() + except FileNotFoundError as e: + return [TextContent(type="text", text=f"not configured: {e}")] + entries = cache.list_all() info = { "version": __version__, + "config_path": str(CONFIG_PATH), "model_path": str(cfg.model_path), - "state_file": str(cfg.state_path), - "state_bytes": cfg.state_path.stat().st_size if cfg.state_path.exists() else 0, - "last_update": last.isoformat() if last else None, - "scan_paths": [str(p) for p in cfg.git.scan_paths], - "bash_enabled": cfg.bash.enabled, + "n_ctx": cfg.n_ctx, + "n_gpu_layers": cfg.n_gpu_layers, + "cache_count": len(entries), } return [TextContent(type="text", text=json.dumps(info, indent=2))] - if name == "update": - cfg = load_config() - result = await asyncio.to_thread(run_update, cfg) - _reset_session() # next ask/standup sees the freshly written state - summary = ( - f"Ingested {result['ingested']} events " - f"({result['git']} commits + {result['bash']} shell sessions) " - f"in {result['elapsed_s']:.1f}s. " - f"State now {result['state_size']:,} bytes." - ) if result["ingested"] else ( - f"No new activity since {result['since'].strftime('%Y-%m-%d %H:%M')}." - ) - return [TextContent(type="text", text=summary)] - return [TextContent(type="text", text=f"unknown tool: {name}")] diff --git a/memwalk/snapshot.py b/memwalk/snapshot.py deleted file mode 100644 index c819970..0000000 --- a/memwalk/snapshot.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Daily snapshot rotation for the rolling state file.""" - -from __future__ import annotations - -import shutil -from datetime import datetime, timedelta - -from .config import Config - - -def maybe_snapshot(cfg: Config) -> bool: - """If current.memb exists and no snapshot for today, copy it. Returns True if snapshotted.""" - if not cfg.state_path.exists(): - return False - - today = datetime.now().strftime("%Y-%m-%d") - target = cfg.snapshots_dir / f"{today}.memb" - if target.exists(): - return False - - cfg.snapshots_dir.mkdir(parents=True, exist_ok=True) - shutil.copy2(cfg.state_path, target) - return True - - -def prune_old(cfg: Config, keep_days: int = 90) -> int: - """Delete snapshots older than @keep_days. Returns number deleted.""" - if not cfg.snapshots_dir.exists(): - return 0 - cutoff = (datetime.now() - timedelta(days=keep_days)).strftime("%Y-%m-%d") - deleted = 0 - for snap in cfg.snapshots_dir.glob("*.memb"): - if snap.stem < cutoff: - snap.unlink() - deleted += 1 - return deleted diff --git a/memwalk/sources/__init__.py b/memwalk/sources/__init__.py deleted file mode 100644 index 0eb8d5e..0000000 --- a/memwalk/sources/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Event sources — each module exposes `collect(since: datetime) -> list[Event]`.""" - -from dataclasses import dataclass -from datetime import datetime - - -@dataclass(frozen=True) -class Event: - """A timestamped, human-readable activity record.""" - source: str # "git", "bash", "obsidian", ... - ts: datetime - summary: str # one-line description used in ingestion blocks - detail: str = "" # optional longer body diff --git a/memwalk/sources/bash.py b/memwalk/sources/bash.py deleted file mode 100644 index 93ef878..0000000 --- a/memwalk/sources/bash.py +++ /dev/null @@ -1,153 +0,0 @@ -""" -Bash history source. - -Parses ~/.bash_history, handling the optional HISTTIMEFORMAT prefix -(`#\\n` blocks). Filters out noise (short / dupe / common -navigation / secret-looking lines) and groups remaining commands into -sessions separated by a configurable idle gap. -""" - -from __future__ import annotations - -import re -from dataclasses import dataclass -from datetime import datetime -from pathlib import Path - -from . import Event - -# Commands too short or too common to be worth remembering -_SKIP_EXACT = { - "ls", "ll", "la", "l", "cd", "cd -", "pwd", "clear", "exit", "fg", "bg", - "jobs", "history", "reset", -} -_SKIP_PREFIX = ("ls ", "cd ", "cat ", "less ", "tail ", "head ", "echo ", - "which ", "whereis ", "type ", "man ", "help ") - -# Lines that look like they leak credentials — never ingest -_SECRET_PATTERNS = [ - re.compile(r"(?i)(password|passwd|secret|api[_-]?key|access[_-]?token|bearer)\s*[=:]"), - re.compile(r"(?i)\b(aws|gcp|gh|github|hf|huggingface|openai|anthropic)[_-]?(token|key)"), - re.compile(r"(?i)sk-[a-z0-9-]{20,}"), # OpenAI/Anthropic API keys - re.compile(r"(?i)ghp_[a-z0-9]{30,}"), # GitHub PATs -] - - -def _is_noise(cmd: str) -> bool: - if len(cmd) < 3: - return True - if cmd in _SKIP_EXACT: - return True - for p in _SKIP_PREFIX: - if cmd.startswith(p) and len(cmd) < 25: - return True - return False - - -def _looks_secret(cmd: str) -> bool: - return any(p.search(cmd) for p in _SECRET_PATTERNS) - - -@dataclass -class _Raw: - ts: datetime | None - cmd: str - - -def _parse_history(path: Path) -> list[_Raw]: - """Read history file, returning entries with timestamps where available.""" - if not path.exists(): - return [] - - try: - lines = path.read_text(encoding="utf-8", errors="replace").splitlines() - except OSError: - return [] - - entries: list[_Raw] = [] - pending_ts: datetime | None = None - for line in lines: - if not line: - continue - if line.startswith("#") and line[1:].strip().isdigit(): - # HISTTIMEFORMAT marker - try: - pending_ts = datetime.fromtimestamp(int(line[1:].strip())) - except (ValueError, OSError): - pending_ts = None - continue - entries.append(_Raw(ts=pending_ts, cmd=line.strip())) - pending_ts = None - return entries - - -def collect( - history_file: Path, - since: datetime, - session_gap_min: int = 30, -) -> list[Event]: - """Return cleaned bash events newer than @since, grouped into sessions.""" - raw = _parse_history(history_file) - if not raw: - return [] - - # Filter - clean: list[_Raw] = [] - prev_cmd: str | None = None - for entry in raw: - cmd = entry.cmd - if not cmd or cmd.startswith("#"): - continue - if _is_noise(cmd) or _looks_secret(cmd): - continue - if cmd == prev_cmd: - continue - if entry.ts is not None and entry.ts < since: - continue - clean.append(entry) - prev_cmd = cmd - - if not clean: - return [] - - # Group by session (gap > session_gap_min minutes starts a new one). - # Commands without timestamps are attributed to the previous session. - events: list[Event] = [] - session: list[_Raw] = [] - last_ts: datetime | None = None - fallback_ts = since # used when entries have no timestamps - - def flush_session(items: list[_Raw]) -> None: - if not items: - return - ts = next((it.ts for it in items if it.ts is not None), fallback_ts) - commands = [it.cmd for it in items] - summary = f"shell session ({len(commands)} cmds): {commands[0][:60]}" - detail = "\n".join(f" $ {c}" for c in commands[:30]) - if len(commands) > 30: - detail += f"\n … and {len(commands)-30} more" - events.append(Event(source="bash", ts=ts, summary=summary, detail=detail)) - - gap = session_gap_min * 60 - for entry in clean: - if entry.ts is not None and last_ts is not None: - if (entry.ts - last_ts).total_seconds() > gap: - flush_session(session) - session = [] - session.append(entry) - if entry.ts is not None: - last_ts = entry.ts - flush_session(session) - return events - - -def format_block(events: list[Event]) -> str: - if not events: - return "" - parts = ["shell activity:"] - for e in events: - date = e.ts.strftime("%Y-%m-%d %H:%M") - parts.append(f"\n [{date}] {e.summary}") - if e.detail: - parts.append(e.detail) - return "\n".join(parts) diff --git a/memwalk/sources/git.py b/memwalk/sources/git.py deleted file mode 100644 index 2f0c33c..0000000 --- a/memwalk/sources/git.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Git event source — walks configured paths for repos and collects commits.""" - -from __future__ import annotations - -import subprocess -from datetime import datetime, timezone -from pathlib import Path - -from . import Event - - -def find_repos(roots: list[Path]) -> list[Path]: - """Return all distinct git repositories under the given roots (non-recursive - one level deep, plus the root itself if it's a repo).""" - repos: list[Path] = [] - seen: set[Path] = set() - for root in roots: - if not root.exists(): - continue - candidates: list[Path] = [] - if (root / ".git").exists(): - candidates.append(root) - for entry in root.iterdir(): - if entry.is_dir() and (entry / ".git").exists(): - candidates.append(entry) - for c in candidates: - real = c.resolve() - if real not in seen: - seen.add(real) - repos.append(c) - return repos - - -def collect(roots: list[Path], since: datetime) -> list[Event]: - """Collect commits across @roots that landed after @since.""" - since_iso = since.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S") - events: list[Event] = [] - - for repo in find_repos(roots): - try: - out = subprocess.check_output( - [ - "git", "-C", str(repo), "log", - f"--since={since_iso}", - "--no-merges", - "--date=iso-strict", - "--pretty=format:%H%x09%ad%x09%an%x09%s", - ], - text=True, stderr=subprocess.DEVNULL, - ) - except subprocess.CalledProcessError: - continue - - for line in out.splitlines(): - parts = line.split("\t", 3) - if len(parts) != 4: - continue - sha, iso, author, subject = parts - try: - ts = datetime.fromisoformat(iso) - except ValueError: - continue - events.append(Event( - source="git", - ts=ts, - summary=f"[{repo.name}] {subject}", - detail=f"commit {sha[:10]} by {author}", - )) - - return events - - -def format_block(events: list[Event]) -> str: - """Group commits by repo (extracted from `[name] subject`) into a readable block.""" - if not events: - return "" - - by_repo: dict[str, list[Event]] = {} - for e in events: - # Strip leading "[repo] " prefix to recover repo name - repo = e.summary.split("]", 1)[0].lstrip("[") if e.summary.startswith("[") else "unknown" - by_repo.setdefault(repo, []).append(e) - - parts: list[str] = ["git activity:"] - for repo, items in sorted(by_repo.items()): - items.sort(key=lambda x: x.ts) - parts.append(f"\n {repo} ({len(items)} commits):") - for e in items: - date = e.ts.strftime("%Y-%m-%d") - subject = e.summary.split("] ", 1)[-1] - parts.append(f" {date} {subject}") - return "\n".join(parts) diff --git a/pyproject.toml b/pyproject.toml index 2603d5e..cbcde3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,13 +4,13 @@ build-backend = "setuptools.build_meta" [project] name = "memwalk" -version = "0.1.0" -description = "Walk through your work memory — semantic recall of git, shell and notes via local SSM models" +version = "0.2.0" +description = "Ask AI about any codebase — local, cached, SSM-state-backed exploration via memba + Nemotron" readme = "README.md" license = { text = "MIT" } requires-python = ">=3.10" -keywords = ["llm", "memory", "ssm", "mamba", "nemotron", "memba", "personal-ai", "cli"] +keywords = ["llm", "ssm", "mamba", "nemotron", "memba", "code-search", "mcp", "cli"] classifiers = [ "Development Status :: 3 - Alpha", "Environment :: Console", @@ -24,7 +24,10 @@ classifiers = [ ] dependencies = [ - "memba>=0.1.0", + # memba is not on PyPI yet — install from github. + # When memba ships to PyPI this becomes `memba>=0.2.0` and memwalk + # can itself be published. + "memba @ git+https://github.com/emil28092005/Memba.git@main", "typer>=0.9.0", "rich>=13.0.0", "mcp>=1.0.0",