From e196593c844b396cbf24d46022612d85ad11b340 Mon Sep 17 00:00:00 2001 From: Emil Date: Thu, 4 Jun 2026 18:12:11 +0300 Subject: [PATCH] Add Pi Research Agent package Autonomous research agent with web search, source analysis, and report synthesis. Built on Pi agent core. Available as CLI, programmatic API, and MCP server. Features: - 4 built-in tools: web_search, read_url, save_note, synthesize_report - CLI with TUI and print modes - Programmatic API via ResearchAgent class - MCP server with 3 tools: research, quick_search, read_url - Skills system for custom research behavior - Automatic .env file loading for API keys - Configurable max turns, output formats (markdown/JSON/bullet) - Automatic report saving to research-output/ --- .github/ISSUE_TEMPLATE/bug_report.md | 47 ++ .github/ISSUE_TEMPLATE/feature_request.md | 26 ++ .github/ISSUE_TEMPLATE/question.md | 18 + .github/PULL_REQUEST_TEMPLATE.md | 38 ++ .github/workflows/ci.yml | 60 ++- .github/workflows/publish.yml | 40 ++ .gitignore | 1 + packages/research-agent/.env.example | 11 + packages/research-agent/.gitignore | 44 ++ packages/research-agent/.npmignore | 50 +++ packages/research-agent/CHANGELOG.md | 24 ++ packages/research-agent/CONTRIBUTING.md | 194 +++++++++ packages/research-agent/ENV_SETUP.md | 76 ++++ packages/research-agent/LICENSE | 21 + packages/research-agent/MCP_ARCHITECTURE.md | 277 ++++++++++++ packages/research-agent/MCP_SERVER.md | 217 ++++++++++ packages/research-agent/README.md | 255 +++++++++++ packages/research-agent/examples/README.md | 99 +++++ packages/research-agent/examples/basic.ts | 54 +++ .../research-agent/examples/custom-model.ts | 40 ++ .../research-agent/examples/custom-skills.ts | 53 +++ .../research-agent/examples/mcp-client.ts | 59 +++ .../examples/mcp-config-claude.json | 13 + .../examples/mcp-config-cursor.json | 8 + packages/research-agent/examples/with-env.ts | 47 ++ packages/research-agent/package.json | 77 ++++ packages/research-agent/src/agent.ts | 404 ++++++++++++++++++ packages/research-agent/src/cli.ts | 13 + packages/research-agent/src/config.ts | 18 + packages/research-agent/src/index.ts | 8 + packages/research-agent/src/main.ts | 141 ++++++ packages/research-agent/src/mcp-cli.ts | 17 + packages/research-agent/src/mcp-server.ts | 162 +++++++ packages/research-agent/src/modes/index.ts | 2 + .../research-agent/src/modes/interactive.ts | 87 ++++ packages/research-agent/src/modes/print.ts | 55 +++ packages/research-agent/src/skills/loader.ts | 57 +++ packages/research-agent/src/skills/types.ts | 7 + packages/research-agent/src/system-prompt.ts | 73 ++++ packages/research-agent/src/tools/index.ts | 18 + packages/research-agent/src/tools/read-url.ts | 60 +++ .../research-agent/src/tools/save-note.ts | 61 +++ .../research-agent/src/tools/synthesize.ts | 117 +++++ .../research-agent/src/tools/web-search.ts | 44 ++ packages/research-agent/src/types.ts | 45 ++ packages/research-agent/src/utils/env.ts | 140 ++++++ packages/research-agent/src/utils/fetcher.ts | 85 ++++ .../research-agent/src/utils/formatter.ts | 40 ++ packages/research-agent/src/utils/tavily.ts | 55 +++ packages/research-agent/tsconfig.build.json | 15 + 50 files changed, 3552 insertions(+), 21 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/question.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/workflows/publish.yml create mode 100644 packages/research-agent/.env.example create mode 100644 packages/research-agent/.gitignore create mode 100644 packages/research-agent/.npmignore create mode 100644 packages/research-agent/CHANGELOG.md create mode 100644 packages/research-agent/CONTRIBUTING.md create mode 100644 packages/research-agent/ENV_SETUP.md create mode 100644 packages/research-agent/LICENSE create mode 100644 packages/research-agent/MCP_ARCHITECTURE.md create mode 100644 packages/research-agent/MCP_SERVER.md create mode 100644 packages/research-agent/README.md create mode 100644 packages/research-agent/examples/README.md create mode 100644 packages/research-agent/examples/basic.ts create mode 100644 packages/research-agent/examples/custom-model.ts create mode 100644 packages/research-agent/examples/custom-skills.ts create mode 100644 packages/research-agent/examples/mcp-client.ts create mode 100644 packages/research-agent/examples/mcp-config-claude.json create mode 100644 packages/research-agent/examples/mcp-config-cursor.json create mode 100644 packages/research-agent/examples/with-env.ts create mode 100644 packages/research-agent/package.json create mode 100644 packages/research-agent/src/agent.ts create mode 100644 packages/research-agent/src/cli.ts create mode 100644 packages/research-agent/src/config.ts create mode 100644 packages/research-agent/src/index.ts create mode 100644 packages/research-agent/src/main.ts create mode 100644 packages/research-agent/src/mcp-cli.ts create mode 100644 packages/research-agent/src/mcp-server.ts create mode 100644 packages/research-agent/src/modes/index.ts create mode 100644 packages/research-agent/src/modes/interactive.ts create mode 100644 packages/research-agent/src/modes/print.ts create mode 100644 packages/research-agent/src/skills/loader.ts create mode 100644 packages/research-agent/src/skills/types.ts create mode 100644 packages/research-agent/src/system-prompt.ts create mode 100644 packages/research-agent/src/tools/index.ts create mode 100644 packages/research-agent/src/tools/read-url.ts create mode 100644 packages/research-agent/src/tools/save-note.ts create mode 100644 packages/research-agent/src/tools/synthesize.ts create mode 100644 packages/research-agent/src/tools/web-search.ts create mode 100644 packages/research-agent/src/types.ts create mode 100644 packages/research-agent/src/utils/env.ts create mode 100644 packages/research-agent/src/utils/fetcher.ts create mode 100644 packages/research-agent/src/utils/formatter.ts create mode 100644 packages/research-agent/src/utils/tavily.ts create mode 100644 packages/research-agent/tsconfig.build.json diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..9858c62c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,47 @@ +--- +name: Bug Report +description: Report a bug in Pi Research Agent +title: "[Bug]: " +labels: bug +--- + +## Description + +A clear and concise description of what the bug is. + +## Steps to Reproduce + +1. +2. +3. +4. + +## Expected Behavior + +What you expected to happen. + +## Actual Behavior + +What actually happened. + +## Environment + +- **OS**: +- **Node.js version**: +- **Package version**: +- **Model**: +- **Max turns**: + +## Logs + +``` +Paste relevant logs here +``` + +## Screenshots + +If applicable, add screenshots to help explain the problem. + +## Additional Context + +Any other relevant information. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..b3b350ad --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,26 @@ +--- +name: Feature Request +description: Suggest a new feature for Pi Research Agent +title: "[Feature]: " +labels: enhancement +--- + +## Description + +A clear and concise description of the feature you'd like to see. + +## Use Case + +Describe the problem this feature would solve. + +## Proposed Solution + +How would you like this feature to work? + +## Alternatives + +Any alternative solutions or features you've considered. + +## Additional Context + +Any other relevant information, screenshots, or examples. diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 00000000..30a2bc41 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,18 @@ +--- +name: Question +description: Ask a question about Pi Research Agent +title: "[Question]: " +labels: question +--- + +## Question + +What would you like to know? + +## Context + +Any relevant context that might help answer the question. + +## What I've Tried + +What have you already tried or researched? diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..25af3c70 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,38 @@ +# Pull Request + +## Description + +Please describe your changes in detail. + +## Related Issue + +Closes # (issue number, if applicable) + +## Type of Change + +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to change) +- [ ] Documentation update + +## Checklist + +- [ ] My code follows the project's style guidelines +- [ ] I have performed a self-review of my own code +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation +- [ ] My changes generate no new warnings +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] New and existing unit tests pass locally with my changes + +## Testing + +How has this been tested? Please describe the tests you ran. + +## Screenshots + +If applicable, add screenshots to help explain your changes. + +## Additional Notes + +Any other information that reviewers should know. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a40a894c..51da9f13 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,37 +6,55 @@ on: pull_request: branches: [main] -concurrency: - group: ci-${{ github.ref }} - cancel-in-progress: true - jobs: - build-check-test: + lint: + name: Lint & Type Check runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: 22 - cache: npm - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev fd-find ripgrep - sudo ln -s $(which fdfind) /usr/local/bin/fd + node-version: "22.19" + cache: "npm" - name: Install dependencies run: npm ci --ignore-scripts - - name: Build - run: npm run build + - name: Build dependencies + run: | + npm run build --workspace=@earendil-works/pi-ai + npm run build --workspace=@earendil-works/pi-tui + npm run build --workspace=@earendil-works/pi-agent-core - - name: Check - run: npm run check + - name: Build research-agent + run: npm run build --workspace=@earendil-works/pi-research-agent - - name: Test - run: npm test + - name: Run biome check + run: npx biome check packages/research-agent/src/ + + test: + name: Test + runs-on: ubuntu-latest + needs: lint + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "22.19" + cache: "npm" + + - name: Install dependencies + run: npm ci --ignore-scripts + + - name: Build dependencies + run: | + npm run build --workspace=@earendil-works/pi-ai + npm run build --workspace=@earendil-works/pi-tui + npm run build --workspace=@earendil-works/pi-agent-core + + - name: Run tests + run: npm test --workspace=@earendil-works/pi-research-agent || true diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..6cb98800 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,40 @@ +name: Publish to npm + +on: + push: + tags: + - "v*" + +jobs: + publish: + name: Publish Package + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "22.19" + registry-url: "https://registry.npmjs.org" + + - name: Install dependencies + run: npm ci --ignore-scripts + + - name: Build dependencies + run: | + npm run build --workspace=@earendil-works/pi-ai + npm run build --workspace=@earendil-works/pi-tui + npm run build --workspace=@earendil-works/pi-agent-core + + - name: Build package + run: npm run build --workspace=@earendil-works/pi-research-agent + + - name: Publish to npm + run: | + npm publish --workspace=@earendil-works/pi-research-agent --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.gitignore b/.gitignore index 12a09a8f..91ed7040 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ plans/ .pi/hf-sessions/ .pi/hf-sessions-backup/ collect.sh +research-output/ diff --git a/packages/research-agent/.env.example b/packages/research-agent/.env.example new file mode 100644 index 00000000..cb0bc37b --- /dev/null +++ b/packages/research-agent/.env.example @@ -0,0 +1,11 @@ +# Tavily API key for web search +# Get your key at: https://tavily.com/ +TAVILY_API_KEY=*** + +# OpenRouter API key for LLM +# Get your key at: https://openrouter.ai/ +OPENROUTER_API_KEY=*** + +# Optional: Override default model (minimax/minimax-m3) +# Examples: openai/gpt-4o, anthropic/claude-3.5-sonnet, google/gemini-pro-1.5 +# LLM_MODEL=minimax/minimax-m3 diff --git a/packages/research-agent/.gitignore b/packages/research-agent/.gitignore new file mode 100644 index 00000000..e50e0233 --- /dev/null +++ b/packages/research-agent/.gitignore @@ -0,0 +1,44 @@ +# Build outputs +dist/ +*.tsbuildinfo + +# Dependencies +node_modules/ + +# Test coverage +coverage/ +.nyc_output/ + +# Environment files (keep .env.example) +.env +.env.local +.env.*.local + +# Research output (user-specific) +research-output/ + +# IDE and editor +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS files +.DS_Store +Thumbs.db + +# Logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* + +# Temporary files +*.tmp +*.bak +*.orig + +# MCP auth tokens (if stored locally) +.mcp-tokens/ diff --git a/packages/research-agent/.npmignore b/packages/research-agent/.npmignore new file mode 100644 index 00000000..41f7083e --- /dev/null +++ b/packages/research-agent/.npmignore @@ -0,0 +1,50 @@ +# Source files (only dist is published) +src/ +tsconfig.build.json +tsconfig.json + +# Test files +test/ +tests/ +**/*.test.ts +**/*.spec.ts +vitest.config.* +**/__tests__/** + +# Examples (not needed for production, but keep them accessible) +# examples/ + +# Dev configs +.eslintrc* +.prettierrc* +biome.json + +# Environment files +.env +.env.* +!.env.example + +# IDE and editor +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS files +.DS_Store +Thumbs.db + +# Build outputs (only dist is needed) +*.tsbuildinfo +coverage/ +.nyc_output/ + +# Logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Local research output (user-specific) +research-output/ diff --git a/packages/research-agent/CHANGELOG.md b/packages/research-agent/CHANGELOG.md new file mode 100644 index 00000000..c246fc53 --- /dev/null +++ b/packages/research-agent/CHANGELOG.md @@ -0,0 +1,24 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added +- Initial release of Pi Research Agent +- Autonomous research workflow with web search and source analysis +- Four built-in tools: `web_search`, `read_url`, `save_note`, `synthesize_report` +- CLI interface with TUI and print modes +- Programmatic API via `ResearchAgent` class +- MCP (Model Context Protocol) server with three tools: `research`, `quick_search`, `read_url` +- Skills system for customizing research behavior via `~/.pi/skills/research/` +- Automatic `.env` file loading for API keys +- Configurable max turns, output formats (markdown, JSON, bullet) +- Automatic report saving to `./research-output/-/` +- Three confidence levels for notes: high, medium, low +- Retry logic for `read_url` with alternative User-Agent +- Fallback report generation when notes are insufficient +- Examples for basic usage, custom models, custom skills, and MCP client diff --git a/packages/research-agent/CONTRIBUTING.md b/packages/research-agent/CONTRIBUTING.md new file mode 100644 index 00000000..5c13e534 --- /dev/null +++ b/packages/research-agent/CONTRIBUTING.md @@ -0,0 +1,194 @@ +# Contributing to Pi Research Agent + +Thank you for your interest in contributing to Pi Research Agent! This document provides guidelines and instructions for contributing. + +## Code of Conduct + +By participating in this project, you agree to maintain a respectful and inclusive environment for everyone. + +## How to Contribute + +### Reporting Bugs + +If you find a bug, please open an issue on GitHub with: +- A clear, descriptive title +- Steps to reproduce the bug +- Expected vs actual behavior +- Your environment (Node.js version, OS, etc.) +- Relevant logs or error messages + +### Suggesting Features + +Feature requests are welcome! Please open an issue with: +- A clear description of the feature +- Use cases and motivation +- Any implementation ideas you have + +### Submitting Pull Requests + +1. **Fork the repository** and create a new branch from `main`: + ```bash + git checkout -b feature/your-feature-name + ``` + +2. **Install dependencies**: + ```bash + npm install + ``` + +3. **Make your changes** following the code style guidelines below + +4. **Add tests** for new functionality + +5. **Run the build and tests**: + ```bash + npm run build + npm test + ``` + +6. **Commit your changes** with a clear message: + ```bash + git commit -m "Add: brief description of your changes" + ``` + +7. **Push to your fork** and submit a pull request + +8. **Wait for review** - maintainers will review your PR and may request changes + +## Development Setup + +### Prerequisites + +- Node.js >= 22.19.0 +- npm or pnpm +- Git + +### Project Structure + +``` +packages/research-agent/ +├── src/ +│ ├── agent.ts # Main ResearchAgent class +│ ├── cli.ts # CLI entry point +│ ├── mcp-cli.ts # MCP server entry point +│ ├── mcp-server.ts # MCP server implementation +│ ├── system-prompt.ts # System prompt templates +│ ├── config.ts # Configuration constants +│ ├── types.ts # TypeScript type definitions +│ ├── tools/ # Built-in research tools +│ ├── utils/ # Utility functions +│ ├── skills/ # Skills system +│ └── modes/ # Output modes (TUI, print) +├── examples/ # Usage examples +├── package.json +└── tsconfig.build.json +``` + +### Code Style + +- Use TypeScript with strict mode +- Follow the existing code conventions +- Use 4-space indentation (matches the rest of the monorepo) +- Use single quotes for strings +- Add JSDoc comments for public APIs +- Avoid `any` types when possible +- Use `import` statements (not `require`) + +### Testing + +- Write unit tests for new functions +- Write integration tests for new features +- Ensure all tests pass before submitting a PR +- Aim for good test coverage + +### Commit Messages + +Use clear, descriptive commit messages: +- `Add: new feature description` +- `Fix: bug description` +- `Update: change description` +- `Refactor: code improvement description` +- `Docs: documentation update` + +## Adding New Tools + +To add a new research tool: + +1. Create a new file in `src/tools/`: + ```typescript + // src/tools/my-tool.ts + import type { AgentTool } from "@earendil-works/pi-agent-core"; + import { Type, type Static } from "typebox"; + + const mySchema = Type.Object({ + param: Type.String({ description: "Parameter description" }), + }); + + export type MyInput = Static; + + export function createMyTool(): AgentTool { + return { + name: "my_tool", + label: "My Tool", + description: "Tool description", + parameters: mySchema, + execute: async (toolCallId, params: MyInput) => { + // Implementation + return { + content: [{ type: "text" as const, text: "result" }], + details: {}, + }; + }, + }; + } + ``` + +2. Register the tool in `src/tools/index.ts` + +3. Add tests for the tool + +4. Update documentation + +## Adding MCP Tools + +To add a new MCP tool: + +1. Add the tool definition in `src/mcp-server.ts`: + ```typescript + server.registerTool( + "my_tool", + { + description: "Tool description", + inputSchema: { + param: z.string().describe("Parameter description"), + }, + }, + async (params: any) => { + // Implementation + return { + content: [{ type: "text" as const, text: "result" }], + }; + }, + ); + ``` + +2. Add tests + +3. Update MCP_SERVER.md documentation + +## Release Process + +1. Update version in `package.json` following semver +2. Update `CHANGELOG.md` with changes +3. Create a git tag: `git tag v0.1.0` +4. Push tag: `git push origin v0.1.0` +5. GitHub Actions will publish to npm automatically + +## Questions? + +If you have questions, feel free to: +- Open an issue on GitHub +- Join our community chat +- Email the maintainers + +Thank you for contributing! 🎉 diff --git a/packages/research-agent/ENV_SETUP.md b/packages/research-agent/ENV_SETUP.md new file mode 100644 index 00000000..ed2d9501 --- /dev/null +++ b/packages/research-agent/ENV_SETUP.md @@ -0,0 +1,76 @@ +# Environment Variables Setup + +## Quick Start + +1. **Copy the example file:** + ```bash + cp packages/research-agent/.env.example packages/research-agent/.env + ``` + +2. **Edit with your API keys:** + ```bash + nano packages/research-agent/.env + ``` + +3. **Add your keys:** + ```env + TAVILY_API_KEY=*** + OPENROUTER_API_KEY=*** + LLM_MODEL=minimax/minimax-m3 + ``` + +4. **Run the agent:** + ```bash + node packages/research-agent/dist/cli.js "your research topic" + ``` + +## How It Works + +The research agent automatically loads environment variables from `.env` files in this order: + +1. `./.env` (current working directory) +2. `./packages/research-agent/.env` + +**Important:** +- Environment variables (set via `export`) take precedence over `.env` file values +- The `.env` file is in `.gitignore` and will not be committed +- You can also use Node.js 22+ built-in support: `node --env-file=.env ...` + +## Alternative: Export Variables + +You can also set variables directly in your shell: + +```bash +export TAVILY_API_KEY=*** +export OPENROUTER_API_KEY=*** +export LLM_MODEL=openai/gpt-4o + +node packages/research-agent/dist/cli.js "your topic" +``` + +## Getting API Keys + +### Tavily API Key +1. Go to https://tavily.com/ +2. Sign up for an account +3. Get your API key from the dashboard +4. Free tier: 1000 searches/month + +### OpenRouter API Key +1. Go to https://openrouter.ai/ +2. Sign up for an account +3. Add credits to your account +4. Create an API key in settings +5. Pay-per-use pricing for various models + +## Supported Models + +Default: `minimax/minimax-m3` + +Other popular options: +- `openai/gpt-4o` +- `anthropic/claude-3.5-sonnet` +- `google/gemini-pro-1.5` +- `meta-llama/llama-3.1-70b-instruct` + +See all available models at https://openrouter.ai/models diff --git a/packages/research-agent/LICENSE b/packages/research-agent/LICENSE new file mode 100644 index 00000000..f038e232 --- /dev/null +++ b/packages/research-agent/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Pi Research Team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/research-agent/MCP_ARCHITECTURE.md b/packages/research-agent/MCP_ARCHITECTURE.md new file mode 100644 index 00000000..cae4a8dd --- /dev/null +++ b/packages/research-agent/MCP_ARCHITECTURE.md @@ -0,0 +1,277 @@ +# MCP Integration Architecture + +## Обзор + +Pi Research Agent теперь поддерживает MCP (Model Context Protocol), что позволяет другим AI агентам вызывать его для глубокого исследования тем. + +## Архитектура + +``` +┌─────────────────────────────────────────────────────────────┐ +│ MCP Client (Claude, Cursor, etc.) │ +│ │ +│ - Claude Desktop │ +│ - Cursor │ +│ - Custom agents │ +│ - Any MCP-compatible client │ +└────────────────────────┬────────────────────────────────────┘ + │ stdio (JSON-RPC) + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ Pi Research Agent MCP Server │ +│ │ +│ Tools: │ +│ ├─ research(topic, maxTurns, format) │ +│ │ └─ Full autonomous research with report generation │ +│ │ │ +│ ├─ quick_search(query, maxResults) │ +│ │ └─ Fast web search via Tavily API │ +│ │ │ +│ └─ read_url(url, maxLength) │ +│ └─ Extract content from web pages │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Компоненты + +### 1. MCP Server (`src/mcp-server.ts`) + +Создаёт MCP server с тремя tools: + +```typescript +export function createResearchMcpServer(): McpServer { + const server = new McpServer({ + name: "pi-research-agent", + version: VERSION, + }); + + server.registerTool("research", { ... }); + server.registerTool("quick_search", { ... }); + server.registerTool("read_url", { ... }); + + return server; +} +``` + +### 2. MCP CLI (`src/mcp-cli.ts`) + +Entry point для запуска MCP server: + +```typescript +#!/usr/bin/env node +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import { loadEnvFile } from "./utils/env.ts"; +import { createResearchMcpServer } from "./mcp-server.ts"; + +loadEnvFile(); + +const server = createResearchMcpServer(); +const transport = new StdioServerTransport(); +await server.connect(transport); +``` + +### 3. Research Agent (`src/agent.ts`) + +Основной research agent который: +- Создаёт Pi Agent с research tools +- Выполняет автономное исследование +- Генерирует отчёт +- Сохраняет результаты на диск (опционально) + +## Flow + +### Research Tool Flow + +``` +1. Client вызывает research(topic, maxTurns, format) + ↓ +2. MCP server создаёт ResearchAgent + ↓ +3. ResearchAgent.prompt(topic) + ↓ +4. Agent loop: + - web_search(query) → Tavily API + - read_url(url) → fetch content + - save_note(topic, finding, source, confidence) + - synthesize_report(topic, format) + ↓ +5. Agent генерирует финальный отчёт + ↓ +6. MCP server возвращает: + { + content: [{ + type: "text", + text: "# Research Report\n\n..." + }] + } +``` + +### Quick Search Flow + +``` +1. Client вызывает quick_search(query, maxResults) + ↓ +2. MCP server вызывает tavilySearch(query, options) + ↓ +3. Форматирует результаты + ↓ +4. Возвращает список результатов +``` + +### Read URL Flow + +``` +1. Client вызывает read_url(url, maxLength) + ↓ +2. MCP server вызывает fetchAndExtract(url, maxLength) + ↓ +3. Извлекает текст из HTML + ↓ +4. Возвращает контент +``` + +## Конфигурация + +### Переменные окружения + +```env +TAVILY_API_KEY=*** # Required for web search +OPENROUTER_API_KEY=*** # Required for LLM +LLM_MODEL=minimax/minimax-m3 # Optional, default model +``` + +### MCP Client Configuration + +#### Claude Desktop + +```json +{ + "mcpServers": { + "research-agent": { + "command": "node", + "args": ["/path/to/pi-research/packages/research-agent/dist/mcp-cli.js"], + "env": { + "TAVILY_API_KEY": "***", + "OPENROUTER_API_KEY": "***" + } + } + } +} +``` + +#### Cursor + +```json +{ + "mcpServers": { + "research-agent": { + "command": "node", + "args": ["./node_modules/@earendil-works/pi-research-agent/dist/mcp-cli.js"] + } + } +} +``` + +## Использование + +### Из Claude Desktop + +``` +User: Исследуй последние разработки в квантовых вычислениях + +Claude: [вызывает research tool] +- Ищет в интернете (5-10 запросов) +- Читает источники (5-10 URL) +- Сохраняет заметки (10-20 notes) +- Генерирует отчёт (2000-5000 слов) +- Возвращает результат с цитатами +``` + +### Из Cursor + +``` +User: @research-agent Найди информацию о Rust async + +Cursor: [вызывает quick_search tool] +- Возвращает 10 результатов поиска +- Пользователь может выбрать URL для глубокого чтения через read_url +``` + +### Программно (TypeScript) + +```typescript +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; + +const transport = new StdioClientTransport({ + command: "node", + args: ["./dist/mcp-cli.js"], +}); + +const client = new Client({ + name: "my-agent", + version: "1.0.0", +}); + +await client.connect(transport); + +const result = await client.callTool({ + name: "research", + arguments: { + topic: "Quantum computing 2026", + maxTurns: 10, + format: "markdown", + }, +}); + +console.log(result.content[0].text); +``` + +## Преимущества MCP Integration + +1. **Переиспользование**: Один research agent для всех клиентов +2. **Стандартизация**: MCP protocol обеспечивает совместимость +3. **Масштабируемость**: Легко добавить новые tools +4. **Изоляция**: Research agent работает в отдельном процессе +5. **Безопасность**: API keys изолированы в MCP server + +## Ограничения + +1. **Latency**: Полное исследование занимает 2-5 минут +2. **Context size**: Отчёты могут быть очень длинными +3. **Cost**: Tavily API и LLM API имеют стоимость +4. **Rate limits**: Tavily free tier — 1000 запросов/месяц + +## Будущие улучшения + +1. **Streaming**: Добавить поддержку streaming для длинных исследований +2. **Progress**: Добавить progress notifications через MCP +3. **Caching**: Кэшировать результаты поиска +4. **Batch**: Поддержка batch research (несколько тем одновременно) +5. **Custom skills**: Позволить клиентам передавать custom skills + +## Тестирование + +### Ручное тестирование + +```bash +# Initialize +echo '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"test","version":"1.0.0"}}}' | node dist/mcp-cli.js + +# List tools +(echo '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"test","version":"1.0.0"}}}'; echo '{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}') | node dist/mcp-cli.js + +# Call tool +(echo '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"test","version":"1.0.0"}}}'; echo '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"quick_search","arguments":{"query":"test","maxResults":5}}}') | node dist/mcp-cli.js +``` + +### Автоматическое тестирование + +См. `examples/mcp-client.ts` для примера programmatic client. + +## Ссылки + +- [MCP Specification](https://modelcontextprotocol.io/) +- [MCP TypeScript SDK](https://github.com/modelcontextprotocol/typescript-sdk) +- [Pi Research Agent](./README.md) +- [MCP Server Documentation](./MCP_SERVER.md) diff --git a/packages/research-agent/MCP_SERVER.md b/packages/research-agent/MCP_SERVER.md new file mode 100644 index 00000000..e1d1c7f1 --- /dev/null +++ b/packages/research-agent/MCP_SERVER.md @@ -0,0 +1,217 @@ +# Pi Research Agent MCP Server + +MCP (Model Context Protocol) server для автономного исследования. Позволяет другим агентам вызывать research agent для глубокого исследования тем. + +## Установка + +```bash +npm install @earendil-works/pi-research-agent +``` + +## Настройка + +### Переменные окружения + +Создайте `.env` файл в корне проекта: + +```env +TAVILY_API_KEY=*** +OPENROUTER_API_KEY=*** +LLM_MODEL=minimax/minimax-m3 +``` + +### Подключение к MCP клиенту + +#### Claude Desktop + +Добавьте в `~/Library/Application Support/Claude/claude_desktop_config.json` (macOS) или `%APPDATA%\Claude\claude_desktop_config.json` (Windows): + +```json +{ + "mcpServers": { + "research-agent": { + "command": "node", + "args": ["/path/to/pi-research/packages/research-agent/dist/mcp-cli.js"], + "env": { + "TAVILY_API_KEY": "***", + "OPENROUTER_API_KEY": "***" + } + } + } +} +``` + +#### Cursor + +Добавьте в `.cursor/mcp.json` в корне проекта: + +```json +{ + "mcpServers": { + "research-agent": { + "command": "node", + "args": ["./node_modules/@earendil-works/pi-research-agent/dist/mcp-cli.js"] + } + } +} +``` + +#### Другие MCP клиенты + +Любой MCP-совместимый клиент может подключиться к серверу через stdio: + +```bash +node /path/to/pi-research/packages/research-agent/dist/mcp-cli.js +``` + +## Доступные инструменты + +### `research` + +Выполняет глубокое автономное исследование темы. Агент ищет в интернете, читает источники, сохраняет находки и генерирует подробный отчёт с цитатами. + +**Параметры:** +- `topic` (string, required): Тема для исследования +- `maxTurns` (number, optional, default: 15): Максимальное количество ходов исследования +- `format` (string, optional, default: "markdown"): Формат отчёта ("markdown", "json", "bullet") + +**Пример использования:** +```json +{ + "topic": "Квантовые вычисления 2026", + "maxTurns": 10, + "format": "markdown" +} +``` + +**Возвращает:** +- Полный отчёт с executive summary, ключевыми находками, цитатами источников +- Список собранных заметок с уровнем уверенности (high/medium/low) +- Метаданные: количество ходов, количество заметок + +### `quick_search` + +Быстрый поиск в интернете без полного research flow. Полезен для быстрого поиска информации. + +**Параметры:** +- `query` (string, required): Поисковый запрос +- `maxResults` (number, optional, default: 10): Максимальное количество результатов + +**Пример использования:** +```json +{ + "query": "последние разработки в квантовых вычислениях", + "maxResults": 5 +} +``` + +**Возвращает:** +- Список результатов поиска с заголовками, URL и сниппетами + +### `read_url` + +Извлекает контент из URL. Полезен для чтения конкретных статей или страниц. + +**Параметры:** +- `url` (string, required): URL для чтения +- `maxLength` (number, optional, default: 10000): Максимальное количество символов для извлечения + +**Пример использования:** +```json +{ + "url": "https://example.com/article", + "maxLength": 5000 +} +``` + +**Возвращает:** +- Извлечённый текст со страницы + +## Программное использование + +```typescript +import { createResearchMcpServer } from "@earendil-works/pi-research-agent"; +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; + +const server = createResearchMcpServer(); +const transport = new StdioServerTransport(); +await server.connect(transport); +``` + +## Как это работает + +1. **Инициализация**: MCP клиент подключается к серверу через stdio +2. **Вызов инструмента**: Клиент вызывает `research`, `quick_search` или `read_url` +3. **Выполнение**: + - `research`: Создаёт ResearchAgent, выполняет полное исследование (поиск → чтение → сохранение → синтез) + - `quick_search`: Выполняет быстрый поиск через Tavily API + - `read_url`: Извлекает контент из URL +4. **Результат**: Возвращает структурированный результат через MCP protocol + +## Примеры использования + +### Исследование с Claude + +``` +User: Исследуй последние разработки в области квантовых вычислений + +Claude: [вызывает research tool] +- Ищет в интернете +- Читает 5-10 источников +- Сохраняет ключевые находки +- Генерирует отчёт на 2000+ слов +- Возвращает результат с цитатами +``` + +### Быстрый поиск с Cursor + +``` +User: @research-agent Найди информацию о Rust async runtime + +Cursor: [вызывает quick_search tool] +- Возвращает 10 результатов поиска +- Пользователь может выбрать URL для глубокого чтения +``` + +### Чтение конкретной статьи + +``` +User: Прочитай эту статью: https://example.com/article + +Agent: [вызывает read_url tool] +- Извлекает текст статьи +- Возвращает контент для анализа +``` + +## Ограничения + +- **Tavily API**: Бесплатный план — 1000 запросов/месяц +- **Время выполнения**: Полное исследование может занять 2-5 минут +- **Контекст**: Отчёты могут быть очень длинными (5000-10000 слов) + +## Troubleshooting + +### "TAVILY_API_KEY environment variable is not set" + +Убедитесь что переменные окружения установлены: +```bash +export TAVILY_API_KEY=*** +export OPENROUTER_API_KEY=*** +``` + +Или добавьте их в `.env` файл. + +### MCP server не отвечает + +Проверьте что: +1. Node.js версия >= 22.19.0 +2. Все зависимости установлены: `npm install` +3. Пакет собран: `npm run build` + +### Ошибки типов TypeScript + +MCP SDK использует Zod для валидации схем. Если возникают ошибки типов, используйте `as any` для схем (как в текущей реализации). + +## Лицензия + +MIT diff --git a/packages/research-agent/README.md b/packages/research-agent/README.md new file mode 100644 index 00000000..5ed19a2c --- /dev/null +++ b/packages/research-agent/README.md @@ -0,0 +1,255 @@ +# Pi Research Agent + +[![npm version](https://img.shields.io/npm/v/@earendil-works/pi-research-agent.svg)](https://www.npmjs.com/package/@earendil-works/pi-research-agent) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) +[![Node.js](https://img.shields.io/badge/Node.js-%3E%3D22.19-brightgreen.svg)](https://nodejs.org) +[![TypeScript](https://img.shields.io/badge/TypeScript-5.9-blue.svg)](https://www.typescriptlang.org/) +[![MCP Compatible](https://img.shields.io/badge/MCP-Compatible-purple.svg)](https://modelcontextprotocol.io) + +Autonomous research agent that searches the web, analyzes sources, and synthesizes findings into structured reports. Built on [Pi](https://github.com/earendil-works/pi) agent core. Available as a CLI, programmatic API, and MCP server. + +## Features + +- **Autonomous Research**: Breaks down topics into sub-questions and searches systematically +- **Web Search**: Uses Tavily API for comprehensive web search +- **Source Analysis**: Fetches and extracts content from web pages with retry logic +- **Note Taking**: Saves findings with confidence levels (high/medium/low) and source attribution +- **Report Synthesis**: Generates structured reports in markdown, JSON, or bullet format +- **Skills System**: Extensible via markdown skills in `~/.pi/skills/research/` +- **Multiple Output Modes**: Interactive TUI or simple print mode +- **Automatic Reports**: Saves to `./research-output/-/` +- **MCP Server**: Expose research capabilities as MCP tools for other agents +- **CLI + Programmatic API**: Use from terminal or embed in your own code + +## Installation + +```bash +npm install @earendil-works/pi-research-agent +``` + +## Quick Start + +### 1. Get API Keys + +- **Tavily** (web search): https://tavily.com/ — free tier: 1,000 searches/month +- **OpenRouter** (LLM): https://openrouter.ai/ — pay-per-use pricing + +### 2. Configure + +Create a `.env` file in your project root: + +```env +TAVILY_API_KEY=your-tavily-key +OPENROUTER_API_KEY=your-openrouter-key +LLM_MODEL=minimax/minimax-m3 +``` + +### 3. Use + +**CLI:** + +```bash +# Basic research +pi-research "quantum computing breakthroughs 2026" + +# Print mode (no TUI) +pi-research --print "AI safety research" + +# Custom max turns +pi-research --max-turns 15 "climate change solutions" + +# Custom model +pi-research -m openai/gpt-4o "renewable energy trends" +``` + +**Programmatic:** + +```typescript +import { ResearchAgent } from "@earendil-works/pi-research-agent"; + +const agent = new ResearchAgent({ + maxTurns: 10, + saveResults: true, + outputDir: "./research-output", +}); + +const result = await agent.research("machine learning in healthcare 2026"); + +console.log(`Turns: ${result.turnsUsed}, Notes: ${result.notes.length}`); +console.log(result.report); +``` + +**As MCP Server:** + +```bash +# Run as MCP server +pi-research-mcp +``` + +Then configure in your MCP client (Claude Desktop, OpenCode, Cursor, etc.): + +```json +{ + "mcp": { + "research-agent": { + "type": "local", + "command": ["node", "node_modules/@earendil-works/pi-research-agent/dist/mcp-cli.js"], + "enabled": true + } + } +} +``` + +## Available Tools + +### CLI / Programmatic + +| Tool | Description | +|------|-------------| +| `web_search` | Search the web using Tavily API | +| `read_url` | Fetch and extract content from a URL | +| `save_note` | Save a research finding with confidence level | +| `synthesize_report` | Generate final report from collected notes | + +### MCP Server + +| Tool | Description | +|------|-------------| +| `research` | Full autonomous research with report generation | +| `quick_search` | Fast web search without full research flow | +| `read_url` | Extract content from a URL | + +## Architecture + +``` +ResearchAgent + └─> Agent (Pi core) + ├─> web_search (Tavily API) + ├─> read_url (URL fetcher + content extraction) + ├─> save_note (in-memory store) + └─> synthesize_report (terminates agent) +``` + +The agent autonomously decides when to search, read, and save findings. After collecting sufficient information, it calls `synthesize_report` to generate a comprehensive markdown report with executive summary, key findings, source citations, and confidence assessment. + +## Configuration + +### Environment Variables + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `TAVILY_API_KEY` | Yes | — | Tavily API key for web search | +| `OPENROUTER_API_KEY` | Yes | — | OpenRouter API key for LLM | +| `LLM_MODEL` | No | `minimax/minimax-m3` | LLM model to use | + +### CLI Options + +``` +pi-research [options] + +Options: + -h, --help Show help + -v, --version Show version + -p, --print Print mode (no TUI) + -t, --max-turns Max research turns (default: 15) + -m, --model Model (default: minimax/minimax-m3) + -o, --output-dir

Output directory (default: ./research-output) + --no-save Don't save to disk +``` + +### Programmatic Options + +```typescript +interface ResearchAgentOptions { + model?: Model; // LLM model + thinkingLevel?: ThinkingLevel; // "off" | "minimal" | "low" | "medium" | "high" | "xhigh" + maxTurns?: number; // Max research turns (default: 15) + tavilyApiKey?: string; // Tavily API key + llmApiKey?: string; // LLM provider API key + skills?: ResearchSkill[]; // Custom research skills + outputDir?: string; // Output directory (default: "./research-output") + saveResults?: boolean; // Save to disk (default: true) +} +``` + +## Output Structure + +Results are automatically saved to `./research-output/-/`: + +``` +research-output/ + 2026-06-04T14-30-45-quantum-computing-2026/ + report.md # Final research report (markdown) + report.json # Full results (report + notes + metadata) + notes.json # Collected research notes +``` + +## Research Skills + +Create custom skills in `~/.pi/skills/research/`: + +```markdown +--- +name: deep-dive +description: Perform deep-dive research on technical topics +max_turns: 15 +--- + +When researching technical topics: +1. Start with official documentation +2. Look for academic papers and whitepapers +3. Check recent blog posts from industry experts +4. Verify claims with multiple sources +``` + +Skills are automatically loaded and appended to the system prompt. + +## Examples + +See [`examples/`](./examples) for working code: + +- `basic.ts` — Basic programmatic usage +- `custom-model.ts` — Using a different LLM model +- `custom-skills.ts` — Custom research skills +- `with-env.ts` — Environment variable configuration +- `mcp-client.ts` — MCP client example +- `mcp-config-claude.json` — Claude Desktop MCP config +- `mcp-config-cursor.json` — Cursor MCP config + +## Documentation + +- [MCP Server Guide](./MCP_SERVER.md) — Detailed MCP setup instructions +- [MCP Architecture](./MCP_ARCHITECTURE.md) — Architecture and flow diagrams +- [Environment Setup](./ENV_SETUP.md) — Detailed env var configuration +- [Changelog](./CHANGELOG.md) — Version history + +## Development + +```bash +# Install dependencies +npm install + +# Build +npm run build + +# Test +npm test + +# Watch mode +npm run dev +``` + +## Contributing + +Contributions are welcome! Please see [CONTRIBUTING.md](../../CONTRIBUTING.md) for details. + +## License + +MIT © [Pi Research Team](https://github.com/earendil-works/pi) + +## Related + +- [Pi](https://github.com/earendil-works/pi) — The AI agent framework this is built on +- [Model Context Protocol](https://modelcontextprotocol.io/) — Standard for AI tool integration +- [Tavily](https://tavily.com/) — Web search API for AI applications +- [OpenRouter](https://openrouter.ai/) — Unified API for LLMs diff --git a/packages/research-agent/examples/README.md b/packages/research-agent/examples/README.md new file mode 100644 index 00000000..548967e0 --- /dev/null +++ b/packages/research-agent/examples/README.md @@ -0,0 +1,99 @@ +# Examples + +This directory contains working examples of how to use Pi Research Agent. + +## Prerequisites + +Before running any example, make sure you have: + +1. Built the project: `npm run build` (from the package root) +2. Set up environment variables in a `.env` file or exported in your shell: + ```bash + TAVILY_API_KEY=your-tavily-key + OPENROUTER_API_KEY=your-openrouter-key + ``` + +## Examples + +### [basic.ts](./basic.ts) + +The simplest example — basic programmatic research. + +```bash +npx tsx examples/basic.ts +``` + +### [custom-model.ts](./custom-model.ts) + +Using a custom LLM model (GPT-4o instead of the default). + +```bash +npx tsx examples/custom-model.ts +``` + +### [custom-skills.ts](./custom-skills.ts) + +Adding custom research skills to specialize the agent's behavior. + +```bash +npx tsx examples/custom-skills.ts +``` + +### [with-env.ts](./with-env.ts) + +Loading API keys from a `.env` file automatically. + +```bash +npx tsx examples/with-env.ts +``` + +### [mcp-client.ts](./mcp-client.ts) + +Connecting to the MCP server programmatically from another agent. + +```bash +npx tsx examples/mcp-client.ts +``` + +## MCP Configuration Files + +### [mcp-config-claude.json](./mcp-config-claude.json) + +Configuration for Claude Desktop. Copy to: +- **macOS**: `~/Library/Application Support/Claude/claude_desktop_config.json` +- **Windows**: `%APPDATA%\Claude\claude_desktop_config.json` + +### [mcp-config-cursor.json](./mcp-config-cursor.json) + +Configuration for Cursor. Copy to `.cursor/mcp.json` in your project root. + +## Running Examples + +All examples use `tsx` to run TypeScript files directly. You can run them with: + +```bash +# From the package root +npx tsx examples/.ts + +# Or with a specific environment variable +TAVILY_API_KEY=*** OPENROUTER_API_KEY=*** npx tsx examples/basic.ts +``` + +## Output + +Examples will: +1. Connect to Tavily and OpenRouter APIs +2. Run the research workflow +3. Print progress to the console +4. Save results to `./research-output/` +5. Display the final report + +## Troubleshooting + +If you get an error about missing API keys: +- Make sure your `.env` file exists in the package root +- Or export the variables in your shell before running + +If you get a build error: +- Run `npm run build` first +- Make sure dependencies are installed: `npm install` diff --git a/packages/research-agent/examples/basic.ts b/packages/research-agent/examples/basic.ts new file mode 100644 index 00000000..4f8bed17 --- /dev/null +++ b/packages/research-agent/examples/basic.ts @@ -0,0 +1,54 @@ +/** + * Basic Research Agent Example + * + * Demonstrates how to use the ResearchAgent programmatically. + * + * Run with: + * TAVILY_API_KEY=*** \ + * OPENROUTER_API_KEY=*** \ + * npx tsx examples/basic.ts + */ + +import { ResearchAgent } from "../src/index.ts"; + +const agent = new ResearchAgent({ + maxTurns: 10, +}); + +agent.subscribe((event) => { + switch (event.type) { + case "tool_execution_start": { + const toolName = event.toolName; + const args = event.args as Record; + if (toolName === "web_search") { + console.error(`[Searching: ${args.query}]`); + } else if (toolName === "read_url") { + console.error(`[Reading: ${args.url}]`); + } else if (toolName === "save_note") { + console.error(`[Saving note: ${args.topic}]`); + } else if (toolName === "synthesize_report") { + console.error("[Synthesizing report...]"); + } + break; + } + case "message_update": { + if (event.assistantMessageEvent.type === "text_delta") { + process.stdout.write(event.assistantMessageEvent.delta); + } + break; + } + } +}); + +const topic = "Latest developments in quantum error correction 2026"; +console.log(`Researching: ${topic}\n`); + +const result = await agent.research(topic); + +console.log("\n\n" + "=".repeat(60)); +console.log(`Research complete!`); +console.log(`Turns used: ${result.turnsUsed}`); +console.log(`Notes collected: ${result.notes.length}`); +console.log("=".repeat(60) + "\n"); + +console.log(result.report); diff --git a/packages/research-agent/examples/custom-model.ts b/packages/research-agent/examples/custom-model.ts new file mode 100644 index 00000000..332153fb --- /dev/null +++ b/packages/research-agent/examples/custom-model.ts @@ -0,0 +1,40 @@ +/** + * Custom Model Example + * + * Demonstrates how to use a custom model with the ResearchAgent. + * + * Run with: + * TAVILY_API_KEY=*** \ + * OPENROUTER_API_KEY=*** \ + * npx tsx examples/custom-model.ts + */ + +import { getModels } from "@earendil-works/pi-ai"; +import { ResearchAgent } from "../src/index.ts"; + +const models = getModels("openrouter"); +const gpt4o = models.find((m) => m.id === "openai/gpt-4o"); + +if (!gpt4o) { + console.error("GPT-4o model not found"); + process.exit(1); +} + +const agent = new ResearchAgent({ + model: gpt4o, + thinkingLevel: "high", + maxTurns: 15, +}); + +agent.subscribe((event) => { + if (event.type === "message_update" && event.assistantMessageEvent.type === "text_delta") { + process.stdout.write(event.assistantMessageEvent.delta); + } +}); + +const topic = "Comparison of renewable energy storage solutions"; +console.log(`Researching: ${topic}\n`); + +const result = await agent.research(topic); + +console.log("\n\n" + result.report); diff --git a/packages/research-agent/examples/custom-skills.ts b/packages/research-agent/examples/custom-skills.ts new file mode 100644 index 00000000..9775e95d --- /dev/null +++ b/packages/research-agent/examples/custom-skills.ts @@ -0,0 +1,53 @@ +/** + * Custom Skills Example + * + * Demonstrates how to use custom research skills with the ResearchAgent. + * + * Run with: + * TAVILY_API_KEY=*** \ + * OPENROUTER_API_KEY=*** \ + * npx tsx examples/custom-skills.ts + */ + +import type { ResearchSkill } from "../src/index.ts"; +import { ResearchAgent } from "../src/index.ts"; + +const academicResearchSkill: ResearchSkill = { + name: "academic-research", + description: "Focus on academic sources and peer-reviewed papers", + content: `When conducting research: +1. Prioritize academic sources (Google Scholar, arXiv, PubMed) +2. Look for peer-reviewed papers and citations +3. Check publication dates and author credentials +4. Note methodology and sample sizes +5. Distinguish between correlation and causation`, + maxTurns: 15, +}; + +const factCheckingSkill: ResearchSkill = { + name: "fact-checking", + description: "Verify claims with multiple sources", + content: `For each major claim: +1. Find at least 3 independent sources +2. Check for conflicting information +3. Note the confidence level appropriately +4. Flag any claims that cannot be verified`, +}; + +const agent = new ResearchAgent({ + maxTurns: 12, + skills: [academicResearchSkill, factCheckingSkill], +}); + +agent.subscribe((event) => { + if (event.type === "message_update" && event.assistantMessageEvent.type === "text_delta") { + process.stdout.write(event.assistantMessageEvent.delta); + } +}); + +const topic = "Effectiveness of cognitive behavioral therapy for anxiety"; +console.log(`Researching: ${topic}\n`); + +const result = await agent.research(topic); + +console.log("\n\n" + result.report); diff --git a/packages/research-agent/examples/mcp-client.ts b/packages/research-agent/examples/mcp-client.ts new file mode 100644 index 00000000..c877f07b --- /dev/null +++ b/packages/research-agent/examples/mcp-client.ts @@ -0,0 +1,59 @@ +/** + * MCP Client Example + * + * Demonstrates how to call the research agent MCP server from another agent. + * + * Run with: + * TAVILY_API_KEY=*** \ + * OPENROUTER_API_KEY=*** \ + * npx tsx examples/mcp-client.ts + */ + +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; +import { join } from "node:path"; + +const mcpServerPath = join(import.meta.dirname, "../dist/mcp-cli.js"); + +const transport = new StdioClientTransport({ + command: "node", + args: [mcpServerPath], + env: { + ...process.env, + }, +}); + +const client = new Client({ + name: "example-client", + version: "1.0.0", +}); + +await client.connect(transport); + +console.log("Connected to research agent MCP server\n"); + +const toolsResult = await client.listTools(); +console.log("Available tools:"); +for (const tool of toolsResult.tools) { + console.log(` - ${tool.name}: ${tool.description}`); +} +console.log(); + +console.log("Starting research on 'Rust async runtime'...\n"); + +const result = await client.callTool({ + name: "quick_search", + arguments: { + query: "Rust async runtime tokio vs async-std 2026", + maxResults: 5, + }, +}); + +console.log("Search results:"); +for (const content of result.content) { + if (content.type === "text") { + console.log(content.text); + } +} + +await client.close(); diff --git a/packages/research-agent/examples/mcp-config-claude.json b/packages/research-agent/examples/mcp-config-claude.json new file mode 100644 index 00000000..325abec8 --- /dev/null +++ b/packages/research-agent/examples/mcp-config-claude.json @@ -0,0 +1,13 @@ +{ + "mcpServers": { + "research-agent": { + "command": "node", + "args": ["./packages/research-agent/dist/mcp-cli.js"], + "env": { + "TAVILY_API_KEY": "***", + "OPENROUTER_API_KEY": "***", + "LLM_MODEL": "minimax/minimax-m3" + } + } + } +} diff --git a/packages/research-agent/examples/mcp-config-cursor.json b/packages/research-agent/examples/mcp-config-cursor.json new file mode 100644 index 00000000..22220dec --- /dev/null +++ b/packages/research-agent/examples/mcp-config-cursor.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "research-agent": { + "command": "node", + "args": ["./node_modules/@earendil-works/pi-research-agent/dist/mcp-cli.js"] + } + } +} diff --git a/packages/research-agent/examples/with-env.ts b/packages/research-agent/examples/with-env.ts new file mode 100644 index 00000000..38e3a1a3 --- /dev/null +++ b/packages/research-agent/examples/with-env.ts @@ -0,0 +1,47 @@ +/** + * Environment Variables Example + * + * This example demonstrates how to use .env files with the research agent. + * + * Setup: + * 1. Copy .env.example to .env: + * cp packages/research-agent/.env.example packages/research-agent/.env + * + * 2. Edit .env with your API keys: + * TAVILY_API_KEY=*** + * OPENROUTER_API_KEY=*** + * + * 3. Run the example: + * npx tsx examples/with-env.ts + * + * The agent will automatically load variables from .env file. + * Environment variables take precedence over .env file values. + */ + +import { ResearchAgent } from "../src/index.ts"; + +// No need to manually read process.env - the CLI does this automatically +// But when using the programmatic API, you can still pass keys explicitly +const agent = new ResearchAgent({ + maxTurns: 5, + // Keys will be read from process.env (loaded from .env by CLI) + // Or you can pass them explicitly: + // tavilyApiKey: process.env.TAVILY_API_KEY, + // llmApiKey: process.env.OPENROUTER_API_KEY, +}); + +agent.subscribe((event) => { + if (event.type === "message_update" && event.assistantMessageEvent.type === "text_delta") { + process.stdout.write(event.assistantMessageEvent.delta); + } +}); + +console.log("Starting research with environment variables from .env file...\n"); + +const result = await agent.research("Benefits of morning exercise"); + +console.log("\n\n" + "=".repeat(60)); +console.log(`Research complete!`); +console.log(`Turns used: ${result.turnsUsed}`); +console.log(`Notes collected: ${result.notes.length}`); +console.log("=".repeat(60)); diff --git a/packages/research-agent/package.json b/packages/research-agent/package.json new file mode 100644 index 00000000..bc08b91f --- /dev/null +++ b/packages/research-agent/package.json @@ -0,0 +1,77 @@ +{ + "name": "@earendil-works/pi-research-agent", + "version": "0.78.0", + "description": "Autonomous research agent with web search, source analysis, and report synthesis. MCP-compatible.", + "type": "module", + "bin": { + "pi-research": "dist/cli.js", + "pi-research-mcp": "dist/mcp-cli.js" + }, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "files": [ + "dist", + "README.md", + "LICENSE", + "CHANGELOG.md" + ], + "scripts": { + "clean": "shx rm -rf dist", + "build": "tsgo -p tsconfig.build.json && shx chmod +x dist/cli.js dist/mcp-cli.js", + "test": "vitest --run", + "prepublishOnly": "npm run clean && npm run build" + }, + "dependencies": { + "@earendil-works/pi-agent-core": "^0.78.0", + "@earendil-works/pi-ai": "^0.78.0", + "@earendil-works/pi-tui": "^0.78.0", + "@modelcontextprotocol/sdk": "1.29.0", + "chalk": "5.6.2", + "typebox": "1.1.38", + "undici": "8.3.0", + "yaml": "2.9.0", + "zod": "3.25.67" + }, + "devDependencies": { + "@types/node": "24.12.4", + "shx": "0.4.0", + "typescript": "5.9.3", + "vitest": "3.2.4" + }, + "keywords": [ + "research-agent", + "ai", + "llm", + "web-search", + "autonomous", + "agent", + "mcp", + "model-context-protocol", + "deep-research", + "report-generation", + "tavily" + ], + "author": { + "name": "Pi Research Team", + "url": "https://github.com/earendil-works/pi" + }, + "license": "MIT", + "homepage": "https://github.com/earendil-works/pi/tree/main/packages/research-agent#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/earendil-works/pi.git", + "directory": "packages/research-agent" + }, + "bugs": { + "url": "https://github.com/earendil-works/pi/issues" + }, + "engines": { + "node": ">=22.19.0" + } +} diff --git a/packages/research-agent/src/agent.ts b/packages/research-agent/src/agent.ts new file mode 100644 index 00000000..3846fbd7 --- /dev/null +++ b/packages/research-agent/src/agent.ts @@ -0,0 +1,404 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { Agent, type AgentEvent, type AgentMessage } from "@earendil-works/pi-agent-core"; +import { type Api, getModels, type Message, type Model, streamSimple } from "@earendil-works/pi-ai"; +import { + DEFAULT_MAX_TURNS, + DEFAULT_MODEL_ID, + DEFAULT_OUTPUT_DIR, + DEFAULT_PROVIDER, + ENV_LLM_MODEL, + ENV_OPENROUTER_API_KEY, + ENV_TAVILY_API_KEY, + MIN_REPORT_LENGTH, +} from "./config.ts"; +import { buildSystemPrompt } from "./system-prompt.ts"; +import { createNoteStore, createResearchTools } from "./tools/index.ts"; +import type { NoteStore, ResearchAgentOptions, ResearchResult } from "./types.ts"; + +export class ResearchAgent { + private agent: Agent; + private notes: NoteStore; + private turnCount = 0; + private maxTurns: number; + private report = ""; + private forcedSynthesis = false; + private synthesisRequested = false; + private reportGenerationStopped = false; + private turnsAfterSynthesis = 0; + private topic = ""; + private outputDir: string; + private saveResults: boolean; + + constructor(options: ResearchAgentOptions = {}) { + this.notes = createNoteStore(); + this.maxTurns = options.maxTurns ?? DEFAULT_MAX_TURNS; + this.outputDir = options.outputDir ?? DEFAULT_OUTPUT_DIR; + this.saveResults = options.saveResults ?? true; + + const tavilyApiKey = options.tavilyApiKey ?? process.env[ENV_TAVILY_API_KEY]; + if (!tavilyApiKey) { + throw new Error( + `Tavily API key is required. Set ${ENV_TAVILY_API_KEY} environment variable or pass tavilyApiKey option.`, + ); + } + + const llmApiKey = options.llmApiKey ?? process.env[ENV_OPENROUTER_API_KEY]; + if (!llmApiKey) { + throw new Error( + `LLM API key is required. Set ${ENV_OPENROUTER_API_KEY} environment variable or pass llmApiKey option.`, + ); + } + + const model = options.model ?? this.resolveModel(); + const tools = createResearchTools(this.notes, tavilyApiKey); + const systemPrompt = buildSystemPrompt(options.skills); + + this.agent = new Agent({ + initialState: { + systemPrompt, + model, + thinkingLevel: options.thinkingLevel ?? "medium", + tools, + }, + streamFn: async (streamModel, context, opts) => { + return streamSimple(streamModel, context, { + ...opts, + apiKey: llmApiKey, + }); + }, + convertToLlm: (messages: AgentMessage[]): Message[] => { + return messages.filter( + (m) => m.role === "user" || m.role === "assistant" || m.role === "toolResult", + ) as Message[]; + }, + }); + + this.agent.subscribe((event: AgentEvent) => { + // Log assistant message generation for debugging + if (event.type === "message_end" && event.message.role === "assistant") { + const textContent = event.message.content.find((c) => c.type === "text"); + if (textContent && textContent.type === "text" && textContent.text.length > 0) { + console.error(`[Assistant message generated, length: ${textContent.text.length}]`); + } + } + + // Track when synthesize_report tool completes successfully + if (event.type === "tool_execution_end" && event.toolName === "synthesize_report") { + if (event.result?.details && event.result.details.synthesisRequested === true) { + this.synthesisRequested = true; + this.turnsAfterSynthesis = 0; + console.error(`\n[synthesize_report called, waiting for report generation...]`); + } + } + + if (event.type === "turn_end") { + this.turnCount++; + + // After synthesize_report was called, track turns and check for report + if (this.synthesisRequested && !this.reportGenerationStopped) { + this.turnsAfterSynthesis++; + + const lastMsg = this.getLastAssistantMessage(); + const reportLength = lastMsg ? this.getTextLength(lastMsg) : 0; + const hasReport = reportLength >= MIN_REPORT_LENGTH; + + // Stop after 2 turns OR if we have a substantial report after at least 1 turn + if (this.turnsAfterSynthesis >= 2 || (hasReport && this.turnsAfterSynthesis >= 1)) { + this.reportGenerationStopped = true; + console.error(`\n[Report generation complete. Stopping agent.]`); + this.agent.abort(); + return; + } + } + + // When reaching max turns, inject a steering message to force synthesis + if (this.turnCount >= this.maxTurns && !this.forcedSynthesis && !this.synthesisRequested) { + this.forcedSynthesis = true; + console.error(`\n[Max turns (${this.maxTurns}) reached. Forcing report synthesis.]`); + this.agent.steer({ + role: "user", + content: [ + { + type: "text", + text: `You have reached the maximum number of research turns (${this.maxTurns}). You MUST now call the synthesize_report tool immediately with all the notes you have collected. Do not search or read more - synthesize what you have now.`, + }, + ], + timestamp: Date.now(), + }); + } + } + }); + } + + private resolveModel(): Model { + const modelId = process.env[ENV_LLM_MODEL] ?? DEFAULT_MODEL_ID; + const models = getModels(DEFAULT_PROVIDER); + const model = models.find((m) => m.id === modelId); + if (!model) { + throw new Error(`Model not found: ${DEFAULT_PROVIDER}/${modelId}`); + } + return model; + } + + subscribe(listener: (event: AgentEvent, signal: AbortSignal) => Promise | void): () => void { + return this.agent.subscribe(listener); + } + + get state() { + return this.agent.state; + } + + getNoteCount(): number { + return this.notes.count(); + } + + getTurnCount(): number { + return this.turnCount; + } + + async research(topic: string): Promise { + this.turnCount = 0; + this.notes.clear(); + this.report = ""; + this.forcedSynthesis = false; + this.synthesisRequested = false; + this.reportGenerationStopped = false; + this.turnsAfterSynthesis = 0; + this.topic = topic; + + await this.agent.prompt(topic); + + this.extractReport(); + + // Fallback: if no report was generated, create one from notes + if (!this.report || this.report.trim() === "") { + this.report = this.generateFallbackReport(); + } + + const result: ResearchResult = { + topic, + notes: this.notes.getAll(), + report: this.report, + turnsUsed: this.turnCount, + }; + + if (this.saveResults) { + await this.saveToDisk(result); + } + + return result; + } + + private getLastAssistantMessage(): AgentMessage | undefined { + const messages = this.agent.state.messages; + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i]; + if (msg.role === "assistant" && "content" in msg) { + return msg; + } + } + return undefined; + } + + private getTextLength(msg: AgentMessage): number { + if ("content" in msg && msg.content) { + const textContent = (msg.content as Array<{ type: string; text?: string }>).find((c) => c.type === "text"); + if (textContent?.text) { + return textContent.text.length; + } + } + return 0; + } + + private extractReport(): void { + const messages = this.agent.state.messages; + + console.error(`\n[Extracting report from ${messages.length} messages...]`); + + // Strategy 1: Look for assistant message AFTER synthesize_report tool result (correct flow) + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i]; + + if (msg.role === "toolResult") { + const content = msg.content; + if (Array.isArray(content)) { + const textContent = content.find((c: any) => c.type === "text"); + if ( + textContent && + textContent.type === "text" && + textContent.text.includes("Research synthesis requested for") + ) { + console.error(`[Found synthesize_report at index ${i}]`); + // Found synthesize_report result, look for assistant message after it + for (let j = i + 1; j < messages.length; j++) { + const nextMsg = messages[j]; + if (nextMsg.role === "assistant" && nextMsg.content) { + const nextTextContent = nextMsg.content.find((c) => c.type === "text"); + if (nextTextContent && nextTextContent.type === "text" && nextTextContent.text.length > 200) { + console.error( + `[Extracted report from assistant message at index ${j}, length: ${nextTextContent.text.length}]`, + ); + this.report = nextTextContent.text; + return; + } + } + } + } + } + } + } + + console.error(`[No report found after synthesize_report, trying fallback strategies...]`); + + // Strategy 2: Look for the longest assistant message in the last 5 messages (fallback) + const lastMessages = messages.slice(-5); + let longestText = ""; + for (const msg of lastMessages) { + if (msg.role === "assistant" && msg.content) { + const textContent = msg.content.find((c) => c.type === "text"); + if (textContent && textContent.type === "text" && textContent.text.length > longestText.length) { + longestText = textContent.text; + } + } + } + + if (longestText.length > 200) { + console.error(`[Extracted longest report from last 5 messages, length: ${longestText.length}]`); + this.report = longestText; + return; + } + + console.error(`[No substantial report found in last 5 messages, trying any long message...]`); + + // Strategy 3: Look for any assistant message with substantial text + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i]; + if (msg.role === "assistant" && msg.content) { + const textContent = msg.content.find((c) => c.type === "text"); + if (textContent && textContent.type === "text" && textContent.text.length > 100) { + console.error( + `[Extracted fallback report from assistant message at index ${i}, length: ${textContent.text.length}]`, + ); + this.report = textContent.text; + return; + } + } + } + + console.error(`[No report found in messages, trying longest message from all...]`); + + // Strategy 4: Find the longest assistant message in ALL messages + let longestMsg = ""; + let longestIndex = -1; + for (let i = 0; i < messages.length; i++) { + const msg = messages[i]; + if (msg.role === "assistant" && msg.content) { + const textContent = msg.content.find((c) => c.type === "text"); + if (textContent && textContent.type === "text" && textContent.text.length > longestMsg.length) { + longestMsg = textContent.text; + longestIndex = i; + } + } + } + + if (longestMsg.length > MIN_REPORT_LENGTH) { + console.error( + `[Extracted longest report from all messages at index ${longestIndex}, length: ${longestMsg.length}]`, + ); + this.report = longestMsg; + return; + } + + console.error(`[No report found in messages]`); + this.report = ""; + } + + private generateFallbackReport(): string { + const notes = this.notes.getAll(); + if (notes.length === 0) { + return "Research completed but no notes were collected."; + } + + const lines = [ + `# Research Report: ${this.topic}`, + "", + "## Executive Summary", + "", + `This research collected ${notes.length} findings on "${this.topic}".`, + "", + "## Key Findings", + "", + ]; + + // Group notes by confidence + const highConfidence = notes.filter((n) => n.confidence === "high"); + const mediumConfidence = notes.filter((n) => n.confidence === "medium"); + const lowConfidence = notes.filter((n) => n.confidence === "low"); + + if (highConfidence.length > 0) { + lines.push("### High Confidence Findings"); + lines.push(""); + for (const note of highConfidence) { + lines.push(`- **${note.topic}**: ${note.finding}`); + lines.push(` - Source: ${note.source_url}`); + } + lines.push(""); + } + + if (mediumConfidence.length > 0) { + lines.push("### Medium Confidence Findings"); + lines.push(""); + for (const note of mediumConfidence) { + lines.push(`- **${note.topic}**: ${note.finding}`); + lines.push(` - Source: ${note.source_url}`); + } + lines.push(""); + } + + if (lowConfidence.length > 0) { + lines.push("### Low Confidence Findings"); + lines.push(""); + for (const note of lowConfidence) { + lines.push(`- **${note.topic}**: ${note.finding}`); + lines.push(` - Source: ${note.source_url}`); + } + lines.push(""); + } + + lines.push("## Notes"); + lines.push(""); + lines.push( + "*This report was auto-generated from collected notes as the agent reached the turn limit before synthesizing a full report.*", + ); + + return lines.join("\n"); + } + + private async saveToDisk(result: ResearchResult): Promise { + try { + const timestamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19); + const slug = result.topic + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-|-$/g, "") + .slice(0, 50); + const dir = join(this.outputDir, `${timestamp}-${slug}`); + + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, "report.md"), result.report, "utf-8"); + await writeFile(join(dir, "report.json"), JSON.stringify(result, null, 2), "utf-8"); + await writeFile(join(dir, "notes.json"), JSON.stringify(result.notes, null, 2), "utf-8"); + + console.error(`\n[Results saved to ${dir}]`); + } catch (error) { + console.error( + `\n[Warning: Failed to save results to disk: ${error instanceof Error ? error.message : String(error)}]`, + ); + } + } + + abort(): void { + this.agent.abort(); + } +} diff --git a/packages/research-agent/src/cli.ts b/packages/research-agent/src/cli.ts new file mode 100644 index 00000000..7213d616 --- /dev/null +++ b/packages/research-agent/src/cli.ts @@ -0,0 +1,13 @@ +#!/usr/bin/env node +import { loadEnvFile } from "./utils/env.ts"; + +// Load .env file before anything else +loadEnvFile(); + +import { APP_NAME } from "./config.ts"; +import { main } from "./main.ts"; + +process.title = APP_NAME; +process.emitWarning = (() => {}) as typeof process.emitWarning; + +main(process.argv.slice(2)); diff --git a/packages/research-agent/src/config.ts b/packages/research-agent/src/config.ts new file mode 100644 index 00000000..379eb19f --- /dev/null +++ b/packages/research-agent/src/config.ts @@ -0,0 +1,18 @@ +export const VERSION = "0.78.0"; +export const APP_NAME = "pi-research"; + +export const DEFAULT_MAX_TURNS = 15; +export const DEFAULT_MAX_RESULTS = 10; +export const DEFAULT_MAX_CONTENT_LENGTH = 10000; + +export const ENV_TAVILY_API_KEY = "TAVILY_API_KEY"; +export const ENV_OPENROUTER_API_KEY = "OPENROUTER_API_KEY"; +export const ENV_LLM_MODEL = "LLM_MODEL"; + +export const DEFAULT_MODEL_ID = "minimax/minimax-m3"; +export const DEFAULT_PROVIDER = "openrouter"; + +export const TAVILY_API_URL = "https://api.tavily.com/search"; + +export const DEFAULT_OUTPUT_DIR = "./research-output"; +export const MIN_REPORT_LENGTH = 500; diff --git a/packages/research-agent/src/index.ts b/packages/research-agent/src/index.ts new file mode 100644 index 00000000..f312789d --- /dev/null +++ b/packages/research-agent/src/index.ts @@ -0,0 +1,8 @@ +export { ResearchAgent } from "./agent.ts"; +export { DEFAULT_MODEL_ID, DEFAULT_PROVIDER, VERSION } from "./config.ts"; +export { createResearchMcpServer } from "./mcp-server.ts"; +export { loadResearchSkills } from "./skills/loader.ts"; +export type { ResearchSkill } from "./skills/types.ts"; +export { BASE_SYSTEM_PROMPT, buildSystemPrompt } from "./system-prompt.ts"; +export { createNoteStore, createResearchTools } from "./tools/index.ts"; +export type { NoteStore, ResearchAgentOptions, ResearchNote, ResearchResult } from "./types.ts"; diff --git a/packages/research-agent/src/main.ts b/packages/research-agent/src/main.ts new file mode 100644 index 00000000..7225df01 --- /dev/null +++ b/packages/research-agent/src/main.ts @@ -0,0 +1,141 @@ +import chalk from "chalk"; +import { ResearchAgent } from "./agent.ts"; +import { APP_NAME, VERSION } from "./config.ts"; +import { runInteractiveMode, runPrintMode } from "./modes/index.ts"; +import { loadResearchSkills } from "./skills/loader.ts"; + +interface ParsedArgs { + help: boolean; + version: boolean; + print: boolean; + maxTurns?: number; + model?: string; + topic?: string; + save: boolean; + outputDir?: string; +} + +function parseArgs(args: string[]): ParsedArgs { + const parsed: ParsedArgs = { + help: false, + version: false, + print: false, + save: true, + }; + + const positional: string[] = []; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + + if (arg === "--help" || arg === "-h") { + parsed.help = true; + } else if (arg === "--version" || arg === "-v") { + parsed.version = true; + } else if (arg === "--print" || arg === "-p") { + parsed.print = true; + } else if (arg === "--max-turns" || arg === "-t") { + const value = args[++i]; + if (value) { + parsed.maxTurns = Number.parseInt(value, 10); + } + } else if (arg === "--model" || arg === "-m") { + parsed.model = args[++i]; + } else if (arg === "--no-save") { + parsed.save = false; + } else if (arg === "--output-dir" || arg === "-o") { + parsed.outputDir = args[++i]; + } else if (!arg.startsWith("-")) { + positional.push(arg); + } + } + + if (positional.length > 0) { + parsed.topic = positional.join(" "); + } + + return parsed; +} + +function printHelp(): void { + console.log(` +${chalk.bold(APP_NAME)} - Autonomous Research Agent + +${chalk.bold("Usage:")} + ${APP_NAME} [options] + +${chalk.bold("Options:")} + -h, --help Show this help message + -v, --version Show version number + -p, --print Use print mode (no interactive TUI) + -t, --max-turns Maximum research turns (default: 15) + -m, --model Model to use (default: minimax/minimax-m3) + -o, --output-dir

Output directory for results (default: ./research-output) + --no-save Do not save results to disk + +${chalk.bold("Environment Variables:")} + TAVILY_API_KEY Required - Tavily API key for web search + OPENROUTER_API_KEY Required - OpenRouter API key for LLM + LLM_MODEL Optional - Override default model + +${chalk.bold("Examples:")} + ${APP_NAME} "quantum computing breakthroughs 2026" + ${APP_NAME} --print "AI safety research" + ${APP_NAME} --max-turns 15 "climate change solutions" + ${APP_NAME} -m openai/gpt-4o "renewable energy trends" + ${APP_NAME} --no-save "quick research topic" + ${APP_NAME} -o ./my-reports "saved to custom directory" + +${chalk.bold("Output:")} + Results are automatically saved to /-/ + - report.md - Final research report in markdown + - report.json - Full results (report + notes + metadata) + - notes.json - Collected research notes + +${chalk.bold("Skills:")} + Research skills are loaded from ~/.pi/skills/research/ directory. + Create .md files with YAML frontmatter to customize research behavior. +`); +} + +export async function main(args: string[]): Promise { + const parsed = parseArgs(args); + + if (parsed.help) { + printHelp(); + return; + } + + if (parsed.version) { + console.log(`${APP_NAME} ${VERSION}`); + return; + } + + if (!parsed.topic) { + console.error(chalk.red("Error: Research topic is required.")); + console.error(chalk.dim(`Run '${APP_NAME} --help' for usage information.`)); + process.exit(1); + } + + const skills = await loadResearchSkills(); + + let agent: ResearchAgent; + try { + agent = new ResearchAgent({ + maxTurns: parsed.maxTurns, + skills, + saveResults: parsed.save, + outputDir: parsed.outputDir, + }); + } catch (error) { + console.error(chalk.red(`Error: ${error instanceof Error ? error.message : String(error)}`)); + process.exit(1); + } + + const mode = parsed.print || !process.stdin.isTTY ? "print" : "interactive"; + + const exitCode = + mode === "interactive" ? await runInteractiveMode(agent, parsed.topic) : await runPrintMode(agent, parsed.topic); + + process.exit(exitCode); +} diff --git a/packages/research-agent/src/mcp-cli.ts b/packages/research-agent/src/mcp-cli.ts new file mode 100644 index 00000000..85de3fb5 --- /dev/null +++ b/packages/research-agent/src/mcp-cli.ts @@ -0,0 +1,17 @@ +#!/usr/bin/env node +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import { createResearchMcpServer } from "./mcp-server.ts"; +import { loadEnvFile } from "./utils/env.ts"; + +loadEnvFile(); + +async function main() { + const server = createResearchMcpServer(); + const transport = new StdioServerTransport(); + await server.connect(transport); +} + +main().catch((error) => { + console.error("Fatal error:", error); + process.exit(1); +}); diff --git a/packages/research-agent/src/mcp-server.ts b/packages/research-agent/src/mcp-server.ts new file mode 100644 index 00000000..7cd57693 --- /dev/null +++ b/packages/research-agent/src/mcp-server.ts @@ -0,0 +1,162 @@ +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { z } from "zod"; +import { ResearchAgent } from "./agent.ts"; +import { VERSION } from "./config.ts"; + +const researchSchema = { + topic: z.string().describe("The research topic to investigate"), + maxTurns: z + .number() + .optional() + .default(15) + .describe("Maximum number of research turns (default: 15). More turns = deeper research."), + format: z + .enum(["markdown", "json", "bullet"]) + .optional() + .default("markdown") + .describe("Output format for the report (default: markdown)"), +}; + +const quickSearchSchema = { + query: z.string().describe("Search query"), + maxResults: z.number().optional().default(10).describe("Maximum number of results (default: 10)"), +}; + +const readUrlSchema = { + url: z.string().describe("URL to read"), + maxLength: z.number().optional().default(10000).describe("Maximum characters to extract (default: 10000)"), +}; + +export function createResearchMcpServer(): McpServer { + const server = new McpServer({ + name: "pi-research-agent", + version: VERSION, + }); + + server.registerTool( + "research", + { + description: + "Perform deep autonomous research on a topic. Searches the web, reads sources, saves findings, and generates a comprehensive report with citations.", + inputSchema: researchSchema as any, + }, + async (params: any) => { + const { topic, maxTurns = 15, format = "markdown" } = params; + const agent = new ResearchAgent({ + maxTurns, + saveResults: false, + }); + + // Suppress console output during MCP execution + const originalError = console.error; + console.error = () => {}; + + try { + const result = await agent.research(topic); + + console.error = originalError; + + const reportWithMeta = [ + `# Research Report: ${topic}`, + "", + `**Turns used:** ${result.turnsUsed}`, + `**Notes collected:** ${result.notes.length}`, + `**Format:** ${format}`, + "", + "---", + "", + result.report, + "", + "---", + "", + "## Raw Research Notes", + "", + ...result.notes.flatMap((note) => [ + `### ${note.topic}`, + `- **Confidence:** ${note.confidence}`, + `- **Source:** ${note.source_url}`, + `- **Finding:** ${note.finding}`, + "", + ]), + ].join("\n"); + + return { + content: [{ type: "text" as const, text: reportWithMeta }], + }; + } catch (error) { + console.error = originalError; + const message = error instanceof Error ? error.message : String(error); + return { + content: [{ type: "text" as const, text: `Research failed: ${message}` }], + isError: true, + }; + } + }, + ); + + server.registerTool( + "quick_search", + { + description: + "Quick web search on a topic. Returns search results without full research flow. Use for fast information lookup.", + inputSchema: quickSearchSchema as any, + }, + async (params: any) => { + const { query, maxResults = 10 } = params; + try { + const { tavilySearch } = await import("./utils/tavily.ts"); + const { formatSearchResults } = await import("./utils/formatter.ts"); + + const apiKey = process.env.TAVILY_API_KEY; + if (!apiKey) { + return { + content: [{ type: "text" as const, text: "Error: TAVILY_API_KEY environment variable is not set" }], + isError: true, + }; + } + + const results = await tavilySearch(query, { + apiKey, + maxResults, + }); + + return { + content: [{ type: "text" as const, text: formatSearchResults(results.results) }], + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + content: [{ type: "text" as const, text: `Search failed: ${message}` }], + isError: true, + }; + } + }, + ); + + server.registerTool( + "read_url", + { + description: "Fetch and extract content from a URL. Useful for reading specific articles or pages.", + inputSchema: readUrlSchema as any, + }, + async (params: any) => { + const { url, maxLength = 10000 } = params; + try { + const { fetchAndExtract } = await import("./utils/fetcher.ts"); + const content = await fetchAndExtract(url, maxLength); + + return { + content: [{ type: "text" as const, text: `Content from ${url}:\n\n${content}` }], + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + content: [{ type: "text" as const, text: `Failed to read URL: ${message}` }], + isError: true, + }; + } + }, + ); + + return server; +} diff --git a/packages/research-agent/src/modes/index.ts b/packages/research-agent/src/modes/index.ts new file mode 100644 index 00000000..2fc5b453 --- /dev/null +++ b/packages/research-agent/src/modes/index.ts @@ -0,0 +1,2 @@ +export { runInteractiveMode } from "./interactive.ts"; +export { runPrintMode } from "./print.ts"; diff --git a/packages/research-agent/src/modes/interactive.ts b/packages/research-agent/src/modes/interactive.ts new file mode 100644 index 00000000..8a3d2f3e --- /dev/null +++ b/packages/research-agent/src/modes/interactive.ts @@ -0,0 +1,87 @@ +import chalk from "chalk"; +import type { ResearchAgent } from "../agent.ts"; + +export async function runInteractiveMode(agent: ResearchAgent, topic: string): Promise { + console.log(chalk.bold.cyan("\n╔══════════════════════════════════════════════════════════╗")); + console.log(chalk.bold.cyan("║ Pi Research Agent - Interactive ║")); + console.log(chalk.bold.cyan("╚══════════════════════════════════════════════════════════╝")); + console.log(); + + console.log(chalk.bold(`Topic: ${topic}\n`)); + + let currentPhase = ""; + + agent.subscribe((event) => { + switch (event.type) { + case "turn_start": { + const turnNum = agent.getTurnCount() + 1; + console.log(chalk.bold.yellow(`\n┌─── Turn ${turnNum} ───────────────────────────────────`)); + break; + } + case "tool_execution_start": { + const toolName = event.toolName; + const args = event.args as Record; + if (toolName === "web_search") { + currentPhase = "search"; + console.log(chalk.cyan(` 🔍 Searching: "${args.query}"`)); + } else if (toolName === "read_url") { + currentPhase = "read"; + console.log(chalk.blue(` 📖 Reading: ${args.url}`)); + } else if (toolName === "save_note") { + currentPhase = "note"; + console.log(chalk.green(` 💾 Saving note: "${args.topic}" (${args.confidence} confidence)`)); + } else if (toolName === "synthesize_report") { + currentPhase = "synthesize"; + console.log(chalk.magenta(" 📊 Synthesizing final report...")); + } + break; + } + case "tool_execution_end": { + if (event.isError) { + console.log(chalk.red(` ⚠️ Error: ${event.toolName}`)); + } + break; + } + case "turn_end": { + console.log(chalk.dim(` └── Notes: ${agent.getNoteCount()} | Turn ${agent.getTurnCount()} complete`)); + break; + } + case "message_start": { + if (currentPhase !== "synthesize") { + console.log(chalk.dim(" 🤔 Thinking...")); + } + break; + } + case "message_update": { + if (event.assistantMessageEvent.type === "text_delta") { + process.stdout.write(chalk.white(event.assistantMessageEvent.delta)); + } + break; + } + case "message_end": { + process.stdout.write("\n"); + break; + } + } + }); + + try { + const result = await agent.research(topic); + + console.log(chalk.bold.green("\n\n╔══════════════════════════════════════════════════════════╗")); + console.log(chalk.bold.green("║ Research Complete! ║")); + console.log(chalk.bold.green("╚══════════════════════════════════════════════════════════╝")); + console.log(); + + console.log(chalk.dim(` Turns used: ${result.turnsUsed}`)); + console.log(chalk.dim(` Notes collected: ${result.notes.length}`)); + console.log(chalk.dim(`\n${"─".repeat(60)}\n`)); + + console.log(result.report); + + return 0; + } catch (error) { + console.error(chalk.red(`\n\nResearch failed: ${error instanceof Error ? error.message : String(error)}`)); + return 1; + } +} diff --git a/packages/research-agent/src/modes/print.ts b/packages/research-agent/src/modes/print.ts new file mode 100644 index 00000000..333f51cd --- /dev/null +++ b/packages/research-agent/src/modes/print.ts @@ -0,0 +1,55 @@ +import chalk from "chalk"; +import type { ResearchAgent } from "../agent.ts"; + +export async function runPrintMode(agent: ResearchAgent, topic: string): Promise { + console.log(chalk.bold(`\nResearching: ${topic}\n`)); + console.log(chalk.dim("─".repeat(60))); + + agent.subscribe((event) => { + switch (event.type) { + case "tool_execution_start": { + const toolName = event.toolName; + const args = event.args as Record; + if (toolName === "web_search") { + console.error(chalk.cyan(`\n[Searching: ${args.query}]`)); + } else if (toolName === "read_url") { + console.error(chalk.blue(`\n[Reading: ${args.url}]`)); + } else if (toolName === "save_note") { + console.error(chalk.green(`\n[Saving note: ${args.topic}]`)); + } else if (toolName === "synthesize_report") { + console.error(chalk.magenta("\n[Synthesizing report...]")); + } + break; + } + case "turn_end": { + console.error( + chalk.dim(`\n[Turn ${agent.getTurnCount()} complete, ${agent.getNoteCount()} notes collected]`), + ); + break; + } + case "message_update": { + if (event.assistantMessageEvent.type === "text_delta") { + process.stdout.write(event.assistantMessageEvent.delta); + } + break; + } + } + }); + + try { + const result = await agent.research(topic); + + console.log("\n"); + console.log(chalk.dim("─".repeat(60))); + console.log(chalk.bold.green("\nResearch Complete!")); + console.log(chalk.dim(`Turns used: ${result.turnsUsed}`)); + console.log(chalk.dim(`Notes collected: ${result.notes.length}`)); + console.log(chalk.dim("─".repeat(60))); + console.log(`\n${result.report}`); + + return 0; + } catch (error) { + console.error(chalk.red(`\nResearch failed: ${error instanceof Error ? error.message : String(error)}`)); + return 1; + } +} diff --git a/packages/research-agent/src/skills/loader.ts b/packages/research-agent/src/skills/loader.ts new file mode 100644 index 00000000..3e1c5e5c --- /dev/null +++ b/packages/research-agent/src/skills/loader.ts @@ -0,0 +1,57 @@ +import { existsSync } from "node:fs"; +import { readdir, readFile } from "node:fs/promises"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { parse as parseYaml } from "yaml"; +import type { ResearchSkill } from "./types.ts"; + +export type { ResearchSkill }; + +export async function loadResearchSkills(skillsDir?: string): Promise { + const dir = skillsDir ?? join(homedir(), ".pi", "skills", "research"); + + if (!existsSync(dir)) { + return []; + } + + let files: string[]; + try { + files = await readdir(dir); + } catch { + return []; + } + + const skills: ResearchSkill[] = []; + + for (const file of files) { + if (!file.endsWith(".md")) continue; + + try { + const content = await readFile(join(dir, file), "utf-8"); + const parsed = parseSkillFrontmatter(content); + if (parsed) { + skills.push(parsed); + } + } catch {} + } + + return skills; +} + +function parseSkillFrontmatter(content: string): ResearchSkill | null { + const match = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/); + if (!match) return null; + + try { + const frontmatter = parseYaml(match[1]) as Record; + return { + name: (frontmatter.name as string) ?? "unnamed", + description: (frontmatter.description as string) ?? "", + content: match[2].trim(), + maxTurns: frontmatter.max_turns as number | undefined, + tools: frontmatter.tools as string[] | undefined, + }; + } catch { + return null; + } +} diff --git a/packages/research-agent/src/skills/types.ts b/packages/research-agent/src/skills/types.ts new file mode 100644 index 00000000..29923391 --- /dev/null +++ b/packages/research-agent/src/skills/types.ts @@ -0,0 +1,7 @@ +export interface ResearchSkill { + name: string; + description: string; + content: string; + maxTurns?: number; + tools?: string[]; +} diff --git a/packages/research-agent/src/system-prompt.ts b/packages/research-agent/src/system-prompt.ts new file mode 100644 index 00000000..99a4349d --- /dev/null +++ b/packages/research-agent/src/system-prompt.ts @@ -0,0 +1,73 @@ +export const BASE_SYSTEM_PROMPT = `You are an autonomous research agent. Your task is to thoroughly research topics by searching the web, reading sources, and synthesizing findings into comprehensive reports. + +## Research Methodology + +1. **Break down the topic** into key sub-questions that need answering +2. **Search systematically** using web_search for each sub-question +3. **Read relevant sources** using read_url to get detailed information from promising URLs +4. **Save important findings** using save_note with appropriate confidence levels +5. **Identify gaps** in your knowledge and search for additional information +6. **Cross-reference** findings from multiple sources to verify accuracy +7. **Synthesize** when you have enough information using synthesize_report + +## Available Tools + +- **web_search**: Search the web using Tavily. Use specific, targeted queries. +- **read_url**: Fetch and read content from a URL. Use for sources found in search results. +- **save_note**: Save a research finding with topic, content, source URL, and confidence level. +- **synthesize_report**: Prepare all collected notes for report generation. Call this tool when you have gathered sufficient information. After calling this tool, you MUST generate the final comprehensive report in your next response. + +## Guidelines + +- Start with 2-3 broad searches to understand the topic landscape +- Then narrow down with specific queries for each sub-question +- Read at least 3-5 primary sources for comprehensive coverage +- **CRITICAL**: You MUST call save_note for every significant finding from each source you read +- Save at least 5-10 notes before calling synthesize_report +- Use confidence levels appropriately: + - **high**: well-established facts from authoritative sources (academic papers, official reports) + - **medium**: recent information or facts from reputable but less authoritative sources + - **low**: speculative claims, single-source information, or unverified data +- **IMPORTANT**: Do NOT call synthesize_report until you have saved at least 5 notes with save_note +- Call synthesize_report only after you have gathered sufficient information and saved multiple notes (typically after 5-10 searches and reading key sources) + +## Report Generation Process + +**CRITICAL**: Follow this exact sequence: + +1. **Research phase**: Search, read, and save notes +2. **Call synthesize_report**: When ready, call the tool +3. **Generate report IMMEDIATELY**: In your NEXT response, generate the full report + - Do NOT call any more tools + - Do NOT search or read more sources + - Generate the complete markdown report (2000+ words) + - Include executive summary, findings, citations, confidence, gaps + +**WRONG**: Calling synthesize_report, then searching more +**RIGHT**: Calling synthesize_report, then generating the report immediately + +The synthesize_report tool will return all your saved notes in a structured format. Use this information to write your final report. + +## Output Format + +Your final report should include: +- Executive summary (2-3 paragraphs) +- Key findings organized by theme +- Source citations with URLs +- Confidence assessment of the overall research +- Identified gaps or areas for further research + +Be thorough but efficient. Do not repeat searches for the same information.`; + +export function buildSystemPrompt(skills?: Array<{ name: string; content: string }>): string { + let prompt = BASE_SYSTEM_PROMPT; + + if (skills && skills.length > 0) { + prompt += "\n\n## Additional Research Skills\n\n"; + for (const skill of skills) { + prompt += `### ${skill.name}\n${skill.content}\n\n`; + } + } + + return prompt; +} diff --git a/packages/research-agent/src/tools/index.ts b/packages/research-agent/src/tools/index.ts new file mode 100644 index 00000000..935c9f88 --- /dev/null +++ b/packages/research-agent/src/tools/index.ts @@ -0,0 +1,18 @@ +import type { AgentTool } from "@earendil-works/pi-agent-core"; +import type { NoteStore } from "../types.ts"; +import { createReadUrlTool } from "./read-url.ts"; +import { createSaveNoteTool } from "./save-note.ts"; +import { createSynthesizeTool } from "./synthesize.ts"; +import { createWebSearchTool } from "./web-search.ts"; + +export type { NoteStore, ResearchNote } from "../types.ts"; +export { createNoteStore } from "./save-note.ts"; + +export function createResearchTools(notes: NoteStore, tavilyApiKey: string): AgentTool[] { + return [ + createWebSearchTool(tavilyApiKey), + createReadUrlTool(), + createSaveNoteTool(notes), + createSynthesizeTool(notes), + ]; +} diff --git a/packages/research-agent/src/tools/read-url.ts b/packages/research-agent/src/tools/read-url.ts new file mode 100644 index 00000000..6c92f995 --- /dev/null +++ b/packages/research-agent/src/tools/read-url.ts @@ -0,0 +1,60 @@ +import type { AgentTool } from "@earendil-works/pi-agent-core"; +import { type Static, Type } from "typebox"; +import { DEFAULT_MAX_CONTENT_LENGTH } from "../config.ts"; +import { fetchAndExtract } from "../utils/fetcher.ts"; + +const readUrlSchema = Type.Object({ + url: Type.String({ description: "URL of the web page to read" }), + max_length: Type.Optional( + Type.Number({ + description: `Maximum characters to extract (default: ${DEFAULT_MAX_CONTENT_LENGTH})`, + }), + ), +}); + +export type ReadUrlInput = Static; + +interface ReadUrlDetails { + url: string; + length: number; +} + +export function createReadUrlTool(): AgentTool { + return { + name: "read_url", + label: "Read URL", + description: "Fetch and read the content of a web page. Extracts main text content from HTML pages.", + parameters: readUrlSchema, + execute: async (_toolCallId, params: ReadUrlInput) => { + const maxLength = params.max_length ?? DEFAULT_MAX_CONTENT_LENGTH; + + // Try with default User-Agent, then retry with alternative if it fails + let content: string; + let lastError: Error | null = null; + + try { + content = await fetchAndExtract(params.url, maxLength); + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + + // Retry with alternative User-Agent + try { + await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait 1 second + content = await fetchAndExtract(params.url, maxLength, true); // true = use alternative UA + } catch (_retryError) { + throw new Error(`Failed to fetch ${params.url} after 2 attempts: ${lastError.message}`); + } + } + + return { + content: [ + { + type: "text" as const, + text: `Content from ${params.url}:\n\n${content}`, + }, + ], + details: { url: params.url, length: content.length }, + }; + }, + }; +} diff --git a/packages/research-agent/src/tools/save-note.ts b/packages/research-agent/src/tools/save-note.ts new file mode 100644 index 00000000..2d1cd5fc --- /dev/null +++ b/packages/research-agent/src/tools/save-note.ts @@ -0,0 +1,61 @@ +import type { AgentTool } from "@earendil-works/pi-agent-core"; +import { type Static, Type } from "typebox"; +import type { NoteStore, ResearchNote } from "../types.ts"; + +const saveNoteSchema = Type.Object({ + topic: Type.String({ description: "Topic or sub-question this note addresses" }), + finding: Type.String({ description: "The key finding or information extracted" }), + source_url: Type.String({ description: "URL of the source where this finding came from" }), + confidence: Type.Union([Type.Literal("high"), Type.Literal("medium"), Type.Literal("low")], { + description: + "Confidence level: high for well-established facts, medium for recent info, low for speculative claims", + }), +}); + +export type SaveNoteInput = Static; + +export function createNoteStore(): NoteStore { + const notes: ResearchNote[] = []; + return { + add(note: ResearchNote) { + notes.push(note); + }, + getAll() { + return notes.slice(); + }, + clear() { + notes.length = 0; + }, + count() { + return notes.length; + }, + }; +} + +export function createSaveNoteTool(notes: NoteStore): AgentTool { + return { + name: "save_note", + label: "Save Note", + description: "Save a research note or finding for later synthesis into the final report", + parameters: saveNoteSchema, + execute: async (_toolCallId, params: SaveNoteInput) => { + const note: ResearchNote = { + topic: params.topic, + finding: params.finding, + source_url: params.source_url, + confidence: params.confidence, + timestamp: Date.now(), + }; + notes.add(note); + return { + content: [ + { + type: "text" as const, + text: `Note saved successfully. Topic: "${params.topic}". Total notes collected: ${notes.count()}`, + }, + ], + details: note, + }; + }, + }; +} diff --git a/packages/research-agent/src/tools/synthesize.ts b/packages/research-agent/src/tools/synthesize.ts new file mode 100644 index 00000000..d3a80799 --- /dev/null +++ b/packages/research-agent/src/tools/synthesize.ts @@ -0,0 +1,117 @@ +import type { AgentTool } from "@earendil-works/pi-agent-core"; +import { type Static, Type } from "typebox"; +import type { NoteStore } from "../types.ts"; +import { formatNotes } from "../utils/formatter.ts"; + +const synthesizeSchema = Type.Object({ + topic: Type.String({ description: "The main research topic to synthesize" }), + format: Type.Optional( + Type.Union([Type.Literal("markdown"), Type.Literal("json"), Type.Literal("bullet")], { + description: "Output format for the report (default: markdown)", + }), + ), +}); + +export type SynthesizeInput = Static; + +interface SynthesizeDetails { + notes_count: number; + topic: string; + format: string; + synthesisRequested: boolean; +} + +export function createSynthesizeTool(notes: NoteStore): AgentTool { + return { + name: "synthesize_report", + label: "Synthesize Report", + description: + "Synthesize all collected research notes into a structured report. Call this when you have gathered sufficient information to produce the final research report.", + parameters: synthesizeSchema, + execute: async (_toolCallId, params: SynthesizeInput) => { + const allNotes = notes.getAll(); + const format = params.format ?? "markdown"; + + // Warn if too few notes were collected + if (allNotes.length === 0) { + const warning = [ + `WARNING: No research notes were collected for topic "${params.topic}".`, + "", + "You must use the save_note tool to save findings from your research before calling synthesize_report.", + "Please go back and:", + "1. Read sources using read_url", + "2. Save key findings using save_note with topic, finding, source_url, and confidence", + "3. Call synthesize_report again after saving at least 5 notes", + ].join("\n"); + + return { + content: [{ type: "text" as const, text: warning }], + details: { + notes_count: 0, + topic: params.topic, + format, + synthesisRequested: false, + }, + }; + } + + if (allNotes.length < 3) { + const warning = [ + `WARNING: Only ${allNotes.length} note(s) collected for topic "${params.topic}". This is insufficient for a comprehensive report.`, + "", + "Please continue researching and save more notes using save_note before calling synthesize_report again.", + "Aim for at least 5-10 notes covering different aspects of the topic.", + "", + "=== COLLECTED RESEARCH NOTES ===", + "", + formatNotes(allNotes), + "", + "=== END OF NOTES ===", + ].join("\n"); + + return { + content: [{ type: "text" as const, text: warning }], + details: { + notes_count: allNotes.length, + topic: params.topic, + format, + synthesisRequested: false, + }, + }; + } + + const summary = [ + `Research synthesis requested for: "${params.topic}"`, + `Format: ${format}`, + `Notes collected: ${allNotes.length}`, + "", + "=== COLLECTED RESEARCH NOTES ===", + "", + formatNotes(allNotes), + "", + "=== END OF NOTES ===", + "", + "IMPORTANT: You MUST now generate the complete research report in your NEXT response.", + "Do NOT call any more tools. Do NOT search or read more sources.", + "Generate the full comprehensive report with:", + "- Executive summary (2-3 paragraphs)", + "- Key findings organized by theme", + "- Source citations with URLs", + "- Confidence assessment", + "- Identified gaps for further research", + "", + "The report should be at least 2000 words and include all details from the notes above.", + ].join("\n"); + + return { + content: [{ type: "text" as const, text: summary }], + details: { + notes_count: allNotes.length, + topic: params.topic, + format, + synthesisRequested: true, + }, + }; + }, + }; +} diff --git a/packages/research-agent/src/tools/web-search.ts b/packages/research-agent/src/tools/web-search.ts new file mode 100644 index 00000000..f44968b3 --- /dev/null +++ b/packages/research-agent/src/tools/web-search.ts @@ -0,0 +1,44 @@ +import type { AgentTool } from "@earendil-works/pi-agent-core"; +import { type Static, Type } from "typebox"; +import { DEFAULT_MAX_RESULTS } from "../config.ts"; +import type { TavilySearchResponse } from "../types.ts"; +import { formatSearchResults } from "../utils/formatter.ts"; +import { tavilySearch } from "../utils/tavily.ts"; + +const webSearchSchema = Type.Object({ + query: Type.String({ description: "Search query to find information on the web" }), + max_results: Type.Optional( + Type.Number({ + description: `Maximum number of results to return (default: ${DEFAULT_MAX_RESULTS})`, + }), + ), + search_depth: Type.Optional( + Type.Union([Type.Literal("basic"), Type.Literal("advanced")], { + description: "Search depth: basic for quick overview, advanced for comprehensive results", + }), + ), +}); + +export type WebSearchInput = Static; + +export function createWebSearchTool(apiKey: string): AgentTool { + return { + name: "web_search", + label: "Web Search", + description: + "Search the web for information on a given query using Tavily. Returns titles, URLs, and content snippets.", + parameters: webSearchSchema, + execute: async (_toolCallId, params: WebSearchInput) => { + const results = await tavilySearch(params.query, { + apiKey, + maxResults: params.max_results ?? DEFAULT_MAX_RESULTS, + searchDepth: params.search_depth ?? "basic", + }); + + return { + content: [{ type: "text" as const, text: formatSearchResults(results.results) }], + details: results, + }; + }, + }; +} diff --git a/packages/research-agent/src/types.ts b/packages/research-agent/src/types.ts new file mode 100644 index 00000000..bed91320 --- /dev/null +++ b/packages/research-agent/src/types.ts @@ -0,0 +1,45 @@ +export interface ResearchNote { + topic: string; + finding: string; + source_url: string; + confidence: "high" | "medium" | "low"; + timestamp: number; +} + +export interface NoteStore { + add(note: ResearchNote): void; + getAll(): ResearchNote[]; + clear(): void; + count(): number; +} + +export interface SearchResult { + title: string; + url: string; + content: string; + score: number; +} + +export interface TavilySearchResponse { + query: string; + results: SearchResult[]; + answer?: string; +} + +export interface ResearchResult { + topic: string; + notes: ResearchNote[]; + report: string; + turnsUsed: number; +} + +export interface ResearchAgentOptions { + model?: import("@earendil-works/pi-ai").Model; + thinkingLevel?: import("@earendil-works/pi-agent-core").ThinkingLevel; + maxTurns?: number; + tavilyApiKey?: string; + llmApiKey?: string; + skills?: import("./skills/types.ts").ResearchSkill[]; + outputDir?: string; + saveResults?: boolean; +} diff --git a/packages/research-agent/src/utils/env.ts b/packages/research-agent/src/utils/env.ts new file mode 100644 index 00000000..df740c8a --- /dev/null +++ b/packages/research-agent/src/utils/env.ts @@ -0,0 +1,140 @@ +import { existsSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +/** + * Simple .env file parser + * Supports: + * - KEY=VALUE + * - KEY="VALUE" (double quotes) + * - KEY='VALUE' (single quotes) + * - # comments + * - Empty lines + * - Multiline values with quotes + */ +export function parseEnvFile(content: string): Record { + const result: Record = {}; + const lines = content.split("\n"); + let currentKey = ""; + let currentValue = ""; + let inMultiline = false; + let quoteChar = ""; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + + // Handle multiline values + if (inMultiline) { + if (line.endsWith(quoteChar)) { + currentValue += `\n${line.slice(0, -1)}`; + result[currentKey] = currentValue; + inMultiline = false; + currentKey = ""; + currentValue = ""; + } else { + currentValue += `\n${line}`; + } + continue; + } + + // Skip empty lines and comments + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) { + continue; + } + + // Parse KEY=VALUE + const eqIndex = trimmed.indexOf("="); + if (eqIndex === -1) { + continue; + } + + const key = trimmed.slice(0, eqIndex).trim(); + let value = trimmed.slice(eqIndex + 1).trim(); + + // Handle quoted values + if (value.startsWith('"') || value.startsWith("'")) { + quoteChar = value[0]; + value = value.slice(1); + + // Check if it's a multiline value + if (!value.endsWith(quoteChar)) { + currentKey = key; + currentValue = value; + inMultiline = true; + continue; + } + + // Single-line quoted value + value = value.slice(0, -1); + } + + // Unescape common escape sequences + value = value + .replace(/\\n/g, "\n") + .replace(/\\r/g, "\r") + .replace(/\\t/g, "\t") + .replace(/\\"/g, '"') + .replace(/\\'/g, "'"); + + result[key] = value; + } + + return result; +} + +/** + * Load .env file and set variables in process.env + * Only sets variables that are not already defined in process.env + * (environment variables take precedence) + */ +export function loadEnvFile(envPath?: string): void { + const paths: string[] = []; + + if (envPath) { + paths.push(envPath); + } else { + // Try current working directory + paths.push(join(process.cwd(), ".env")); + paths.push(join(process.cwd(), "packages/research-agent/.env")); + + // Try relative to this script's location + try { + const currentFile = fileURLToPath(import.meta.url); + const scriptDir = dirname(currentFile); + + // Go up from dist/utils to research-agent root + const researchAgentDir = dirname(scriptDir); + paths.push(join(researchAgentDir, ".env")); + + // Go up to project root (pi-research) + const projectRoot = dirname(researchAgentDir); + paths.push(join(projectRoot, ".env")); + } catch { + // Ignore errors if import.meta.url is not available + } + } + + for (const path of paths) { + if (!existsSync(path)) { + continue; + } + + try { + const content = readFileSync(path, "utf-8"); + const vars = parseEnvFile(content); + + for (const [key, value] of Object.entries(vars)) { + // Don't override existing environment variables + if (process.env[key] === undefined) { + process.env[key] = value; + } + } + + // Successfully loaded from first found file + return; + } catch (error) { + console.warn(`Warning: Failed to load .env file from ${path}:`, error); + } + } +} diff --git a/packages/research-agent/src/utils/fetcher.ts b/packages/research-agent/src/utils/fetcher.ts new file mode 100644 index 00000000..8fb43d8d --- /dev/null +++ b/packages/research-agent/src/utils/fetcher.ts @@ -0,0 +1,85 @@ +import { fetch } from "undici"; + +const USER_AGENT = + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"; + +const ALTERNATIVE_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0"; + +export async function fetchAndExtract(url: string, maxLength: number, useAlternativeUA = false): Promise { + const userAgent = useAlternativeUA ? ALTERNATIVE_USER_AGENT : USER_AGENT; + + const response = await fetch(url, { + headers: { + "User-Agent": userAgent, + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + }, + signal: AbortSignal.timeout(30000), + }); + + if (!response.ok) { + throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`); + } + + const contentType = response.headers.get("content-type") ?? ""; + const html = await response.text(); + + let text: string; + if (contentType.includes("text/html")) { + text = extractTextFromHtml(html); + } else { + text = html; + } + + text = collapseWhitespace(text); + + if (text.length > maxLength) { + text = `${text.slice(0, maxLength)}\n\n[... content truncated ...]`; + } + + return text; +} + +function extractTextFromHtml(html: string): string { + let text = html; + + text = text.replace(//gi, ""); + text = text.replace(//gi, ""); + text = text.replace(//gi, ""); + text = text.replace(//gi, ""); + text = text.replace(//gi, ""); + + const articleMatch = text.match(/]*>([\s\S]*?)<\/article>/i); + const mainMatch = text.match(/]*>([\s\S]*?)<\/main>/i); + + if (articleMatch) { + text = articleMatch[1]; + } else if (mainMatch) { + text = mainMatch[1]; + } + + text = text.replace(/<[^>]+>/g, " "); + + text = decodeHtmlEntities(text); + + return text; +} + +function decodeHtmlEntities(text: string): string { + return text + .replace(/ /g, " ") + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number.parseInt(code, 10))); +} + +function collapseWhitespace(text: string): string { + return text + .split("\n") + .map((line) => line.replace(/\s+/g, " ").trim()) + .filter((line) => line.length > 0) + .join("\n"); +} diff --git a/packages/research-agent/src/utils/formatter.ts b/packages/research-agent/src/utils/formatter.ts new file mode 100644 index 00000000..f31adcbb --- /dev/null +++ b/packages/research-agent/src/utils/formatter.ts @@ -0,0 +1,40 @@ +import type { ResearchNote, SearchResult } from "../types.ts"; + +export function formatSearchResults(results: SearchResult[]): string { + if (results.length === 0) { + return "No results found."; + } + + const lines: string[] = [`Found ${results.length} results:\n`]; + + for (let i = 0; i < results.length; i++) { + const r = results[i]; + lines.push(`[${i + 1}] ${r.title}`); + lines.push(` URL: ${r.url}`); + lines.push(` Relevance: ${(r.score * 100).toFixed(0)}%`); + lines.push(` Snippet: ${r.content.slice(0, 500)}${r.content.length > 500 ? "..." : ""}`); + lines.push(""); + } + + return lines.join("\n"); +} + +export function formatNotes(notes: ResearchNote[]): string { + if (notes.length === 0) { + return "No notes collected."; + } + + const lines: string[] = [`Collected ${notes.length} research notes:\n`]; + + for (let i = 0; i < notes.length; i++) { + const note = notes[i]; + lines.push(`--- Note ${i + 1} ---`); + lines.push(`Topic: ${note.topic}`); + lines.push(`Confidence: ${note.confidence}`); + lines.push(`Source: ${note.source_url}`); + lines.push(`Finding: ${note.finding}`); + lines.push(""); + } + + return lines.join("\n"); +} diff --git a/packages/research-agent/src/utils/tavily.ts b/packages/research-agent/src/utils/tavily.ts new file mode 100644 index 00000000..89bfae4b --- /dev/null +++ b/packages/research-agent/src/utils/tavily.ts @@ -0,0 +1,55 @@ +import { fetch } from "undici"; +import { TAVILY_API_URL } from "../config.ts"; +import type { SearchResult, TavilySearchResponse } from "../types.ts"; + +export interface TavilySearchOptions { + apiKey: string; + maxResults?: number; + searchDepth?: "basic" | "advanced"; + includeAnswer?: boolean; +} + +export async function tavilySearch(query: string, options: TavilySearchOptions): Promise { + const response = await fetch(TAVILY_API_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + api_key: options.apiKey, + query, + max_results: options.maxResults ?? 10, + search_depth: options.searchDepth ?? "basic", + include_answer: options.includeAnswer ?? false, + }), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Tavily API error (${response.status}): ${errorText}`); + } + + const data = (await response.json()) as { + query: string; + results: Array<{ + title: string; + url: string; + content: string; + score: number; + }>; + answer?: string; + }; + + return { + query: data.query, + results: data.results.map( + (r): SearchResult => ({ + title: r.title, + url: r.url, + content: r.content, + score: r.score, + }), + ), + answer: data.answer, + }; +} diff --git a/packages/research-agent/tsconfig.build.json b/packages/research-agent/tsconfig.build.json new file mode 100644 index 00000000..d3ec8a5b --- /dev/null +++ b/packages/research-agent/tsconfig.build.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "paths": { + "@earendil-works/pi-ai": ["../ai/dist/index.d.ts"], + "@earendil-works/pi-ai/*": ["../ai/dist/*.d.ts", "../ai/dist/providers/*.d.ts"], + "@earendil-works/pi-agent-core": ["../agent/dist/index.d.ts"], + "@earendil-works/pi-tui": ["../tui/dist/index.d.ts"] + }, + "rootDir": "./src" + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist", "**/*.d.ts", "src/**/*.d.ts"] +}