#!/usr/bin/env python3 """ SDR Benchmark Runner for DeepResearchGym evaluation. Usage: cd /root/sdr_repo python ../Strategy/research/sdr_benchmark_runner.py --queries scientific --n 5 python ../Strategy/research/sdr_benchmark_runner.py --queries drg --n 10 Output: saves {id}.a and {id}.q files for DeepResearchGym eval_quality_async.py. Requires SDR dependencies (run from sdr_repo with uv): cd /root/sdr_repo && uv run python ../Strategy/research/sdr_benchmark_runner.py DeepResearchGym evaluation (needs OpenAI key in deepresearch_benchmarking/keys.env): cd /root/Strategy/research/deepresearch_benchmarking python eval_quality_async.py --subfolder SDR --open_ai_model gpt-4.1-mini """ import asyncio import json import sys import os import argparse from pathlib import Path from datetime import datetime # Add SDR to path when running from sdr_repo sys.path.insert(0, str(Path(__file__).parent.parent.parent / "sdr_repo")) sys.path.insert(0, str(Path(__file__).parent.parent.parent / "sdr_repo" / "src")) # Scientific queries relevant to SDR's domain (academic search focus) SCIENTIFIC_QUERIES = [ {"id": "sci_001", "query": "What are the state-of-the-art methods for operator splitting in machine learning optimization?"}, {"id": "sci_002", "query": "How does FlashAttention improve transformer training efficiency compared to standard attention?"}, {"id": "sci_003", "query": "What is the current evidence on neural operators for solving partial differential equations?"}, {"id": "sci_004", "query": "How do large language models perform on scientific reasoning benchmarks like GPQA and SciCode?"}, {"id": "sci_005", "query": "What are the most effective approaches for few-shot learning in low-resource scientific domains?"}, {"id": "sci_006", "query": "How does mechanistic interpretability help understand transformer circuits and features?"}, {"id": "sci_007", "query": "What are the convergence guarantees for proximal gradient methods with regularization?"}, {"id": "sci_008", "query": "How do multi-agent LLM systems compare to single-agent systems on complex research tasks?"}, {"id": "sci_009", "query": "What is the role of synthetic data in improving scientific AI systems?"}, {"id": "sci_010", "query": "How effective are retrieval-augmented generation systems for scientific literature synthesis?"}, ] # Output directory compatible with DeepResearchGym eval scripts OUTPUT_DIR = Path("/root/Strategy/research/sdr_reports/SDR") async def run_single_query(graph, query_id: str, query_text: str, output_dir: Path) -> dict: """Run SDR on a single query and save output.""" from langchain_core.messages import HumanMessage from open_deep_research.configuration import Config print(f"[{datetime.now():%H:%M:%S}] Running query {query_id}: {query_text[:80]}...") # Save query file q_file = output_dir / f"{query_id}.q" q_file.write_text(query_text, encoding="utf-8") config = Config() try: # Invoke graph - skip clarification for benchmark result = await graph.ainvoke( {"messages": [HumanMessage(content=query_text)]}, config={ "configurable": { **config.model_dump(), "allow_clarification": False, # No user interaction in benchmark mode } } ) # Extract final report from messages report = "" for msg in reversed(result.get("messages", [])): if hasattr(msg, "content") and isinstance(msg.content, str) and len(msg.content) > 500: report = msg.content break if not report: report = result.get("final_report", "No report generated") # Save answer file a_file = output_dir / f"{query_id}.a" a_file.write_text(report, encoding="utf-8") print(f"[{datetime.now():%H:%M:%S}] ✅ {query_id}: {len(report)} chars saved") return {"id": query_id, "status": "ok", "chars": len(report)} except Exception as e: error_msg = f"ERROR: {type(e).__name__}: {e}" print(f"[{datetime.now():%H:%M:%S}] ❌ {query_id}: {error_msg}") a_file = output_dir / f"{query_id}.a" a_file.write_text(error_msg, encoding="utf-8") return {"id": query_id, "status": "error", "error": str(e)} async def main(queries: list[dict], output_dir: Path, parallel: bool = False): """Run SDR benchmark on a list of queries.""" from open_deep_research.deep_researcher import create_deep_researcher_graph output_dir.mkdir(parents=True, exist_ok=True) print(f"Output dir: {output_dir}") print(f"Running {len(queries)} queries {'in parallel' if parallel else 'sequentially'}...") graph = create_deep_researcher_graph().compile() results = [] if parallel: tasks = [run_single_query(graph, q["id"], q["query"], output_dir) for q in queries] results = await asyncio.gather(*tasks, return_exceptions=True) else: for q in queries: result = await run_single_query(graph, q["id"], q["query"], output_dir) results.append(result) # Save run summary summary = { "run_time": datetime.now().isoformat(), "total": len(queries), "ok": sum(1 for r in results if isinstance(r, dict) and r.get("status") == "ok"), "errors": sum(1 for r in results if isinstance(r, dict) and r.get("status") == "error"), "results": [r for r in results if isinstance(r, dict)], } summary_file = output_dir / "run_summary.json" summary_file.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") print(f"\n=== Summary ===") print(f"OK: {summary['ok']}/{summary['total']}, Errors: {summary['errors']}") print(f"Summary saved to: {summary_file}") # Instructions for DeepResearchGym evaluation drg_dir = Path("/root/Strategy/research/deepresearch_benchmarking") if drg_dir.exists(): print(f""" === Next: DeepResearchGym Evaluation === 1. Add OpenAI key to {drg_dir}/keys.env: OPENAI_API_KEY=sk-... 2. Modify eval_quality_async.py: change data path to: /root/Strategy/research/sdr_reports/ 3. Run evaluation: cd {drg_dir} python eval_quality_async.py --subfolder SDR --open_ai_model gpt-4.1-mini """) if __name__ == "__main__": parser = argparse.ArgumentParser(description="Run SDR on benchmark queries") parser.add_argument( "--queries", choices=["scientific", "drg"], default="scientific", help="Query set: 'scientific' (custom SDR-relevant) or 'drg' (DeepResearchGym sample)" ) parser.add_argument("--n", type=int, default=5, help="Number of queries to run") parser.add_argument("--parallel", action="store_true", help="Run queries in parallel (faster but more API calls)") parser.add_argument("--output-dir", type=str, default=str(OUTPUT_DIR), help="Output directory") args = parser.parse_args() if args.queries == "scientific": queries = SCIENTIFIC_QUERIES[:args.n] else: # Load from DeepResearchGym drg_queries_file = Path("/root/Strategy/research/deepresearch_benchmarking/queries/researchy_queries_sample_doc_click_100.jsonl") if not drg_queries_file.exists(): print(f"DeepResearchGym queries not found: {drg_queries_file}") sys.exit(1) with drg_queries_file.open() as f: queries = [json.loads(line) for line in f][:args.n] output_dir = Path(args.output_dir) asyncio.run(main(queries, output_dir, args.parallel))