/benchg-ts - TypeScript Migration Benchmark: Genesis vs Ralph
Executes a comparative benchmark between Genesis and Ralph orchestrators for TypeScript migration projects.
/plugin marketplace add jleechanorg/claude-commands/plugin install claude-commands@claude-commands-marketplacecommands/Command Summary: Execute comprehensive TypeScript migration benchmark comparing Genesis and Ralph orchestration systems with automated monitoring, validation, and reporting
Usage: /benchg-ts
Purpose: Run complete TypeScript migration benchmark workflow with automated execution, live monitoring, consensus review, and structured reporting
When this command is invoked:
echo "π PREREQUISITE VERIFICATION"
echo "============================"
# Verify source repository
if [[ ! -d "/Users/$USER/projects/worktree_ralph" ]]; then
echo "β ERROR: Source repository not found at /Users/$USER/projects/worktree_ralph"
exit 1
fi
echo "β
Source repository found"
# Verify credentials
if [[ ! -f "/Users/$USER/projects/worktree_ralph/testing_http/testing_full/.env" ]]; then
echo "β ERROR: Credentials (.env) not found at testing_http/testing_full/.env"
exit 1
fi
echo "β
Credentials found"
# Verify engineering design document
if [[ ! -f "/Users/$USER/projects/worktree_ralph/roadmap/mvp_site_typescript_migration_eng_design.md" ]]; then
echo "β ERROR: Engineering design document not found"
exit 1
fi
echo "β
Engineering design found"
# Verify test cases
TEST_CASE_COUNT=$(find /Users/$USER/projects/worktree_ralph/testing_llm -name "*.md" 2>/dev/null | wc -l)
if [[ $TEST_CASE_COUNT -eq 0 ]]; then
echo "β ERROR: No test cases found in testing_llm/ directory"
exit 1
fi
echo "β
Found $TEST_CASE_COUNT test cases"
# Verify Ralph installation
if [[ ! -d "/Users/$USER/projects_other/ralph-orchestrator" ]]; then
echo "β ERROR: Ralph not found at /Users/$USER/projects_other/ralph-orchestrator"
exit 1
fi
echo "β
Ralph installation found"
# Verify Genesis installation
if [[ ! -f "/Users/$USER/projects/worktree_ralph/genesis/genesis.py" ]]; then
echo "β ERROR: Genesis not found at /Users/$USER/projects/worktree_ralph/genesis/genesis.py"
exit 1
fi
echo "β
Genesis installation found"
# Verify benchmark goal files
if [[ ! -f "/Users/$USER/projects/worktree_ralph/roadmap/genesis_typescript_migration_benchmark.md" ]]; then
echo "β ERROR: Genesis benchmark goal file not found"
exit 1
fi
if [[ ! -f "/Users/$USER/projects/worktree_ralph/roadmap/ralph_typescript_migration_benchmark.md" ]]; then
echo "β ERROR: Ralph benchmark goal file not found"
exit 1
fi
echo "β
Benchmark goal files found"
# Verify target directories don't exist
if [[ -d "/Users/$USER/projects_other/worldai_genesis2" ]]; then
echo "β ERROR: worldai_genesis2 already exists - remove before starting benchmark"
echo " Run: rm -rf /Users/$USER/projects_other/worldai_genesis2"
exit 1
fi
if [[ -d "/Users/$USER/projects_other/worldai_ralph2" ]]; then
echo "β ERROR: worldai_ralph2 already exists - remove before starting benchmark"
echo " Run: rm -rf /Users/$USER/projects_other/worldai_ralph2"
exit 1
fi
echo "β
Clean state verified - no existing benchmark directories"
echo ""
echo "β
All prerequisites verified - ready to start benchmark"
echo ""
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
BENCHMARK_LOG="/tmp/typescript_migration_benchmark_$TIMESTAMP.log"
echo "π TYPESCRIPT MIGRATION BENCHMARK: Genesis vs Ralph"
echo "===================================================="
echo ""
echo "π Configuration:"
echo " Source: /Users/$USER/projects/worktree_ralph/$PROJECT_ROOT/"
echo " Genesis Target: /Users/$USER/projects_other/worldai_genesis2"
echo " Ralph Target: /Users/$USER/projects_other/worldai_ralph2"
echo " Max Iterations: 50 per agent"
echo " Test Cases: $TEST_CASE_COUNT files in testing_llm/"
echo " Log File: $BENCHMARK_LOG"
echo ""
echo "π― Objective: Migrate Python Flask backend to TypeScript FastMCP server"
echo "π Success Criteria: Production server running, all tools listed, tests passing"
echo ""
echo "𧬠STARTING GENESIS ORCHESTRATOR"
echo "=================================="
echo ""
GENESIS_SESSION="genesis-ts-migration-$(date +%Y%m%d-%H%M%S)"
GENESIS_DIR="/Users/$USER/projects_other/worldai_genesis2"
echo "π Launching Genesis with orchestration command:"
echo " Session: $GENESIS_SESSION"
echo " Working Dir: $GENESIS_DIR"
echo " Goal File: roadmap/genesis_typescript_migration_benchmark.md"
echo ""
# Execute Genesis using /gene command
cd /Users/$USER/projects/worktree_ralph
# Create Genesis orchestration command
python3 genesis/genesis.py \
roadmap/genesis_typescript_migration_benchmark.md \
50 \
--verbose \
2>&1 | tee -a "$BENCHMARK_LOG" | tee "/tmp/genesis_benchmark_$TIMESTAMP.log" &
GENESIS_PID=$!
echo "β
Genesis started (PID: $GENESIS_PID)"
echo ""
# Give Genesis time to initialize
sleep 15
echo "π€ STARTING RALPH ORCHESTRATOR"
echo "==============================="
echo ""
RALPH_SESSION="ralph-ts-migration-$(date +%Y%m%d-%H%M%S)"
RALPH_DIR="/Users/$USER/projects_other/worldai_ralph2"
echo "π Launching Ralph with orchestration command:"
echo " Session: $RALPH_SESSION"
echo " Working Dir: $RALPH_DIR"
echo " Goal File: roadmap/ralph_typescript_migration_benchmark.md"
echo ""
# Execute Ralph
cd /Users/$USER/projects_other/ralph-orchestrator
python -m ralph_orchestrator \
/Users/$USER/projects/worktree_ralph/roadmap/ralph_typescript_migration_benchmark.md \
--agent codex \
--max-iterations 50 \
--verbose \
2>&1 | tee -a "$BENCHMARK_LOG" | tee "/tmp/ralph_benchmark_$TIMESTAMP.log" &
RALPH_PID=$!
echo "β
Ralph started (PID: $RALPH_PID)"
echo ""
# Give Ralph time to initialize
sleep 15
echo "π MONITORING BOTH ORCHESTRATORS"
echo "================================="
echo ""
MONITORING_DURATION=28800 # 8 hours maximum (TypeScript migration is complex)
START_TIME=$(date +%s)
echo "β° Monitoring for up to $((MONITORING_DURATION / 3600)) hours..."
echo "β±οΈ Check interval: 60 seconds"
echo "π Progress indicators: commit counts, file counts"
echo ""
# Monitor loop
while true; do
CURRENT_TIME=$(date +%s)
ELAPSED=$((CURRENT_TIME - START_TIME))
if [[ $ELAPSED -gt $MONITORING_DURATION ]]; then
echo ""
echo "β° Maximum monitoring time reached ($((MONITORING_DURATION / 3600)) hours)"
echo "β οΈ Stopping monitoring - agents may still be running"
break
fi
# Check Genesis status
if ps -p $GENESIS_PID > /dev/null 2>&1; then
GENESIS_RUNNING="Running"
if [[ -d "$GENESIS_DIR" ]]; then
GENESIS_COMMITS=$(cd "$GENESIS_DIR" 2>/dev/null && git log --oneline 2>/dev/null | wc -l | tr -d ' ' || echo "0")
GENESIS_FILES=$(find "$GENESIS_DIR" -name "*.ts" -o -name "*.js" 2>/dev/null | wc -l | tr -d ' ')
else
GENESIS_COMMITS="0"
GENESIS_FILES="0"
fi
else
GENESIS_RUNNING="Stopped"
if [[ -d "$GENESIS_DIR" ]]; then
GENESIS_COMMITS=$(cd "$GENESIS_DIR" 2>/dev/null && git log --oneline 2>/dev/null | wc -l | tr -d ' ' || echo "0")
GENESIS_FILES=$(find "$GENESIS_DIR" -name "*.ts" -o -name "*.js" 2>/dev/null | wc -l | tr -d ' ')
else
GENESIS_COMMITS="0"
GENESIS_FILES="0"
fi
fi
# Check Ralph status
if ps -p $RALPH_PID > /dev/null 2>&1; then
RALPH_RUNNING="Running"
if [[ -d "$RALPH_DIR" ]]; then
RALPH_COMMITS=$(cd "$RALPH_DIR" 2>/dev/null && git log --oneline 2>/dev/null | wc -l | tr -d ' ' || echo "0")
RALPH_FILES=$(find "$RALPH_DIR" -name "*.ts" -o -name "*.js" 2>/dev/null | wc -l | tr -d ' ')
else
RALPH_COMMITS="0"
RALPH_FILES="0"
fi
else
RALPH_RUNNING="Stopped"
if [[ -d "$RALPH_DIR" ]]; then
RALPH_COMMITS=$(cd "$RALPH_DIR" 2>/dev/null && git log --oneline 2>/dev/null | wc -l | tr -d ' ' || echo "0")
RALPH_FILES=$(find "$RALPH_DIR" -name "*.ts" -o -name "*.js" 2>/dev/null | wc -l | tr -d ' ')
else
RALPH_COMMITS="0"
RALPH_FILES="0"
fi
fi
# Status update
printf "\r$(date +'%H:%M:%S') [%3dm] Genesis: %-8s (%2dc/%2df) | Ralph: %-8s (%2dc/%2df)" \
$((ELAPSED / 60)) \
"$GENESIS_RUNNING" "$GENESIS_COMMITS" "$GENESIS_FILES" \
"$RALPH_RUNNING" "$RALPH_COMMITS" "$RALPH_FILES"
# Check if both completed
if [[ "$GENESIS_RUNNING" == "Stopped" ]] && [[ "$RALPH_RUNNING" == "Stopped" ]]; then
echo ""
echo ""
echo "β
Both orchestrators completed"
break
fi
# Wait before next check
sleep 60
done
echo ""
echo "β±οΈ Total execution time: $(($ELAPSED / 60)) minutes ($((ELAPSED / 3600)) hours)"
echo ""
echo "π COLLECTING BENCHMARK RESULTS"
echo "==============================="
echo ""
# Genesis results
echo "𧬠Genesis Implementation:"
if [[ -d "$GENESIS_DIR" ]]; then
cd "$GENESIS_DIR"
GENESIS_EXISTS="Yes"
GENESIS_COMMITS=$(git log --oneline 2>/dev/null | wc -l | tr -d ' ')
GENESIS_FILES=$(find . -name "*.ts" -o -name "*.js" 2>/dev/null | wc -l)
GENESIS_LINES=$(find . -name "*.ts" -o -name "*.js" -exec wc -l {} + 2>/dev/null | tail -1 | awk '{print $1}' || echo "0")
GENESIS_PACKAGE_JSON=$([[ -f "package.json" ]] && echo "Yes" || echo "No")
GENESIS_TSCONFIG=$([[ -f "tsconfig.json" ]] && echo "Yes" || echo "No")
GENESIS_ENV=$([[ -f ".env" ]] && echo "Yes" || echo "No")
GENESIS_FIREBASE=$([[ -f "firebase-service-account.json" ]] && echo "Yes" || echo "No")
echo " β
Repository exists"
echo " π Location: $GENESIS_DIR"
echo " π Commits: $GENESIS_COMMITS"
echo " π TS/JS Files: $GENESIS_FILES"
echo " π Lines of Code: $GENESIS_LINES"
echo " π¦ package.json: $GENESIS_PACKAGE_JSON"
echo " π§ tsconfig.json: $GENESIS_TSCONFIG"
echo " π Credentials: .env=$GENESIS_ENV, firebase=$GENESIS_FIREBASE"
else
echo " β Repository does not exist"
GENESIS_EXISTS="No"
GENESIS_COMMITS=0
GENESIS_FILES=0
GENESIS_LINES=0
GENESIS_PACKAGE_JSON="No"
GENESIS_TSCONFIG="No"
GENESIS_ENV="No"
GENESIS_FIREBASE="No"
fi
echo ""
# Ralph results
echo "π€ Ralph Implementation:"
if [[ -d "$RALPH_DIR" ]]; then
cd "$RALPH_DIR"
RALPH_EXISTS="Yes"
RALPH_COMMITS=$(git log --oneline 2>/dev/null | wc -l | tr -d ' ')
RALPH_FILES=$(find . -name "*.ts" -o -name "*.js" 2>/dev/null | wc -l)
RALPH_LINES=$(find . -name "*.ts" -o -name "*.js" -exec wc -l {} + 2>/dev/null | tail -1 | awk '{print $1}' || echo "0")
RALPH_PACKAGE_JSON=$([[ -f "package.json" ]] && echo "Yes" || echo "No")
RALPH_TSCONFIG=$([[ -f "tsconfig.json" ]] && echo "Yes" || echo "No")
RALPH_ENV=$([[ -f ".env" ]] && echo "Yes" || echo "No")
RALPH_FIREBASE=$([[ -f "firebase-service-account.json" ]] && echo "Yes" || echo "No")
echo " β
Repository exists"
echo " π Location: $RALPH_DIR"
echo " π Commits: $RALPH_COMMITS"
echo " π TS/JS Files: $RALPH_FILES"
echo " π Lines of Code: $RALPH_LINES"
echo " π¦ package.json: $RALPH_PACKAGE_JSON"
echo " π§ tsconfig.json: $RALPH_TSCONFIG"
echo " π Credentials: .env=$RALPH_ENV, firebase=$RALPH_FIREBASE"
else
echo " β Repository does not exist"
RALPH_EXISTS="No"
RALPH_COMMITS=0
RALPH_FILES=0
RALPH_LINES=0
RALPH_PACKAGE_JSON="No"
RALPH_TSCONFIG="No"
RALPH_ENV="No"
RALPH_FIREBASE="No"
fi
echo ""
echo "π LIVE BUILD & TEST VALIDATION"
echo "==============================="
echo ""
# Test Genesis implementation
echo "𧬠Testing Genesis Implementation:"
if [[ "$GENESIS_EXISTS" == "Yes" ]] && [[ "$GENESIS_PACKAGE_JSON" == "Yes" ]]; then
cd "$GENESIS_DIR"
echo " π¦ Installing dependencies..."
if npm install &>/dev/null; then
echo " β
npm install succeeded"
else
echo " β npm install failed"
fi
echo " π¨ Building project..."
if npm run build &>/dev/null; then
echo " β
npm run build succeeded"
GENESIS_BUILD="Pass"
else
echo " β οΈ npm run build failed"
GENESIS_BUILD="Fail"
fi
echo " π§ͺ Running tests..."
if npm test &>/dev/null; then
echo " β
npm test passed"
GENESIS_TESTS="Pass"
else
echo " β οΈ npm test failed or no tests found"
GENESIS_TESTS="Fail/None"
fi
echo " π Testing server startup..."
timeout 30 npm start &>/tmp/genesis_server_test.log &
SERVER_PID=$!
sleep 10
if curl -s http://localhost:3001/health | grep -q "ok"; then
echo " β
Server started, health endpoint responds"
GENESIS_SERVER="Pass"
else
echo " β οΈ Server failed to start or health check failed"
GENESIS_SERVER="Fail"
fi
kill $SERVER_PID 2>/dev/null
else
echo " β Cannot test - no package.json or repository"
GENESIS_BUILD="N/A"
GENESIS_TESTS="N/A"
GENESIS_SERVER="N/A"
fi
echo ""
# Test Ralph implementation
echo "π€ Testing Ralph Implementation:"
if [[ "$RALPH_EXISTS" == "Yes" ]] && [[ "$RALPH_PACKAGE_JSON" == "Yes" ]]; then
cd "$RALPH_DIR"
echo " π¦ Installing dependencies..."
if npm install &>/dev/null; then
echo " β
npm install succeeded"
else
echo " β npm install failed"
fi
echo " π¨ Building project..."
if npm run build &>/dev/null; then
echo " β
npm run build succeeded"
RALPH_BUILD="Pass"
else
echo " β οΈ npm run build failed"
RALPH_BUILD="Fail"
fi
echo " π§ͺ Running tests..."
if npm test &>/dev/null; then
echo " β
npm test passed"
RALPH_TESTS="Pass"
else
echo " β οΈ npm test failed or no tests found"
RALPH_TESTS="Fail/None"
fi
echo " π Testing server startup..."
timeout 30 npm start &>/tmp/ralph_server_test.log &
SERVER_PID=$!
sleep 10
if curl -s http://localhost:3001/health | grep -q "ok"; then
echo " β
Server started, health endpoint responds"
RALPH_SERVER="Pass"
else
echo " β οΈ Server failed to start or health check failed"
RALPH_SERVER="Fail"
fi
kill $SERVER_PID 2>/dev/null
else
echo " β Cannot test - no package.json or repository"
RALPH_BUILD="N/A"
RALPH_TESTS="N/A"
RALPH_SERVER="N/A"
fi
echo ""
echo "π― CODE QUALITY CONSENSUS REVIEW"
echo "================================"
echo ""
# Only proceed if both have implementations
if [[ "$GENESIS_EXISTS" == "Yes" ]] && [[ "$RALPH_EXISTS" == "Yes" ]]; then
echo "π Running consensus code review on both implementations..."
echo ""
# Run /cons on Genesis
echo "𧬠Genesis Code Review:"
cd "$GENESIS_DIR"
/cons 2>&1 | head -50
echo ""
echo "π€ Ralph Code Review:"
cd "$RALPH_DIR"
/cons 2>&1 | head -50
echo ""
echo "β
Consensus reviews completed (see logs for full details)"
else
echo "β οΈ Skipping consensus review - at least one implementation missing"
fi
echo ""
echo "π BENCHMARK SUMMARY REPORT"
echo "==========================="
echo ""
echo "π TypeScript Migration Benchmark: Genesis vs Ralph"
echo "π
Completion Date: $(date)"
echo "β±οΈ Total Duration: $(($ELAPSED / 60)) minutes ($((ELAPSED / 3600)) hours)"
echo ""
echo "π QUANTITATIVE RESULTS"
echo "------------------------"
printf "| %-20s | %-15s | %-15s | %-15s |\n" "Metric" "Genesis" "Ralph" "Winner"
printf "| %-20s | %-15s | %-15s | %-15s |\n" "--------------------" "---------------" "---------------" "---------------"
printf "| %-20s | %-15s | %-15s | %-15s |\n" "Repository Created" "$GENESIS_EXISTS" "$RALPH_EXISTS" "$([[ "$GENESIS_EXISTS" == "Yes" ]] && [[ "$RALPH_EXISTS" == "Yes" ]] && echo "Both" || echo "N/A")"
printf "| %-20s | %-15s | %-15s | %-15s |\n" "Commits" "$GENESIS_COMMITS" "$RALPH_COMMITS" "$([[ $GENESIS_COMMITS -gt $RALPH_COMMITS ]] && echo "Genesis" || [[ $RALPH_COMMITS -gt $GENESIS_COMMITS ]] && echo "Ralph" || echo "Tie")"
printf "| %-20s | %-15s | %-15s | %-15s |\n" "Files Created" "$GENESIS_FILES" "$RALPH_FILES" "$([[ $GENESIS_FILES -gt $RALPH_FILES ]] && echo "Genesis" || [[ $RALPH_FILES -gt $GENESIS_FILES ]] && echo "Ralph" || echo "Tie")"
printf "| %-20s | %-15s | %-15s | %-15s |\n" "Lines of Code" "$GENESIS_LINES" "$RALPH_LINES" "$([[ $GENESIS_LINES -gt $RALPH_LINES ]] && echo "Genesis" || [[ $RALPH_LINES -gt $GENESIS_LINES ]] && echo "Ralph" || echo "Tie")"
printf "| %-20s | %-15s | %-15s | %-15s |\n" "Credentials Copied" "$([[ "$GENESIS_ENV" == "Yes" ]] && [[ "$GENESIS_FIREBASE" == "Yes" ]] && echo "Yes" || echo "No")" "$([[ "$RALPH_ENV" == "Yes" ]] && [[ "$RALPH_FIREBASE" == "Yes" ]] && echo "Yes" || echo "No")" "Both/Neither"
printf "| %-20s | %-15s | %-15s | %-15s |\n" "Build Status" "$GENESIS_BUILD" "$RALPH_BUILD" "$([[ "$GENESIS_BUILD" == "Pass" ]] && [[ "$RALPH_BUILD" != "Pass" ]] && echo "Genesis" || [[ "$RALPH_BUILD" == "Pass" ]] && [[ "$GENESIS_BUILD" != "Pass" ]] && echo "Ralph" || echo "Both/Neither")"
printf "| %-20s | %-15s | %-15s | %-15s |\n" "Test Status" "$GENESIS_TESTS" "$RALPH_TESTS" "$([[ "$GENESIS_TESTS" == "Pass" ]] && [[ "$RALPH_TESTS" != "Pass" ]] && echo "Genesis" || [[ "$RALPH_TESTS" == "Pass" ]] && [[ "$GENESIS_TESTS" != "Pass" ]] && echo "Ralph" || echo "Both/Neither")"
printf "| %-20s | %-15s | %-15s | %-15s |\n" "Server Startup" "$GENESIS_SERVER" "$RALPH_SERVER" "$([[ "$GENESIS_SERVER" == "Pass" ]] && [[ "$RALPH_SERVER" != "Pass" ]] && echo "Genesis" || [[ "$RALPH_SERVER" == "Pass" ]] && [[ "$GENESIS_SERVER" != "Pass" ]] && echo "Ralph" || echo "Both/Neither")"
echo ""
echo "π― SUCCESS CRITERIA ANALYSIS"
echo "-----------------------------"
echo ""
echo "Genesis Completion Status:"
[[ "$GENESIS_EXISTS" == "Yes" ]] && echo " β
Repository created" || echo " β Repository not created"
[[ "$GENESIS_COMMITS" -gt 0 ]] && echo " β
Has commits ($GENESIS_COMMITS)" || echo " β No commits"
[[ "$GENESIS_ENV" == "Yes" ]] && [[ "$GENESIS_FIREBASE" == "Yes" ]] && echo " β
Credentials copied" || echo " β Credentials missing"
[[ "$GENESIS_BUILD" == "Pass" ]] && echo " β
Build succeeds" || echo " β Build fails"
[[ "$GENESIS_SERVER" == "Pass" ]] && echo " β
Server starts" || echo " β Server doesn't start"
echo ""
echo "Ralph Completion Status:"
[[ "$RALPH_EXISTS" == "Yes" ]] && echo " β
Repository created" || echo " β Repository not created"
[[ "$RALPH_COMMITS" -gt 0 ]] && echo " β
Has commits ($RALPH_COMMITS)" || echo " β No commits"
[[ "$RALPH_ENV" == "Yes" ]] && [[ "$RALPH_FIREBASE" == "Yes" ]] && echo " β
Credentials copied" || echo " β Credentials missing"
[[ "$RALPH_BUILD" == "Pass" ]] && echo " β
Build succeeds" || echo " β Build fails"
[[ "$RALPH_SERVER" == "Pass" ]] && echo " β
Server starts" || echo " β Server doesn't start"
echo ""
echo "π DETAILED LOGS"
echo "----------------"
echo "Benchmark Log: $BENCHMARK_LOG"
echo "Genesis Log: /tmp/genesis_benchmark_$TIMESTAMP.log"
echo "Ralph Log: /tmp/ralph_benchmark_$TIMESTAMP.log"
echo ""
echo "π IMPLEMENTATION LOCATIONS"
echo "---------------------------"
echo "Genesis: $GENESIS_DIR"
echo "Ralph: $RALPH_DIR"
echo ""
echo "π NEXT STEPS"
echo "-------------"
echo "1. Review detailed logs for execution traces"
echo "2. Examine code quality in both repositories"
echo "3. Compare architectural approaches"
echo "4. Analyze which agent better handled the TypeScript migration"
echo "5. Document lessons learned for future benchmarks"
echo ""
echo "β
Benchmark completed successfully!"
echo ""
/cons code review on both implementationsThis command:
roadmap/genesis_typescript_migration_benchmark.md and roadmap/ralph_typescript_migration_benchmark.mdgenesis/genesis.pyralph-orchestrator Python module/cons for code quality review/benchgPerfect for comprehensive evaluation of TypeScript migration capabilities and orchestration system comparison.