OpenEvidence CI Integration

Overview

Integrate OpenEvidence testing and validation into CI/CD pipelines for healthcare applications.

Prerequisites

• GitHub, GitLab, or other CI platform
• OpenEvidence sandbox API credentials
• Test suite configured
• Secret management in CI platform

Instructions

Step 1: GitHub Actions Workflow

# .github/workflows/openevidence-ci.yml
name: OpenEvidence CI

on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main]

env:
  NODE_VERSION: '20'

jobs:
  lint-and-typecheck:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Lint
        run: npm run lint

      - name: Type check
        run: npm run typecheck

  unit-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Run unit tests
        run: npm run test:unit -- --coverage

      - name: Upload coverage
        uses: codecov/codecov-action@v4
        with:
          files: ./coverage/lcov.info
          fail_ci_if_error: false

  integration-tests:
    runs-on: ubuntu-latest
    # Only run on main branch or explicit trigger
    if: github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'run-integration')
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Run integration tests
        env:
          OPENEVIDENCE_API_KEY: ${{ secrets.OPENEVIDENCE_SANDBOX_API_KEY }}
          OPENEVIDENCE_ORG_ID: ${{ secrets.OPENEVIDENCE_SANDBOX_ORG_ID }}
          OPENEVIDENCE_BASE_URL: https://api.sandbox.openevidence.com
        run: npm run test:integration
        timeout-minutes: 10

  clinical-validation:
    runs-on: ubuntu-latest
    needs: [unit-tests]
    if: github.ref == 'refs/heads/main'
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Run clinical validation tests
        env:
          OPENEVIDENCE_API_KEY: ${{ secrets.OPENEVIDENCE_SANDBOX_API_KEY }}
          OPENEVIDENCE_ORG_ID: ${{ secrets.OPENEVIDENCE_SANDBOX_ORG_ID }}
        run: npm run test:clinical-validation

      - name: Upload validation report
        uses: actions/upload-artifact@v4
        with:
          name: clinical-validation-report
          path: reports/clinical-validation.json

Step 2: Test Configuration

// vitest.config.ts
import { defineConfig } from 'vitest/config';

export default defineConfig({
  test: {
    globals: true,
    environment: 'node',
    include: ['tests/**/*.test.ts'],
    exclude: ['tests/integration/**', 'tests/clinical-validation/**'],
    coverage: {
      provider: 'v8',
      reporter: ['text', 'lcov'],
      exclude: ['tests/**', 'node_modules/**'],
    },
    testTimeout: 10000,
  },
});

// vitest.config.integration.ts
import { defineConfig, mergeConfig } from 'vitest/config';
import baseConfig from './vitest.config';

export default mergeConfig(baseConfig, defineConfig({
  test: {
    include: ['tests/integration/**/*.test.ts'],
    exclude: [],
    testTimeout: 60000, // 60s for API calls
    retry: 2, // Retry flaky integration tests
    maxConcurrency: 1, // Avoid rate limits
  },
}));

Step 3: Unit Tests with Mocks

// tests/unit/clinical-query.test.ts
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { queryClinicalEvidence } from '../../src/services/clinical-query';

// Mock the OpenEvidence SDK
vi.mock('@openevidence/sdk', () => ({
  OpenEvidenceClient: vi.fn().mockImplementation(() => ({
    query: vi.fn().mockResolvedValue({
      id: 'test-query-123',
      answer: 'Mock clinical answer for testing purposes.',
      citations: [
        { source: 'Test Journal', title: 'Test Article', year: 2025 }
      ],
      confidence: 0.95,
      lastUpdated: '2025-01-01',
    }),
  })),
}));

describe('Clinical Query Service', () => {
  beforeEach(() => {
    vi.clearAllMocks();
  });

  it('should return formatted clinical response', async () => {
    const result = await queryClinicalEvidence('Test question');

    expect(result.answer).toBeDefined();
    expect(result.citations).toHaveLength(1);
    expect(result.confidence).toBeGreaterThan(0);
  });

  it('should handle query errors gracefully', async () => {
    const { OpenEvidenceClient } = await import('@openevidence/sdk');
    vi.mocked(OpenEvidenceClient).mockImplementation(() => ({
      query: vi.fn().mockRejectedValue(new Error('API Error')),
    } as any));

    await expect(queryClinicalEvidence('Test question'))
      .rejects.toThrow('API Error');
  });
});

Step 4: Integration Tests

// tests/integration/openevidence-api.test.ts
import { describe, it, expect, beforeAll } from 'vitest';
import { OpenEvidenceClient } from '@openevidence/sdk';

describe('OpenEvidence API Integration', () => {
  let client: OpenEvidenceClient;

  beforeAll(() => {
    // Ensure we're using sandbox credentials
    const baseUrl = process.env.OPENEVIDENCE_BASE_URL;
    if (!baseUrl?.includes('sandbox')) {
      throw new Error('Integration tests must use sandbox environment');
    }

    client = new OpenEvidenceClient({
      apiKey: process.env.OPENEVIDENCE_API_KEY!,
      orgId: process.env.OPENEVIDENCE_ORG_ID!,
      baseUrl,
    });
  });

  it('should successfully query clinical evidence', async () => {
    const response = await client.query({
      question: 'What is the half-life of aspirin?',
      context: {
        specialty: 'pharmacology',
        urgency: 'routine',
      },
    });

    expect(response.id).toBeDefined();
    expect(response.answer).toBeDefined();
    expect(response.answer.length).toBeGreaterThan(50);
    expect(response.citations.length).toBeGreaterThan(0);
    expect(response.confidence).toBeGreaterThan(0.5);
  }, 30000);

  it('should handle invalid queries appropriately', async () => {
    await expect(client.query({
      question: '',
      context: { specialty: 'internal-medicine', urgency: 'routine' },
    })).rejects.toThrow();
  });

  it('should return citations with required fields', async () => {
    const response = await client.query({
      question: 'What are the first-line treatments for hypertension?',
      context: { specialty: 'cardiology', urgency: 'routine' },
    });

    response.citations.forEach(citation => {
      expect(citation.source).toBeDefined();
      expect(citation.title).toBeDefined();
      expect(citation.year).toBeGreaterThan(2000);
    });
  }, 30000);
});

Step 5: Clinical Validation Tests

// tests/clinical-validation/known-answers.test.ts
import { describe, it, expect } from 'vitest';
import { OpenEvidenceClient } from '@openevidence/sdk';

interface ClinicalValidationCase {
  question: string;
  specialty: string;
  expectedKeywords: string[];
  mustNotContain?: string[];
}

const VALIDATION_CASES: ClinicalValidationCase[] = [
  {
    question: 'What is the first-line treatment for type 2 diabetes in adults?',
    specialty: 'endocrinology',
    expectedKeywords: ['metformin', 'lifestyle', 'HbA1c'],
    mustNotContain: ['insulin-first'],
  },
  {
    question: 'What are the contraindications for aspirin?',
    specialty: 'internal-medicine',
    expectedKeywords: ['bleeding', 'allergy', 'ulcer'],
  },
  {
    question: 'What is the target LDL for secondary prevention of cardiovascular disease?',
    specialty: 'cardiology',
    expectedKeywords: ['LDL', 'mg/dL', 'statin'],
  },
];

describe('Clinical Validation - Known Answers', () => {
  const client = new OpenEvidenceClient({
    apiKey: process.env.OPENEVIDENCE_API_KEY!,
    orgId: process.env.OPENEVIDENCE_ORG_ID!,
  });

  const results: any[] = [];

  VALIDATION_CASES.forEach((testCase, index) => {
    it(`should answer correctly: ${testCase.question.substring(0, 50)}...`, async () => {
      const response = await client.query({
        question: testCase.question,
        context: {
          specialty: testCase.specialty,
          urgency: 'routine',
        },
      });

      const answerLower = response.answer.toLowerCase();

      // Check expected keywords
      const foundKeywords = testCase.expectedKeywords.filter(
        kw => answerLower.includes(kw.toLowerCase())
      );

      // Check must-not-contain
      const forbiddenFound = testCase.mustNotContain?.filter(
        kw => answerLower.includes(kw.toLowerCase())
      ) || [];

      // Record results for report
      results.push({
        question: testCase.question,
        expectedKeywords: testCase.expectedKeywords,
        foundKeywords,
        forbiddenFound,
        confidence: response.confidence,
        citationCount: response.citations.length,
        passed: foundKeywords.length >= testCase.expectedKeywords.length / 2 &&
                forbiddenFound.length === 0,
      });

      // Assert
      expect(foundKeywords.length).toBeGreaterThanOrEqual(
        Math.ceil(testCase.expectedKeywords.length / 2),
        `Missing keywords: ${testCase.expectedKeywords.filter(k => !foundKeywords.includes(k)).join(', ')}`
      );
      expect(forbiddenFound).toHaveLength(0);
      expect(response.confidence).toBeGreaterThan(0.7);
    }, 60000);
  });

  afterAll(async () => {
    // Write validation report
    const fs = await import('fs/promises');
    await fs.mkdir('reports', { recursive: true });
    await fs.writeFile(
      'reports/clinical-validation.json',
      JSON.stringify({
        timestamp: new Date().toISOString(),
        totalTests: results.length,
        passed: results.filter(r => r.passed).length,
        failed: results.filter(r => !r.passed).length,
        results,
      }, null, 2)
    );
  });
});

Step 6: Secret Management

# Store these as GitHub Secrets:
# - OPENEVIDENCE_SANDBOX_API_KEY
# - OPENEVIDENCE_SANDBOX_ORG_ID
# - OPENEVIDENCE_PROD_API_KEY (for deployment only)
# - OPENEVIDENCE_PROD_ORG_ID (for deployment only)

# For local development, use .env.test:
# OPENEVIDENCE_API_KEY=oe_sandbox_***
# OPENEVIDENCE_ORG_ID=org_sandbox_***
# OPENEVIDENCE_BASE_URL=https://api.sandbox.openevidence.com

Output

• CI workflow running on push and PR
• Unit tests with mocked SDK
• Integration tests against sandbox
• Clinical validation test suite
• Coverage reports

Package.json Scripts

{
  "scripts": {
    "test": "vitest",
    "test:unit": "vitest run -c vitest.config.ts",
    "test:integration": "vitest run -c vitest.config.integration.ts",
    "test:clinical-validation": "vitest run tests/clinical-validation",
    "test:ci": "vitest run --coverage",
    "typecheck": "tsc --noEmit",
    "lint": "eslint src tests"
  }
}

Error Handling

CI Issue	Cause	Solution
Integration test timeout	API slow or rate limited	Increase timeout, add retry
Secret not found	Missing GitHub secret	Add secret in repo settings
Flaky tests	Network variability	Add retries, improve assertions
Coverage drop	New code untested	Add tests, adjust thresholds

Resources

• GitHub Actions Documentation
• Vitest Documentation

Next Steps

For deployment integration, see openevidence-deploy-integration.