Add Playwright E2E test suite (25 tests across 4 specs)

- playwright.config.ts: headless CI setup with JSON/HTML reporters
- e2e/global-setup.ts: app reachability check before tests
- e2e/auth.spec.ts: 6 auth tests (sign-in, session, protected routes)
- e2e/compare.spec.ts: 8 API tests (validation, SSE flow, DB persistence)
- e2e/comparisons.spec.ts: 6 CRUD tests (list, detail, 404, view counts)
- e2e/user.spec.ts: 5 user tests (stats, pagination, session binding)
- e2e/helpers.ts: shared SSE parsing, auth, URL resolution utilities
- e2e/README.md: setup and run instructions
- AGENTS.md: updated with LLM provider notes and testing docs
- .gitignore: added playwright report/result directories
- package.json: added @playwright/test and test:e2e scripts
This commit is contained in:
Hermes Agent
2026-04-27 20:48:05 -07:00
parent 8f64ccd2f6
commit 26c7ad4d7b
12 changed files with 683 additions and 1 deletions

4
.gitignore vendored
View File

@@ -12,6 +12,10 @@
# testing
/coverage
/playwright-report/
/playwright-results.json
/test-results/
/playwright/.cache/
# next.js
/.next/

View File

@@ -3,3 +3,59 @@
This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` before writing any code. Heed deprecation notices.
<!-- END:nextjs-agent-rules -->
# Comparaison — Project Agent Rules
## LLM Provider
- **Base URL**: `https://nano-gpt.com/api/v1` (not `api.nano-gpt.com`)
- **API Key**: Use `NANOGPT_API_KEY` env var from the host's Docker `.env`
- **Model**: `minimax/minimax-m2.7` (supports `response_format: {type: json_object}`)
- **Schema tolerance**: The LLM output may deviate from the requested schema. The provider code (`src/lib/llm/providers/openai.ts`) includes `normalizeResult()` to handle common deviations (overallScore as object, sources as strings, dimensions as array of objects). Do NOT assume the LLM returns the exact schema.
## Development
```bash
# Local dev
npm run dev
# Build Docker image
docker compose build
# Run container
docker compose up -d
# Run E2E tests
npm install
npm run test:e2e:install
npm run test:e2e
```
## Deployment
Deployed to `ubuntu:/srv/compose/comparaison/` via `docker compose`.
Traefik routes `comparaison.local.tophermayor.com` → container port 3000.
## Environment Variables
Required in `.env`:
- `DATABASE_URL` — PostgreSQL connection string
- `BETTER_AUTH_SECRET` — Session secret (min 32 chars)
- `BETTER_AUTH_URL` — Auth callback URL
- `LLM_API_KEY` — NanoGPT API key (Hermes `NANOGPT_API_KEY` works)
- `LLM_BASE_URL``https://nano-gpt.com/api/v1`
- `LLM_MODEL``minimax/minimax-m2.7`
## Key Routes
| Route | Method | Auth | Description |
|-------|--------|------|-------------|
| `/api/compare` | POST | Yes | Start SSE comparison stream |
| `/api/comparisons/[slug]` | GET | Yes | Get comparison by slug |
| `/api/user/comparisons` | GET | Yes | List user's comparisons |
| `/api/user/stats` | GET | Yes | User statistics |
| `/api/auth/sign-in/email` | POST | No | Email/password sign-in |
## Testing
E2E tests live in `e2e/` and use Playwright. See `e2e/README.md` for setup.

76
e2e/README.md Normal file
View File

@@ -0,0 +1,76 @@
# E2E Tests
Playwright-based end-to-end tests covering auth, the compare API, comparisons CRUD, and user stats.
## Setup
```bash
# Install dependencies (includes Playwright)
npm install
# Install Chromium browser for Playwright
npm run test:e2e:install
# Copy and fill in test credentials
cp .env.test.example .env.test.local
# Edit .env.test.local with your test user credentials
```
## Running Tests
```bash
# Run all tests (headless)
npm run test:e2e
# Run with headed browser (visible)
npm run test:e2e:headed
# Run with Playwright UI (step-through debugger)
npm run test:e2e:ui
# View the HTML report from last run
npm run test:e2e:report
```
## Prerequisites
The Comparaison Docker container must be running on the target host:
```bash
ssh bear@192.168.50.61 'cd /srv/compose/comparaison && docker compose up -d'
```
## Environment Variables
| Variable | Default | Description |
|---|---|---|
| `E2E_HOST` | `192.168.50.61` | Host running the Docker container |
| `E2E_BASE_URL` | `http://localhost:3000` | App URL for browser navigation |
| `E2E_TARGET_HOST` | `192.168.50.61` | Host for API calls via Traefik |
| `E2E_TEST_EMAIL` | `admin@admin.com` | Test user email |
| `E2E_TEST_PASSWORD` | `adminpass` | Test user password |
## Test Structure
```
e2e/
global-setup.ts # Verifies app is reachable before tests
global-teardown.ts # Cleanup after tests
helpers.ts # Shared utilities (auth, SSE parsing, URL resolution)
auth.spec.ts # Authentication flows (6 tests)
compare.spec.ts # Compare API including SSE and DB persistence (8 tests)
comparisons.spec.ts # List and detail views (6 tests)
user.spec.ts # User stats and account (5 tests)
```
## CI Integration
In CI (GitHub Actions, Gitea Actions), set environment variables and run:
```bash
npm install
npm run test:e2e:install
npm run test:e2e
```
The JSON results file (`playwright-results.json`) can be uploaded as a CI artifact.

72
e2e/auth.spec.ts Normal file
View File

@@ -0,0 +1,72 @@
import { test, expect, type Page } from "@playwright/test";
import { resolveUrl, getSessionCookie, testUser } from "./helpers";
/**
* E2E tests for authentication flows.
* Covers: sign-in, sign-out, protected routes, and session persistence.
*/
test.describe("Auth", () => {
// ─── Helpers ───────────────────────────────────────────────────────────────
async function signInViaUI(page: Page, email = testUser.email, password = testUser.password) {
await page.goto(resolveUrl("/"));
await page.getByRole("link", { name: /sign in/i }).click();
await page.getByPlaceholder("you@example.com").fill(email);
await page.getByPlaceholder("your password").fill(password);
await page.getByRole("button", { name: /sign in/i }).click();
await page.waitForURL("**/");
}
// ─── Tests ────────────────────────────────────────────────────────────────
test("1. Homepage loads without auth", async ({ page }) => {
const res = await page.request.get(resolveUrl("/"));
expect(res.status()).toBeGreaterThanOrEqual(200);
expect(res.status()).toBeLessThan(400);
});
test("2. Sign in via API returns session cookie", async ({ page }) => {
const res = await page.request.post(resolveUrl("/api/auth/sign-in/email"), {
data: { email: testUser.email, password: testUser.password },
headers: { "Content-Type": "application/json" },
});
expect(res.status()).toBe(200);
const setCookie = res.headers()["set-cookie"] ?? "";
expect(setCookie).toContain("session_token=");
});
test("3. Sign in with wrong credentials returns 401", async ({ page }) => {
const res = await page.request.post(resolveUrl("/api/auth/sign-in/email"), {
data: { email: "bad@example.com", password: "wrongpass" },
headers: { "Content-Type": "application/json" },
});
expect(res.status()).toBe(401);
});
test("4. Protected API route requires auth", async ({ page }) => {
const res = await page.request.post(resolveUrl("/api/compare"), {
data: { items: ["A", "B"] },
headers: { "Content-Type": "application/json" },
});
expect(res.status()).toBe(401);
const body = await res.json();
expect(body.error).toMatch(/auth/i);
});
test("5. Authenticated session persists across requests", async ({ page }) => {
const cookie = await getSessionCookie(page);
// Should be able to hit protected endpoints with the cookie
const res = await page.request.get(resolveUrl("/api/user/stats"), {
headers: { Cookie: cookie },
});
expect(res.status()).toBe(200);
});
test("6. UI sign-in flow works", async ({ page }) => {
await signInViaUI(page);
// After sign-in, user should see their name or a dashboard link
const bodyText = await page.content();
expect(bodyText.toLowerCase()).not.toContain("sign in");
});
});

161
e2e/compare.spec.ts Normal file
View File

@@ -0,0 +1,161 @@
import { test, expect } from "@playwright/test";
import { resolveUrl, getSessionCookie, parseSSEStream, lastSSEOfType } from "./helpers";
/**
* E2E tests for the /api/compare endpoint.
* Covers: request validation, full SSE flow, success/failure states, and DB persistence.
*/
test.describe("Compare API", () => {
let cookie: string;
test.beforeEach(async ({ page }) => {
cookie = await getSessionCookie(page);
});
// ─── Input Validation ────────────────────────────────────────────────────
test("1. Rejects fewer than 2 items", async ({ page }) => {
const res = await page.request.post(resolveUrl("/api/compare"), {
data: { items: ["React"] },
headers: { "Content-Type": "application/json", Cookie: cookie },
});
expect(res.status()).toBe(400);
const body = await res.json();
expect(body.error).toMatch(/2 items|at least/i);
});
test("2. Rejects more than 10 items", async ({ page }) => {
const res = await page.request.post(resolveUrl("/api/compare"), {
data: { items: Array.from({ length: 11 }, (_, i) => `Item${i}`) },
headers: { "Content-Type": "application/json", Cookie: cookie },
});
expect(res.status()).toBe(400);
const body = await res.json();
expect(body.error).toMatch(/10|maximum/i);
});
test("3. Rejects empty item names", async ({ page }) => {
const res = await page.request.post(resolveUrl("/api/compare"), {
data: { items: ["React", " ", "Vue"] },
headers: { "Content-Type": "application/json", Cookie: cookie },
});
expect(res.status()).toBe(400);
const body = await res.json();
expect(body.error).toMatch(/empty/i);
});
test("4. Rejects unauthenticated requests", async ({ page }) => {
const res = await page.request.post(resolveUrl("/api/compare"), {
data: { items: ["React", "Vue"] },
headers: { "Content-Type": "application/json" },
});
expect(res.status()).toBe(401);
});
// ─── Happy Path ──────────────────────────────────────────────────────────
test("5. Compare returns SSE stream with progress events", async ({ page }) => {
const res = await page.request.post(resolveUrl("/api/compare"), {
data: { query: "Python vs JavaScript", items: ["Python", "JavaScript"] },
headers: {
"Content-Type": "application/json",
Cookie: cookie,
Accept: "text/event-stream",
},
});
expect(res.status()).toBe(200);
expect(res.headers()["content-type"]).toContain("text/event-stream");
const body = await res.body();
if (!body) { test.skip(); return; }
const bodyStr = Buffer.from(body).toString("utf-8");
const events = await parseSSEStream(bodyStr);
// Should have at least one progress event
const progressEvents = events.filter((e) => e.status !== undefined);
expect(progressEvents.length).toBeGreaterThan(0);
});
test("6. Full compare flow completes successfully", async ({ page }) => {
const res = await page.request.post(resolveUrl("/api/compare"), {
data: { query: "React vs Vue", items: ["React", "Vue"] },
headers: {
"Content-Type": "application/json",
Cookie: cookie,
Accept: "text/event-stream",
},
timeout: 120_000,
});
expect(res.status()).toBe(200);
const body = Buffer.from((await res.body()) ?? []).toString("utf-8");
const events = await parseSSEStream(body);
// Must have a final status
const finalProgress = lastSSEOfType(events, "progress");
expect(finalProgress).toBeDefined();
expect(["completed", "failed"]).toContain(finalProgress!.status);
if (finalProgress!.status === "completed") {
const doneEvent = lastSSEOfType(events, "done") as Record<string, unknown> | undefined;
expect(doneEvent).toBeDefined();
const doneData = doneEvent!.done as Record<string, unknown>;
expect(doneData).toHaveProperty("id");
expect(doneData).toHaveProperty("slug");
expect((doneData.data as Record<string, unknown>)).toHaveProperty("summary");
expect((doneData.data as Record<string, unknown>)).toHaveProperty("items");
}
});
test("7. Comparison is persisted to DB after completion", async ({ page }) => {
// Run a comparison
const res = await page.request.post(resolveUrl("/api/compare"), {
data: { query: "Rust vs Go", items: ["Rust", "Go"] },
headers: { "Content-Type": "application/json", Cookie: cookie, Accept: "text/event-stream" },
timeout: 120_000,
});
expect(res.status()).toBe(200);
const body = Buffer.from((await res.body()) ?? []).toString("utf-8");
const events = await parseSSEStream(body);
const doneEvent = lastSSEOfType(events, "done") as Record<string, unknown> | undefined;
if (!doneEvent) { test.skip(); return; }
const { slug } = doneEvent.done as { slug: string };
expect(slug).toBeTruthy();
// Fetch it back
const getRes = await page.request.get(resolveUrl(`/api/comparisons/${slug}`), {
headers: { Cookie: cookie },
});
expect(getRes.status()).toBe(200);
const comparison = await getRes.json();
expect(comparison.slug).toBe(slug);
expect(comparison.status).toBe("completed");
});
test("8. Compare with custom dimensions", async ({ page }) => {
const res = await page.request.post(resolveUrl("/api/compare"), {
data: {
query: "React vs Angular",
items: ["React", "Angular"],
dimensions: ["Performance", "Learning Curve", "Ecosystem"],
},
headers: {
"Content-Type": "application/json",
Cookie: cookie,
Accept: "text/event-stream",
},
timeout: 120_000,
});
expect(res.status()).toBe(200);
const body = Buffer.from((await res.body()) ?? []).toString("utf-8");
const events = await parseSSEStream(body);
const finalProgress = lastSSEOfType(events, "progress");
expect(finalProgress).toBeDefined();
expect(["completed", "failed"]).toContain(finalProgress!.status);
});
});

95
e2e/comparisons.spec.ts Normal file
View File

@@ -0,0 +1,95 @@
import { test, expect } from "@playwright/test";
import { resolveUrl, getSessionCookie, parseSSEStream, lastSSEOfType } from "./helpers";
/**
* E2E tests for comparison list and detail views.
* Covers: /api/comparisons/[slug], /api/user/comparisons
*/
test.describe("Comparisons", () => {
let cookie: string;
let testSlug: string;
test.beforeAll(async ({ page }) => {
cookie = await getSessionCookie(page);
// Create a comparison to ensure we have data
const res = await page.request.post(resolveUrl("/api/compare"), {
data: { query: "Cats vs Dogs", items: ["Cats", "Dogs"] },
headers: { "Content-Type": "application/json", Cookie: cookie, Accept: "text/event-stream" },
timeout: 120_000,
});
if (res.ok()) {
const body = Buffer.from((await res.body()) ?? []).toString("utf-8");
const events = await parseSSEStream(body);
const done = lastSSEOfType(events, "done") as Record<string, unknown> | undefined;
if (done) {
testSlug = (done.done as Record<string, unknown>).slug as string;
}
}
});
test("1. User comparisons list returns 200 for authenticated user", async ({ page }) => {
const res = await page.request.get(resolveUrl("/api/user/comparisons"), {
headers: { Cookie: cookie },
});
expect(res.status()).toBe(200);
const data = await res.json();
expect(Array.isArray(data)).toBe(true);
});
test("2. User comparisons list requires auth", async ({ page }) => {
const res = await page.request.get(resolveUrl("/api/user/comparisons"));
expect(res.status()).toBe(401);
});
test("3. Get comparison by slug returns 200 with data", async ({ page }) => {
if (!testSlug) { test.skip(); return; }
const res = await page.request.get(resolveUrl(`/api/comparisons/${testSlug}`), {
headers: { Cookie: cookie },
});
expect(res.status()).toBe(200);
const data = await res.json();
expect(data).toHaveProperty("slug", testSlug);
expect(data).toHaveProperty("status");
expect(["completed", "failed", "researching"]).toContain(data.status);
});
test("4. Get comparison by non-existent slug returns 404", async ({ page }) => {
const res = await page.request.get(resolveUrl("/api/comparisons/this-slug-does-not-exist-xyz123"), {
headers: { Cookie: cookie },
});
expect(res.status()).toBe(404);
});
test("5. Comparison slug increments view count", async ({ page }) => {
if (!testSlug) { test.skip(); return; }
const res1 = await page.request.get(resolveUrl(`/api/comparisons/${testSlug}`), {
headers: { Cookie: cookie },
});
const viewsBefore = (await res1.json()).viewCount ?? 0;
const res2 = await page.request.get(resolveUrl(`/api/comparisons/${testSlug}`), {
headers: { Cookie: cookie },
});
const viewsAfter = (await res2.json()).viewCount ?? 0;
expect(viewsAfter).toBeGreaterThanOrEqual(viewsBefore);
});
test("6. Comparison response includes all required fields", async ({ page }) => {
if (!testSlug) { test.skip(); return; }
const res = await page.request.get(resolveUrl(`/api/comparisons/${testSlug}`), {
headers: { Cookie: cookie },
});
const data = await res.json();
expect(data).toHaveProperty("id");
expect(data).toHaveProperty("title");
expect(data).toHaveProperty("slug");
expect(data).toHaveProperty("status");
expect(data).toHaveProperty("query");
});
});

32
e2e/global-setup.ts Normal file
View File

@@ -0,0 +1,32 @@
import type { FullConfig } from "@playwright/test";
import { baseURL } from "./playwright.config";
/**
* Global setup — runs once before all tests.
* Verifies the app is reachable before running E2E suite.
*/
export default async function globalSetup(_config: FullConfig) {
const url = baseURL.replace("localhost", process.env.E2E_HOST || "192.168.50.61");
console.log(`[E2E Setup] Checking app at: ${url}`);
let attempts = 0;
while (attempts < 10) {
try {
const res = await fetch(url, { signal: AbortSignal.timeout(5000) });
if (res.ok || res.status === 401) {
// 401 means the server is up (auth is working), 200 means homepage
console.log(`[E2E Setup] App reachable (HTTP ${res.status})`);
return;
}
} catch {
// ignore and retry
}
attempts++;
await new Promise((r) => setTimeout(r, 2000));
}
throw new Error(
`[E2E Setup] App not reachable at ${url} after ${attempts} attempts. ` +
"Ensure the Docker container is running: `docker compose up -d`"
);
}

9
e2e/global-teardown.ts Normal file
View File

@@ -0,0 +1,9 @@
import type { FullConfig } from "@playwright/test";
/**
* Global teardown — runs once after all tests finish.
* No-op placeholder for cleanup hooks.
*/
export default async function globalTeardown(_config: FullConfig) {
// Add cleanup logic here if needed (e.g., stop test containers)
}

53
e2e/helpers.ts Normal file
View File

@@ -0,0 +1,53 @@
import { type Page, type Request, type Response, expect } from "@playwright/test";
import { baseURL, targetHost, testUser } from "../playwright.config";
/** Resolve URL relative to the target host (bypasses localhost resolution). */
export function resolveUrl(path: string): string {
const base = baseURL.replace("localhost", targetHost);
return `${base}${path.startsWith("/") ? path : "/" + path}`;
}
/** Sign in as the test user via the API and return the session cookie. */
export async function getSessionCookie(page: Page): Promise<string> {
const res = await page.request.post(resolveUrl("/api/auth/sign-in/email"), {
data: { email: testUser.email, password: testUser.password },
headers: { "Content-Type": "application/json" },
});
if (!res.ok()) {
throw new Error(`Sign-in failed: ${res.status()} ${await res.text()}`);
}
const setCookie = res.headers()["set-cookie"] ?? "";
const match = setCookie.match(/session_token=([^;]+)/);
if (!match) throw new Error("No session_token cookie in sign-in response");
return `session_token=${match[1]}`;
}
/** Parse SSE stream from a fetch response body. */
export async function parseSSEStream(body: string): Promise<Record<string, unknown>[]> {
const events: Record<string, unknown>[] = [];
for (const line of body.split("\n")) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith("event:")) continue;
if (trimmed.startsWith("data:")) {
const dataStr = trimmed.slice(5).trim();
try {
events.push(JSON.parse(dataStr));
} catch {
// skip malformed
}
}
}
return events;
}
/** Extract the last SSE event of a given type. */
export function lastSSEOfType(
events: Record<string, unknown>[],
type: "progress" | "done"
): Record<string, unknown> | undefined {
return events.reverse().find((e) => e[type] !== undefined);
}
// Re-export common helpers
export { expect, resolveUrl as url, testUser };
export type { Page, Request, Response };

77
e2e/user.spec.ts Normal file
View File

@@ -0,0 +1,77 @@
import { test, expect } from "@playwright/test";
import { resolveUrl, getSessionCookie } from "./helpers";
/**
* E2E tests for user endpoints: stats, profile, and account management.
*/
test.describe("User API", () => {
let cookie: string;
test.beforeEach(async ({ page }) => {
cookie = await getSessionCookie(page);
});
test("1. User stats returns 200 for authenticated user", async ({ page }) => {
const res = await page.request.get(resolveUrl("/api/user/stats"), {
headers: { Cookie: cookie },
});
expect(res.status()).toBe(200);
const data = await res.json();
expect(data).toHaveProperty("totalComparisons");
expect(data).toHaveProperty("totalViews");
expect(typeof data.totalComparisons).toBe("number");
expect(typeof data.totalViews).toBe("number");
});
test("2. User stats requires auth", async ({ page }) => {
const res = await page.request.get(resolveUrl("/api/user/stats"));
expect(res.status()).toBe(401);
});
test("3. User stats increments after new comparison", async ({ page }) => {
const beforeRes = await page.request.get(resolveUrl("/api/user/stats"), {
headers: { Cookie: cookie },
});
const before = await beforeRes.json();
const countBefore = before.totalComparisons ?? 0;
// Create a new comparison
const createRes = await page.request.post(resolveUrl("/api/compare"), {
data: { items: ["TypeScript", "JavaScript"] },
headers: {
"Content-Type": "application/json",
Cookie: cookie,
Accept: "text/event-stream",
},
timeout: 120_000,
});
if (createRes.status() === 200) {
// Wait briefly for DB write
await new Promise((r) => setTimeout(r, 2000));
const afterRes = await page.request.get(resolveUrl("/api/user/stats"), {
headers: { Cookie: cookie },
});
const after = await afterRes.json();
expect(after.totalComparisons).toBeGreaterThanOrEqual(countBefore);
}
});
test("4. User comparisons list pagination works", async ({ page }) => {
const res = await page.request.get(resolveUrl("/api/user/comparisons?page=1&limit=5"), {
headers: { Cookie: cookie },
});
expect(res.status()).toBe(200);
});
test("5. Auth session is bound to correct user", async ({ page }) => {
const res = await page.request.get(resolveUrl("/api/user/stats"), {
headers: { Cookie: cookie },
});
const data = await res.json();
// Stats should be non-negative integers
expect(data.totalComparisons).toBeGreaterThanOrEqual(0);
expect(data.totalViews).toBeGreaterThanOrEqual(0);
});
});

View File

@@ -6,7 +6,12 @@
"dev": "next dev",
"build": "next build",
"start": "next start",
"lint": "eslint"
"lint": "eslint",
"test:e2e": "playwright test",
"test:e2e:ui": "playwright test --ui",
"test:e2e:headed": "playwright test --headed",
"test:e2e:report": "playwright show-report",
"test:e2e:install": "playwright install chromium"
},
"dependencies": {
"@base-ui/react": "^1.4.1",
@@ -27,6 +32,7 @@
"tw-animate-css": "^1.4.0"
},
"devDependencies": {
"@playwright/test": "^1.52.0",
"@tailwindcss/postcss": "^4",
"@types/node": "^20",
"@types/react": "^19",

41
playwright.config.ts Normal file
View File

@@ -0,0 +1,41 @@
import { defineConfig, devices } from "@playwright/test";
const baseURL = process.env.E2E_BASE_URL || "http://localhost:3000";
const targetHost = process.env.E2E_TARGET_HOST || "192.168.50.61";
const testUser = {
email: process.env.E2E_TEST_EMAIL || "admin@admin.com",
password: process.env.E2E_TEST_PASSWORD || "adminpass",
};
export default defineConfig({
testDir: "./e2e",
fullyParallel: false,
forbidOnly: !!process.env.CI,
retries: process.env.CI ? 2 : 0,
workers: 1,
reporter: [
["list"],
["html", { open: "never", outputFolder: "playwright-report" }],
["json", { outputFile: "playwright-results.json" }],
],
use: {
baseURL,
trace: "on-first-retry",
screenshot: "only-on-failure",
video: "retain-on-failure",
},
globalSetup: "./e2e/global-setup.ts",
globalTeardown: "./e2e/global-teardown.ts",
projects: [
{
name: "chromium",
use: { ...devices["Desktop Chrome"] },
},
],
timeout: 120_000,
expect: {
timeout: 10_000,
},
});
export { baseURL, targetHost, testUser };