Feat: E2E tests for AI assistant and log drains (#40844)

* updated commands and expose ai key locally

* added tests for AI assistant

* added OPEN_API_KEY for e2e test suite

* updated log drain options

* updated README

.github/workflows/studio-e2e-test.yml

@@ -20,6 +20,9 @@ jobs:
     # Require approval only for pull requests from forks
     environment: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork && 'Studio E2E Tests' || '' }}
 
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
       - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2

e2e/studio/README.md

@@ -16,6 +16,12 @@ cd e2e/studio
 pnpm exec playwright install
 ```
 
+### Environment Variables
+
+Some tests require specific environment variables to be set. If these are not set, the tests will be automatically skipped:
+
+- **`OPENAI_API_KEY`**: Required for the AI Assistant test (`assistant.spec.ts`). Without this variable, the assistant test will be skipped.
+
 ---
 
 ## Running the tests

e2e/studio/features/assistant.spec.ts

@@ -0,0 +1,46 @@
+import { expect } from '@playwright/test'
+import { test } from '../utils/test.js'
+import { toUrl } from '../utils/to-url.js'
+
+test.describe('AI Assistant', async () => {
+  test('Can send a message to the assistant and receive a response', async ({ page, ref }) => {
+    // Skip the test if the OPENAI_API_KEY is not set
+    test.skip(!process.env.OPENAI_API_KEY, 'OPENAI_API_KEY is not set')
+
+    await page.goto(toUrl(`/project/${ref}`))
+
+    // Wait for the page to load
+    await expect(page.getByRole('heading', { level: 1 })).toBeVisible()
+
+    // Click the assistant button to open the assistant panel
+    await page.locator('#assistant-trigger').click()
+
+    // Wait for the assistant panel to be visible
+    await expect(page.getByRole('heading', { name: 'How can I assist you?' })).toBeVisible()
+
+    // Type "hello" in the chat input
+    const chatInput = page.getByRole('textbox', { name: 'Chat to Postgres...' })
+    await chatInput.fill('hello')
+
+    const responsePromise = page.waitForResponse(
+      (response) =>
+        response.url().includes('/api/ai/sql/generate-v4') &&
+        response.request().method() === 'POST',
+      { timeout: 60000 }
+    )
+
+    // Click the send message button
+    const sendButton = page.getByRole('button', { name: 'Send message' })
+    await sendButton.click()
+
+    // Wait for the API request to complete
+    const response = await responsePromise
+
+    // Verify the response was successful
+    expect(response.status()).toBe(200)
+
+    // AI response has values
+    const body = await response.text()
+    expect(body).toContain('data')
+  })
+})

e2e/studio/features/log-drains.spec.ts

@@ -0,0 +1,77 @@
+import { expect } from '@playwright/test'
+import { test } from '../utils/test.js'
+import { toUrl } from '../utils/to-url.js'
+
+const LOG_DRAIN_OPTIONS = [
+  {
+    name: 'Custom Endpoint',
+    buttonText: 'Custom Endpoint Forward logs',
+  },
+  {
+    name: 'Datadog',
+    buttonText: 'Datadog Datadog is a',
+  },
+  {
+    name: 'Loki',
+    buttonText: 'Loki Loki is an open-source',
+  },
+]
+
+test.describe('Log Drains Settings', () => {
+  test.beforeEach(async ({ page, ref }) => {
+    // Navigate to the log drains settings page
+    await page.goto(toUrl(`/project/${ref}/settings/log-drains`))
+
+    // Wait for the page to load
+    await expect(page.getByRole('heading', { name: 'Log Drains', level: 1 }), {
+      message: 'Log Drains heading should be visible',
+    }).toBeVisible()
+  })
+
+  for (const option of LOG_DRAIN_OPTIONS) {
+    test(`Opens ${option.name} panel when clicked`, async ({ page }) => {
+      // Click on the log drain option button
+      const optionButton = page.getByRole('button', { name: option.buttonText })
+      await expect(optionButton, {
+        message: `${option.name} button should be visible`,
+      }).toBeVisible()
+
+      await optionButton.click()
+
+      // Verify that the "Add destination" dialog opens
+      const dialog = page.getByRole('dialog', { name: 'Add destination' })
+      await expect(dialog, {
+        message: `Add destination dialog should be visible for ${option.name}`,
+      }).toBeVisible()
+
+      // Verify the dialog heading
+      await expect(dialog.getByRole('heading', { name: 'Add destination', level: 2 }), {
+        message: 'Dialog heading should be visible',
+      }).toBeVisible()
+
+      // Verify that the Type field shows the correct option
+      const typeCombobox = dialog.getByRole('combobox').first()
+      await expect(typeCombobox, {
+        message: `Type combobox should contain ${option.name}`,
+      }).toContainText(option.name)
+
+      // Close the dialog by pressing Escape
+      await page.keyboard.press('Escape')
+
+      // Verify the dialog is closed
+      await expect(dialog, {
+        message: 'Dialog should be hidden after pressing Escape',
+      }).not.toBeVisible()
+    })
+  }
+
+  test('All log drain options are visible on the page', async ({ page }) => {
+    // Verify all three options are displayed
+    for (const option of LOG_DRAIN_OPTIONS) {
+      const optionButton = page.getByRole('button', { name: option.buttonText })
+      await expect(optionButton, {
+        message: `${option.name} option should be visible`,
+      }).toBeVisible()
+    }
+  })
+})

package.json

@@ -29,7 +29,7 @@
     "test:ui-patterns": "turbo run test --filter=ui-patterns",
     "test:studio": "turbo run test --filter=studio",
     "test:studio:watch": "turbo run test --filter=studio -- watch",
-    "e2e:setup:cli": "supabase start --exclude studio && supabase db reset && supabase status --output json > keys.json && node scripts/generateLocalEnv.js",
+    "e2e:setup:cli": "supabase stop --all --no-backup ; supabase start --exclude studio && supabase db reset && supabase status --output json > keys.json && node scripts/generateLocalEnv.js",
     "e2e:setup": "SKIP_ASSET_UPLOAD=1 pnpm e2e:setup:cli && NODE_ENV=test NODE_OPTIONS=\"--max-old-space-size=4096\" pnpm run build:studio && NODE_ENV=test pnpm --prefix ./apps/studio start --port 8082",
     "e2e": "pnpm --prefix e2e/studio run e2e",
     "e2e:ui": "pnpm --prefix e2e/studio run e2e:ui",

supabase/config.toml

@@ -31,6 +31,8 @@ major_version = 15
 # Port to use for Supabase Studio.
 port = 54323
 
+openai_api_key = "env(OPENAI_API_KEY)"
+
 # Email testing server. Emails sent with the local dev setup are not actually sent - rather, they
 # are monitored, and you can view the emails that would have been sent from the web interface.
 [inbucket]