feat: Run E2E tests aganist Platform Pt.1 (#41032)

* added packages for creating projects

* updated scripts

* remove ami version

* cleaned up common

* updated tests

* refactored helpers

* updated env

* updated config

* updated to reference env

* updated global setup

* updated type logic and scripts

* added mocking of hcaptcha

* added log statements

* updated local env

* update env file

* updated env vars

* updated logging

* updated to remove check

* updated print and project names

* updated helpers

* updated url

* updated setup

* updated storage helpers to account for listing files

* updated setup and tests

* updated timeout only for setup

* updated helper to account for different api response

* added ignores for tests

* updated lock file

* updated database spec to add exact

* updated timeouts

* removed check for table grid footer

* updated test runner

* updated is_platform

* updated playwright config

* updated worker settings

* removed dotenvx

* updated README

* updated to remove comment

* Update e2e/studio/scripts/common/retriedFetch.ts

Co-authored-by: Charis <26616127+charislam@users.noreply.github.com>

---------

Co-authored-by: Charis <26616127+charislam@users.noreply.github.com>
This commit is contained in:
Ali Waseem
2025-12-05 10:50:16 -07:00
committed by GitHub
parent b9908aec55
commit 22cfd2be14
29 changed files with 720 additions and 220 deletions

View File

@@ -0,0 +1,14 @@
# Copy and paste this file and rename it to .env.local
# Required for self hosted tests
# Mise infra running
STUDIO_URL=http://localhost:8082
API_URL=http://localhost:8080
# Required for platform tests
IS_PLATFORM=true
ORG_SLUG=
SUPA_PAT=
EMAIL=
PASSWORD=

View File

@@ -1,9 +1,9 @@
# Copy and paste this file and rename it to .env.local
# Required for self hosted tests
STUDIO_URL=http://localhost:8082
API_URL=http://127.0.0.1:54321
IS_PLATFORM=false
# Used to run e2e tests against vercel previews
VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO=
# Required for platform tests
IS_PLATFORM=false

View File

@@ -2,10 +2,27 @@
## Set up
### Prerequisites
#### For Self-Hosted Tests
- Nothing is required, running with IS_PLATFORM=false should run the tests locally with a self hosted docker container
#### For Platform Tests
1. **Create a platform account** with an email and password, these auths are used for the test
2. **Create an organization** on the platform, this can be done if run locally through `mise fullstack`
3. **Generate a Personal Access Token (PAT)** for API access
4. Configure the environment variables below
### Configure Environment
```bash
cp .env.local.example .env.local
```
Edit `.env.local` and set the appropriate values based on your test environment (see Environment Variables section below).
### Install the playwright browser
⚠️ This should be done in the `e2e/studio` directory
@@ -18,9 +35,45 @@ pnpm exec playwright install
### Environment Variables
Some tests require specific environment variables to be set. If these are not set, the tests will be automatically skipped:
Configure your tests by setting the following environment variables in `.env.local`. We have examples of what required on self hosted and platform:
#### Core Configuration
- **`STUDIO_URL`**: The URL where Studio is running (default: `http://localhost:8082`)
- **`API_URL`**: The Supabase API endpoint (default: `https://localhost:8080`)
- **`IS_PLATFORM`**: Set to `true` for platform tests, `false` for self-hosted (default: `false`)
- When `true`: Tests run serially (1 worker) due to API rate limits
- When `false`: Tests run in parallel (5 workers)
#### Authentication (Required for Platform Tests)
⚠️ **Before running platform tests, you must create an account with an email, password, and organization on the platform you're testing.**
- **`EMAIL`**: Your platform account email (required for authentication)
- **`PASSWORD`**: Your platform account password (required for authentication)
- **`PROJECT_REF`**: Project reference (optional, will be auto-created if not provided)
When both `EMAIL` and `PASSWORD` are set, authentication is automatically enabled. HCaptcha is mocked during test setup.
#### Platform-Specific Variables (Required when `IS_PLATFORM=true`)
- **`ORG_SLUG`**: Organization slug (default: `default`)
- **`SUPA_REGION`**: Supabase region (default: `us-east-1`)
- **`SUPA_PAT`**: Personal Access Token for API authentication (default: `test`)
- **`BRANCH_NAME`**: Name for the test branch/project (default: `e2e-test-local`)
#### Optional Variables
- **`OPENAI_API_KEY`**: Required for the AI Assistant test (`assistant.spec.ts`). Without this variable, the assistant test will be skipped.
- **`VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO`**: Bypass token for Vercel protection (default: `false`)
#### Setup Commands Based on Configuration
The test setup automatically runs different commands based on your environment:
- **Platform + Localhost** (`IS_PLATFORM=true` and `STUDIO_URL=localhost`): Runs `pnpm run e2e:setup:platform`
- **Platform + Remote** (`IS_PLATFORM=true` and remote `STUDIO_URL`): No web server setup
- **Self-hosted** (`IS_PLATFORM=false`): Runs `pnpm run e2e:setup:selfhosted`
---

View File

@@ -1,5 +1,12 @@
import dotenv from 'dotenv'
import path from 'path'
// Load .env.local before reading process.env
dotenv.config({
path: path.resolve(import.meta.dirname, '.env.local'),
override: true,
})
const toBoolean = (value?: string) => {
if (value == null) return false
const normalized = value.trim().toLowerCase()
@@ -7,15 +14,26 @@ const toBoolean = (value?: string) => {
}
export const env = {
STUDIO_URL: process.env.STUDIO_URL,
API_URL: process.env.API_URL || 'https://api.supabase.green',
AUTHENTICATION: toBoolean(process.env.AUTHENTICATION),
STUDIO_URL: process.env.STUDIO_URL || 'http://localhost:8082',
API_URL: process.env.API_URL || 'https://localhost:8080',
IS_PLATFORM: toBoolean(process.env.IS_PLATFORM || 'false'),
EMAIL: process.env.EMAIL,
PASSWORD: process.env.PASSWORD,
PROJECT_REF: process.env.PROJECT_REF || 'default',
IS_PLATFORM: process.env.IS_PLATFORM || 'false',
PROJECT_REF: process.env.PROJECT_REF || undefined,
VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO:
process.env.VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO || 'false',
ORG_SLUG: process.env.ORG_SLUG || 'default',
SUPA_REGION: process.env.SUPA_REGION || 'us-east-1',
SUPA_PAT: process.env.SUPA_PAT || 'test',
BRANCH_NAME: process.env.BRANCH_NAME || `e2e-test-local`,
AUTHENTICATION: Boolean(process.env.EMAIL && process.env.PASSWORD),
IS_APP_RUNNING_ON_LOCALHOST:
process.env.STUDIO_URL?.includes('localhost') || process.env.STUDIO_URL?.includes('127.0.0.1'),
}
export const STORAGE_STATE_PATH = path.join(import.meta.dirname, './playwright/.auth/user.json')

View File

@@ -1,79 +0,0 @@
import { expect } from '@playwright/test'
import { isEnv } from '../env.config'
import { test } from '../utils/test'
/**
* * Example tests for Studio.
* Tips:
* - Use the test utility instead of playwrights test.
* import { test } from '../utils/test'
* - Use the isEnv utility to check the environment.
* import { isEnv } from '../env.config'
* - Make tests easy to debug by adding enough expect() statements.
*/
/**
* * Test that is skipped in self-hosted environment
*/
test('Loads the page 1', async ({ page }) => {
if (isEnv('selfhosted')) return
await page.goto('https://www.supabase.com')
await expect(
page.getByRole('heading', { name: 'Build in a weekend Scale to millions' })
).toBeVisible()
})
/**
* * Test that only runs in staging and production environments
*/
test('Loads the page 2', async ({ page }) => {
if (!isEnv(['staging', 'production'])) return
await page.goto('https://www.supabase.com')
await expect(
page.getByRole('heading', { name: 'Build in a weekend Scale to millions' })
).toBeVisible()
})
/**
* * Test that navigates to a project by ref
* Make sure to set up the project in the `.env.local` file.
*/
test('Navigates to a project by ref', async ({ page, ref }) => {
await page.goto(`${process.env.BASE_URL}/project/${ref}`)
await expect(page.getByRole('heading', { name: 'Project Home' })).toBeVisible()
})
/**
* * Test that mocks some API calls
*/
const mockRes = {
data: [
{
id: 1,
name: 'John Doe',
email: 'john.doe@example.com',
},
{
id: 2,
name: 'Jane Doe',
email: 'jane.doe@example.com',
},
],
}
test.beforeEach(async ({ context }) => {
context.route('*/**/users*', async (route, request) => {
await route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify(mockRes),
})
})
})
test('Mocks some API calls', async ({ page }) => {
// ... Run some code that depends on that API call
})

View File

@@ -2,6 +2,7 @@ import { expect, test as setup } from '@playwright/test'
import dotenv from 'dotenv'
import path from 'path'
import { env, STORAGE_STATE_PATH } from '../env.config.js'
import { setupProjectForTests } from '../scripts/setup-platform-tests.js'
/**
* Run any setup tasks for the tests.
@@ -14,14 +15,13 @@ dotenv.config({
})
const IS_PLATFORM = process.env.IS_PLATFORM
const envHasAuth = env.AUTHENTICATION
const doAuthentication = env.AUTHENTICATION
setup('Global Setup', async ({ page }) => {
console.log(`\n 🧪 Setting up test environment.
- Studio URL: ${env.STUDIO_URL}
- API URL: ${env.API_URL}
- Auth: ${envHasAuth ? 'enabled' : 'disabled'}
- Auth: ${doAuthentication ? 'enabled' : 'disabled'}
- Is Platform: ${IS_PLATFORM}
`)
@@ -66,22 +66,95 @@ To start API locally, run:
console.log(`\n ✅ API is running at ${apiUrl}`)
/**
* Setup Project for tests
*/
const projectRef = await setupProjectForTests()
process.env.PROJECT_REF = projectRef
env.PROJECT_REF = projectRef
/**
* Only run authentication if the environment requires it
*/
if (!env.AUTHENTICATION) {
if (!doAuthentication) {
console.log(`\n 🔑 Skipping authentication for ${env.STUDIO_URL}`)
return
} else {
if (!env.EMAIL || !env.PASSWORD || !env.PROJECT_REF) {
console.error(`Missing environment variables. Check README.md for more information.`)
throw new Error('Missing environment variables')
}
}
const signInUrl = `${studioUrl}/sign-in`
console.log(`\n 🔑 Navigating to sign in page: ${signInUrl}`)
await page.addInitScript(() => {
;(window as any).hcaptcha = {
execute: async (options?: any) => {
console.log('HCaptcha execute called (init script)', options)
// Return HCaptcha's official test token
return { response: '10000000-aaaa-bbbb-cccc-000000000001', key: 'mock' }
},
render: (container: any, options: any) => {
console.log('HCaptcha render called (init script)', container, options)
return 'mock-widget-id'
},
reset: (widgetId?: any) => {
console.log('HCaptcha reset called (init script)', widgetId)
},
remove: (widgetId?: any) => {
console.log('HCaptcha remove called (init script)', widgetId)
},
getResponse: (widgetId?: any) => {
console.log('HCaptcha getResponse called (init script)', widgetId)
return '10000000-aaaa-bbbb-cccc-000000000001'
},
}
})
// Mock HCaptcha to bypass captcha verification in automated tests
// HCaptcha detects automated browsers and will block Playwright
// Also fixes CORS issues with custom Vercel headers being sent to hcaptcha.com
await page.route('**/*hcaptcha.com/**', async (route) => {
const url = route.request().url()
console.log(`\n 🔒 Intercepting HCaptcha request: ${url}`)
// Mock the main hcaptcha script with a stub that auto-resolves
if (url.includes('api.js') || url.includes('hcaptcha.js')) {
console.log(`\n ✅ Mocking HCaptcha script`)
await route.fulfill({
status: 200,
contentType: 'application/javascript',
body: `
console.log('HCaptcha mock loaded from route');
window.hcaptcha = window.hcaptcha || {
execute: async (options) => {
console.log('HCaptcha execute called', options);
return { response: '10000000-aaaa-bbbb-cccc-000000000001', key: 'mock' };
},
render: (container, options) => {
console.log('HCaptcha render called', container, options);
return 'mock-widget-id';
},
reset: (widgetId) => {
console.log('HCaptcha reset called', widgetId);
},
remove: (widgetId) => {
console.log('HCaptcha remove called', widgetId);
},
getResponse: (widgetId) => {
console.log('HCaptcha getResponse called', widgetId);
return '10000000-aaaa-bbbb-cccc-000000000001';
}
};
`,
})
} else {
// For other hcaptcha requests, return success
await route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ success: true }),
})
}
})
await page.goto(signInUrl, { waitUntil: 'networkidle' })
await page.waitForLoadState('domcontentloaded')
await page.waitForLoadState('networkidle')
@@ -126,7 +199,7 @@ To start API locally, run:
// Wait for form elements with increased timeout
const emailInput = page.getByLabel('Email')
const passwordInput = page.getByLabel('Password')
const passwordInput = page.locator('input[type="password"]')
const signInButton = page.getByRole('button', { name: 'Sign In' })
// if found click opt out on telemetry
@@ -145,11 +218,63 @@ To start API locally, run:
await passwordInput.waitFor({ state: 'visible', timeout: 15000 })
await signInButton.waitFor({ state: 'visible', timeout: 15000 })
// Listen for console messages to debug issues
page.on('console', (msg) => {
const type = msg.type()
if (type === 'error' || type === 'warning') {
console.log(`\n 🔍 Browser ${type}: ${msg.text()}`)
}
})
// Track network requests to see what's happening
const authRequests: string[] = []
page.on('request', (request) => {
const url = request.url()
if (url.includes('auth') || url.includes('sign-in') || url.includes('password')) {
authRequests.push(`${request.method()} ${url}`)
console.log(`\n 📡 Auth request: ${request.method()} ${url}`)
}
})
page.on('response', async (response) => {
const url = response.url()
if (url.includes('auth') || url.includes('sign-in') || url.includes('password')) {
const status = response.status()
console.log(`\n 📨 Auth response: ${status} ${url}`)
if (status >= 400) {
try {
const body = await response.text()
console.log(`\n ❌ Error response body: ${body}`)
} catch (e) {
// ignore
}
}
}
})
await emailInput.fill(auth.email ?? '')
await passwordInput.fill(auth.password ?? '')
console.log(`\n 🔐 Submitting sign-in form...`)
await signInButton.click()
await page.waitForURL('**/organizations')
// Wait for successful sign-in by checking we've navigated away from sign-in page
// Could redirect to /organizations, /org/[slug], /new, or /project/default depending on configuration
try {
await page.waitForURL((url) => !url.pathname.includes('/sign-in'), {
timeout: 30_000,
})
console.log(`\n ✅ Successfully signed in, redirected to: ${page.url()}`)
} catch (error) {
console.log(`\n ❌ Sign-in timeout. Current URL: ${page.url()}`)
console.log(`\n 📡 Auth requests made: ${authRequests.join(', ')}`)
// Take a screenshot for debugging
await page.screenshot({ path: 'test-results/sign-in-failure.png', fullPage: true })
console.log(`\n 📸 Screenshot saved to test-results/sign-in-failure.png`)
throw error
}
await page.context().storageState({ path: STORAGE_STATE_PATH })
})

View File

@@ -54,8 +54,8 @@ const createTable = async (page: Page, tableName: string, newColumnName: string)
}
const deleteTable = async (page: Page, tableName: string) => {
await page.getByLabel(`View ${tableName}`).nth(0).click()
await page.getByLabel(`View ${tableName}`).getByRole('button').nth(1).click()
await page.getByLabel(`View ${tableName}`, { exact: true }).nth(0).click()
await page.getByLabel(`View ${tableName}`, { exact: true }).getByRole('button').nth(1).click()
await page.getByText('Delete table').click()
await page.getByRole('checkbox', { name: 'Drop table with cascade?' }).click()
await page.getByRole('button', { name: 'Delete' }).click()
@@ -443,8 +443,9 @@ test.describe.serial('Database', () => {
// Wait for database triggers to be populated
await waitForApiResponse(page, 'pg-meta', ref, 'triggers')
const newTriggerButton = page.getByRole('button', { name: 'New trigger' }).first()
// create new trigger button to exist in public schema
await expect(page.getByRole('button', { name: 'New trigger' })).toBeVisible()
await expect(newTriggerButton).toBeVisible()
// change schema -> realtime
await page.getByTestId('schema-selector').click()
@@ -480,7 +481,7 @@ test.describe.serial('Database', () => {
}
// create new trigger
await page.getByRole('button', { name: 'New trigger' }).click()
await page.getByRole('button', { name: 'New trigger' }).first().click()
await page.getByRole('textbox', { name: 'Name of trigger' }).fill(databaseTriggerName)
await page.getByRole('combobox').first().click()
await page.getByRole('option', { name: `public.${databaseTableName}`, exact: true }).click()
@@ -563,15 +564,21 @@ test.describe.serial('Database', () => {
await page.getByTestId('schema-selector').click()
await page.getByPlaceholder('Find schema...').fill('auth')
await page.getByRole('option', { name: 'auth' }).click()
await page.waitForTimeout(500)
expect(page.getByText('sso_providers_pkey')).toBeVisible()
expect(page.getByText('confirmation_token_idx')).toBeVisible()
await page.waitForTimeout(2000)
const ssoProvidersPkeyRow = page.getByRole('row', { name: 'sso_providers_pkey' })
const confirmationTokenIdxRow = page.getByRole('row', { name: 'confirmation_token_idx' })
const createIndexButton = page.getByRole('button', { name: 'Create index' }).first()
expect(ssoProvidersPkeyRow).toBeVisible()
expect(confirmationTokenIdxRow).toBeVisible()
// create new index button does not exist in other schemas
expect(page.getByRole('button', { name: 'Create index' })).not.toBeVisible()
expect(createIndexButton).not.toBeVisible()
// filter by querying
await page.getByRole('textbox', { name: 'Search for an index' }).fill('users')
await page.waitForTimeout(500)
await page.waitForTimeout(2000)
expect(page.getByText('sso_providers_pkey')).not.toBeVisible()
expect(page.getByText('confirmation_token_idx')).toBeVisible()
@@ -582,7 +589,7 @@ test.describe.serial('Database', () => {
.last()
.click()
await page.getByText('Index:confirmation_token_idx')
await page.waitForTimeout(500) // wait for text content to be visible
await page.waitForTimeout(2000) // wait for text content to be visible
expect(await page.getByRole('presentation').textContent()).toBe(
`CREATE UNIQUE INDEX confirmation_token_idx ON auth.users USING btree (confirmation_token) WHERE ((confirmation_token)::text !~ '^[0-9 ]*$'::text)`
)

View File

@@ -1,6 +1,7 @@
import { expect } from '@playwright/test'
import { test } from '../utils/test.js'
import { toUrl } from '../utils/to-url.js'
import { env } from '../env.config.js'
const LOG_DRAIN_OPTIONS = [
{
@@ -18,6 +19,8 @@ const LOG_DRAIN_OPTIONS = [
]
test.describe('Log Drains Settings', () => {
test.skip(env.IS_PLATFORM, 'Log drains are not supported on platform')
test.beforeEach(async ({ page, ref }) => {
// Navigate to the log drains settings page
await page.goto(toUrl(`/project/${ref}/settings/log-drains`))

View File

@@ -266,8 +266,8 @@ test.describe.serial('RLS Policies', () => {
await expect(page.getByRole('radio', { name: 'SELECT' })).toBeChecked()
// Fill in USING clause - allow all access
const editor = page.getByRole('textbox', { name: 'Editor content;Press Alt+F1' })
await editor.fill('true')
await page.locator('.view-lines').click()
await page.keyboard.type('true')
// Save policy
await page.getByRole('button', { name: 'Save policy' }).click()
@@ -314,8 +314,8 @@ test.describe.serial('RLS Policies', () => {
await page.keyboard.press('Escape')
// Fill in WITH CHECK clause - allow all inserts
const editor = page.getByRole('textbox', { name: 'Editor content;Press Alt+F1' })
await editor.fill('true')
await page.locator('.view-lines').click()
await page.keyboard.type('true')
// Save policy
await page.getByRole('button', { name: 'Save policy' }).click()
@@ -357,8 +357,8 @@ test.describe.serial('RLS Policies', () => {
await page.keyboard.press('Escape')
// Fill in USING clause (UPDATE has both USING and WITH CHECK editors, so use first)
const editor = page.getByRole('textbox', { name: 'Editor content;Press Alt+F1' }).first()
await editor.fill('true')
await page.locator('.view-lines').first().click()
await page.keyboard.type('true')
// Save policy
await page.getByRole('button', { name: 'Save policy' }).click()
@@ -399,8 +399,8 @@ test.describe.serial('RLS Policies', () => {
await page.keyboard.press('Escape')
// Fill in USING clause
const editor = page.getByRole('textbox', { name: 'Editor content;Press Alt+F1' })
await editor.fill('true')
await page.locator('.view-lines').click()
await page.keyboard.type('true')
// Save policy
await page.getByRole('button', { name: 'Save policy' }).click()

View File

@@ -6,6 +6,7 @@ import { test } from '../utils/test.js'
import { toUrl } from '../utils/to-url.js'
import { waitForApiResponseWithTimeout } from '../utils/wait-for-response-with-timeout.js'
import { waitForApiResponse } from '../utils/wait-for-response.js'
import { env } from '../env.config.js'
const sqlSnippetName = 'pw_sql_snippet'
const sqlSnippetNameDuplicate = 'pw_sql_snippet (Duplicate)'
@@ -50,6 +51,11 @@ const deleteFolder = async (page: Page, ref: string, folderName: string) => {
}
test.describe('SQL Editor', () => {
test.skip(
env.IS_PLATFORM,
'This test does not work in hosted environments. Self hosted mode is supported.'
)
let page: Page
test.beforeAll(async ({ browser, ref }) => {
@@ -100,7 +106,7 @@ test.describe('SQL Editor', () => {
// remove sql snippets for "Untitled query" and "pw_sql_snippet"
const privateSnippet = page.getByLabel('private-snippets')
let privateSnippetText = await privateSnippet.textContent()
while (privateSnippetText.includes(newSqlSnippetName)) {
while (privateSnippetText?.includes(newSqlSnippetName)) {
await deleteSqlSnippet(page, ref, newSqlSnippetName)
privateSnippetText =
(await page.getByLabel('private-snippets').count()) > 0
@@ -108,7 +114,7 @@ test.describe('SQL Editor', () => {
: ''
}
while (privateSnippetText.includes(sqlSnippetName)) {
while (privateSnippetText?.includes(sqlSnippetName)) {
await deleteSqlSnippet(page, ref, sqlSnippetName)
privateSnippetText =
(await page.getByLabel('private-snippets').count()) > 0
@@ -298,12 +304,12 @@ hello world`)
await deleteSqlSnippet(page, ref, sqlSnippetNameShare)
}
if ((await page.getByRole('button', { name: 'Shared' }).textContent()).includes('(')) {
if ((await page.getByRole('button', { name: 'Shared' })?.textContent())?.includes('(')) {
const sharedSnippetSection = page.getByLabel('project-level-snippets')
await page.getByRole('button', { name: 'Shared' }).click()
let sharedSnippetText = await sharedSnippetSection.textContent()
while (sharedSnippetText.includes(sqlSnippetNameShare)) {
while (sharedSnippetText?.includes(sqlSnippetNameShare)) {
await sharedSnippetSection.getByText(sqlSnippetName).last().click({ button: 'right' })
await page.getByRole('menuitem', { name: 'Delete query' }).click()
await expect(page.getByRole('heading', { name: 'Confirm to delete query' })).toBeVisible()
@@ -521,7 +527,7 @@ hello world`)
const searchBar = page.getByRole('textbox', { name: 'Search queries...' })
await searchBar.fill('Duplicate')
await expect(page.getByText(sqlSnippetName, { exact: true })).not.toBeVisible()
await expect(page.getByRole('link', { name: sqlSnippetNameDuplicate })).toBeVisible()
await expect(page.getByTitle(sqlSnippetNameDuplicate, { exact: true })).toBeVisible()
await expect(page.getByText('result found')).toBeVisible()
await searchBar.fill('') // clear search bar

View File

@@ -12,6 +12,7 @@ import {
waitForGridDataToLoad,
waitForTableToLoad,
} from '../utils/wait-for-response.js'
import { env } from '../env.config.js'
const tableNamePrefix = 'pw_table'
const columnName = 'pw_column'
@@ -107,7 +108,9 @@ const deleteEnumIfExist = async (page: Page, ref: string, enumName: string) => {
await waitForApiResponse(page, 'pg-meta', ref, 'query?key=', { method: 'POST' })
}
test.describe('table editor', () => {
// Due to rate API rate limits run this test in serial mode on platform.
const testRunner = env.IS_PLATFORM ? test.describe.serial : test.describe
testRunner('table editor', () => {
test.beforeAll(async ({ browser, ref }) => {
await withFileOnceSetup(import.meta.url, async () => {
const ctx = await browser.newContext()
@@ -598,13 +601,7 @@ test.describe('table editor', () => {
// test pagination (page 1 -> page 2)
await expect(page.getByRole('gridcell', { name: 'value 7', exact: true })).toBeVisible()
await expect(page.getByRole('gridcell', { name: 'value 101', exact: true })).not.toBeVisible()
let footer: Locator
if (isCLI()) {
footer = page.getByLabel('Table grid footer')
} else {
footer = page.locator('[data-sentry-component="GridFooter"]')
}
await footer.getByRole('button').nth(1).click()
await page.getByLabel('Table grid footer').getByRole('button').nth(1).click()
await waitForGridDataToLoad(page, ref) // retrieve next page data
await expect(page.getByRole('gridcell', { name: 'value 7', exact: true })).not.toBeVisible()
await expect(page.getByRole('gridcell', { name: 'value 101', exact: true })).toBeVisible()

View File

@@ -12,7 +12,14 @@
"license": "ISC",
"description": "",
"dependencies": {
"dotenv": "^16.5.0",
"@playwright/test": "^1.52.0"
"@playwright/test": "^1.52.0",
"@supabase/supabase-js": "catalog:",
"cross-fetch": "^4.1.0",
"dotenv": "^16.5.0"
},
"devDependencies": {
"@faker-js/faker": "^9.9.0",
"api-types": "workspace:*",
"tsx": "catalog:"
}
}

View File

@@ -1,15 +1,37 @@
import { defineConfig } from '@playwright/test'
import dotenv from 'dotenv'
import path from 'path'
import { env, STORAGE_STATE_PATH } from './env.config.js'
dotenv.config({ path: path.resolve(import.meta.dirname, '.env.local') })
const IS_CI = !!process.env.CI
const WEB_SERVER_TIMEOUT = Number(process.env.WEB_SERVER_TIMEOUT) || 10 * 60 * 1000
const WEB_SERVER_PORT = Number(process.env.WEB_SERVER_PORT) || 8082
// 15 minutes for platform, 2 minutes for self-hosted. Takes longer to setup a full project on platform.
const setupTimeout = env.IS_PLATFORM ? 15 * 60 * 1000 : 120 * 1000
const createWebServerConfig = () => {
if (env.IS_PLATFORM && env.IS_APP_RUNNING_ON_LOCALHOST) {
return {
command: 'pnpm --workspace-root run e2e:setup:platform',
port: WEB_SERVER_PORT,
timeout: WEB_SERVER_TIMEOUT,
reuseExistingServer: true,
}
}
// Apps running on runner using the vercel staging environment
if (env.IS_PLATFORM && !env.IS_APP_RUNNING_ON_LOCALHOST) {
return undefined
}
return {
command: 'pnpm --workspace-root run e2e:setup:selfhosted',
port: WEB_SERVER_PORT,
timeout: WEB_SERVER_TIMEOUT,
reuseExistingServer: true,
}
}
export default defineConfig({
timeout: 120 * 1000,
testDir: './features',
@@ -17,7 +39,9 @@ export default defineConfig({
forbidOnly: IS_CI,
retries: IS_CI ? 5 : 0,
maxFailures: 3,
fullyParallel: true,
// Due to rate API rate limits run tests in serial mode on platform.
fullyParallel: !env.IS_PLATFORM,
workers: env.IS_PLATFORM ? 1 : 5,
use: {
baseURL: env.STUDIO_URL,
screenshot: 'off',
@@ -26,7 +50,8 @@ export default defineConfig({
trace: 'retain-on-failure',
permissions: ['clipboard-read', 'clipboard-write'],
extraHTTPHeaders: {
'x-vercel-protection-bypass': process.env.VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO,
'x-vercel-protection-bypass':
process.env.VERCEL_AUTOMATION_BYPASS_SELFHOSTED_STUDIO || 'false',
'x-vercel-set-bypass-cookie': 'true',
},
},
@@ -34,6 +59,7 @@ export default defineConfig({
{
name: 'setup',
testMatch: /.*\.setup\.ts/,
timeout: setupTimeout,
},
{
name: 'Features',
@@ -43,6 +69,7 @@ export default defineConfig({
use: {
browserName: 'chromium',
screenshot: 'off',
// Only use storage state if authentication is enabled. When AUTHENTICATION=false
// we should not require a pre-generated storage state file.
storageState: env.AUTHENTICATION ? STORAGE_STATE_PATH : undefined,
@@ -54,10 +81,5 @@ export default defineConfig({
['html', { open: 'never' }],
['json', { outputFile: 'test-results/test-results.json' }],
],
webServer: {
command: 'pnpm --workspace-root run e2e:setup',
port: WEB_SERVER_PORT,
timeout: WEB_SERVER_TIMEOUT,
reuseExistingServer: true,
},
webServer: createWebServerConfig(),
})

View File

@@ -0,0 +1,46 @@
import assert from 'assert'
import { setTimeout } from 'timers/promises'
import { PlatformClient } from './platform.js'
const statusWaiterMilliSeconds = parseInt(process.env.STATUS_WAITER_MILLI_SECONDS ?? '3000')
const statusWaiterRetries = parseInt(
process.env.STATUS_WAITER_RETRIES ?? `${900_000 / statusWaiterMilliSeconds}`
)
export const sleep = (ms: number) => setTimeout(ms)
export type WaitForProjectStatusParams = {
platformClient: PlatformClient
ref: string
expectedStatus: string
retries?: number
}
export async function waitForProjectStatus({
platformClient,
ref,
expectedStatus,
retries = statusWaiterRetries,
}: WaitForProjectStatusParams) {
for (let i = 0; i < retries; i++) {
try {
const statusResp = await platformClient.send(`/v1/projects/${ref}`, {}, undefined, 0)
if (statusResp.status != 200) {
console.log(
`Failed to get project status ${statusResp.statusText} ${
statusResp.status
} ${await statusResp.text()}`
)
}
assert(statusResp.status == 200)
const { status } = await statusResp.json()
assert(status == expectedStatus)
return
} catch {
await sleep(statusWaiterMilliSeconds)
}
}
throw new Error(
`did not reach expected status ${expectedStatus} after retries ${retries} x ${statusWaiterMilliSeconds}ms`
)
}

View File

@@ -0,0 +1,36 @@
import retriedFetch from './retriedFetch.js'
export class PlatformClient {
url: string
#accessToken: string
headers: Record<string, string>
constructor({ url, accessToken }: { url: string; accessToken: string }) {
this.url = url
this.#accessToken = accessToken
this.headers = {
Authorization: `Bearer ${this.#accessToken}`,
'content-type': 'application/json',
}
}
send(
endpoint: string,
options?: Omit<RequestInit, 'body'> & { body?: Record<string, unknown> },
timeout?: number,
retries?: number,
delayBase?: number
) {
return retriedFetch(
`${this.url}${endpoint}`,
{
...(options ?? {}),
body: options?.body ? JSON.stringify(options.body) : undefined,
headers: { ...this.headers, ...(options?.headers ?? {}) },
},
timeout,
retries,
delayBase
)
}
}

View File

@@ -0,0 +1,24 @@
import timeoutFetch from './timeoutFetch.js'
export default async function retriedFetch(
input: RequestInfo,
init?: RequestInit,
timeout: number = 10000,
retries: number = 3,
delayBase: number = 200
): Promise<Response> {
for (let i = 0; i < retries; i++) {
try {
const res = await timeoutFetch(input, init, timeout)
if (res.status >= 100 && res.status < 400) {
return res
}
console.log(`Retrying fetch ${i} ${input}`, res.status, res.statusText)
} catch (e) {
console.log(`Retrying fetch ${i} ${input}`, e)
} finally {
await new Promise((resolve) => setTimeout(resolve, delayBase * (i + 1)))
}
}
return await timeoutFetch(input, init, timeout)
}

View File

@@ -0,0 +1,40 @@
export default async function timeoutRequest<T>(
request: Promise<T>,
timeout: number,
abortController?: AbortController
): Promise<T> {
let timer: NodeJS.Timeout | undefined
const cleanup = () => {
if (timer) {
clearTimeout(timer)
timer = undefined
}
}
try {
const timeoutPromise = new Promise<T>((_, reject) => {
timer = setTimeout(() => {
if (abortController) {
abortController.abort()
}
cleanup()
reject(new Error(`Timeout (${timeout}) for task exceeded`))
}, timeout)
})
const result = await Promise.race<T>([
request.catch((err) => {
cleanup()
throw err
}),
timeoutPromise
])
cleanup()
return result
} catch (error) {
cleanup()
throw error
}
}

View File

@@ -0,0 +1,21 @@
import crossFetch from 'cross-fetch'
import timeoutPromise from './timeout.js'
export default async function fetch(
input: RequestInfo,
init?: RequestInit,
timeout: number = 10000
): Promise<Response> {
if (init?.method === 'POST' && timeout === 10000) {
timeout = 15000
}
const controller = new AbortController()
const initWithSignal = init
if (!init?.signal) {
const initWithSignal = {
...init,
signal: controller.signal,
}
}
return timeoutPromise(crossFetch(input, initWithSignal), timeout, controller)
}

View File

@@ -0,0 +1,47 @@
import assert from 'assert'
import { PlatformClient } from './platform.js'
import { sleep } from './helpers.js'
const checkHealth = async (platformClient: PlatformClient, ref: string) => {
// get health of services
const healthResp = await platformClient.send(
`/v1/projects/${ref}/health?services=db,pooler,auth,realtime,rest,storage`
)
assert(
healthResp.status == 200,
`Failed to get health ${healthResp.status}: ${healthResp.statusText}`
)
const health = await healthResp.json()
return health as Health[]
}
type Health = {
name: string
healthy: boolean
status: string
info?: unknown
error?: unknown
}
export const waitForHealthyServices = async (platformClient: PlatformClient, ref: string) => {
// check health 600 times every 2 seconds; 20mins
for (let i = 0; i < 600; i++) {
try {
const health = await checkHealth(platformClient, ref)
// check if all services are healthy
if (health.every((h) => h.healthy)) {
return
}
console.log(`waiting ${i} ... services: ${JSON.stringify(health.filter((h) => !h.healthy))}`)
} catch (e) {
console.log(`waiting ${i} ... errored: ${(e as { message: string }).message}`)
}
await sleep(2000)
}
throw new Error('Services are not healthy')
}

View File

@@ -0,0 +1,90 @@
import assert from 'assert'
import { faker } from '@faker-js/faker'
import { waitForProjectStatus } from '../common/helpers.js'
import { waitForHealthyServices } from '../common/wait-healthy-services.js'
import { PlatformClient } from '../common/platform.js'
export interface CreateProjectParams {
platformClient: PlatformClient
orgSlug: string
supaRegion: string
projectName: string
}
export async function createProject({
platformClient,
orgSlug,
supaRegion,
projectName,
}: CreateProjectParams): Promise<string> {
const dbPass = faker.internet.password()
const createResp = await platformClient.send(
`/v1/projects`,
{
method: 'POST',
body: {
organization_slug: orgSlug,
name: projectName,
region_selection: {
type: 'specific',
code: supaRegion,
},
db_pass: dbPass,
desired_instance_size: 'small',
},
},
60000
)
if (createResp.status != 201) {
console.error('❌ Could not create project')
console.error(await createResp.text())
}
assert(createResp.status == 201, createResp.statusText)
const project = await createResp.json()
const ref = project.ref
console.log(`✨ Created project ${ref}`)
console.log('⏳ Waiting for healthy project...')
// wait for project to be ready
await waitForProjectStatus({ platformClient, ref, expectedStatus: 'ACTIVE_HEALTHY' })
// wait for all services to be healthy
console.log('⏳ Waiting for healthy services...')
await waitForHealthyServices(platformClient, ref)
console.log(`🎉 Project created successfully: ${ref}`)
return ref
}
export interface GetProjectRefParams {
platformClient: PlatformClient
orgSlug: string
supaRegion: string
projectName: string
}
export async function getProjectRef({
platformClient,
orgSlug,
supaRegion,
projectName,
}: GetProjectRefParams): Promise<string | undefined> {
const getResp = await platformClient.send(`/v1/projects`, { method: 'GET' }, 60000)
if (getResp.status != 200) {
console.error('❌ Could not fetch projects')
console.error(await getResp.text())
}
assert(getResp.status == 200, getResp.statusText)
const projects = await getResp.json()
const project = projects.find(
(p: any) => p.organization_slug === orgSlug && p.region === supaRegion && p.name === projectName
)
return project?.ref
}

View File

@@ -0,0 +1,51 @@
import { env } from '../env.config.js'
import { PlatformClient } from './common/platform.js'
import { createProject, getProjectRef } from './helpers/project.js'
export async function setupProjectForTests() {
if (!env.IS_PLATFORM) {
console.log('Not running on platform, skipping project creation')
return 'default'
}
// Will default to e2e-test-<timestamp> if not set
const projectName = env.BRANCH_NAME
// Validate required environment variables
const orgSlug = env.ORG_SLUG
const supaRegion = env.SUPA_REGION
const apiUrl = env.API_URL
const supaPat = env.SUPA_PAT
if (!orgSlug) throw new Error('ORG_SLUG environment variable is required')
if (!supaRegion) throw new Error('SUPA_REGION environment variable is required')
if (!apiUrl) throw new Error('API_URL environment variable is required')
if (!supaPat) throw new Error('SUPA_PAT environment variable is required')
const platformClient = new PlatformClient({
url: apiUrl,
accessToken: supaPat,
})
const existingProjectRef = await getProjectRef({
platformClient,
orgSlug,
supaRegion,
projectName,
})
if (existingProjectRef) {
console.log(`\n ✅ Project found: ${existingProjectRef}, settings as environment variables`)
return existingProjectRef
} else {
console.log(`\n 🔑 Project not found, creating new project...`)
}
const ref = await createProject({
platformClient,
orgSlug,
supaRegion,
projectName,
})
console.log(`\n ✅ Project created: ${ref}, settings as environment variables`)
return ref
}

View File

@@ -4,6 +4,7 @@
"module": "nodenext",
"jsx": "react",
"skipLibCheck": true,
"esModuleInterop": true
"esModuleInterop": true,
"strict": true
}
}

View File

@@ -1,4 +1,4 @@
import { env } from '../env.config'
import { env } from '../env.config.js'
/**
* Returns true if running in CLI/self-hosted mode (locally),
@@ -7,5 +7,5 @@ import { env } from '../env.config'
export function isCLI(): boolean {
// IS_PLATFORM=true = hosted mode
// IS_PLATFORM=false = CLI/self-hosted mode
return env.IS_PLATFORM === 'false'
return !env.IS_PLATFORM
}

View File

@@ -1,6 +1,6 @@
import { expect, Page } from '@playwright/test'
import { toUrl } from './to-url.js'
import { waitForApiResponse } from './wait-for-response.js'
import { toUrl } from './to-url.js'
/**
* Dismisses any visible toast notifications
@@ -130,22 +130,18 @@ export const navigateToBucket = async (page: Page, ref: string, bucketName: stri
const bucketRow = page.getByRole('row').filter({ hasText: bucketName })
await expect(bucketRow, `Bucket row for ${bucketName} should be visible`).toBeVisible()
// Click the bucket and wait for page to load
const navigationPromise = page.waitForURL(
new RegExp(`/storage/files/buckets/${encodeURIComponent(bucketName)}`)
)
const apiPromise = waitForApiResponse(
page,
'storage',
ref,
`buckets/${bucketName}/objects/list`,
{
method: 'POST',
}
// Wait for the objects list API request to complete
const objectsListPromise = page.waitForResponse(
(response) =>
response.url().includes(`/platform/storage/${ref}/buckets/${bucketName}/objects/list`) &&
response.request().method() === 'POST' &&
(response.status() === 200 || response.status() === 201)
)
await bucketRow.click()
await navigationPromise
await apiPromise
// Wait for the API response
await objectsListPromise
// Verify we're in the bucket by checking the breadcrumb or "Edit bucket" button
await expect(
@@ -190,7 +186,7 @@ export const uploadFile = async (page: Page, filePath: string, fileName: string)
await fileInput.setInputFiles(filePath)
// Wait for upload to complete - file should appear in the explorer
await page.waitForTimeout(2000) // Allow time for upload to process
await page.waitForTimeout(15_000) // Allow time for upload to process
// Verify file appears in the explorer by title
await expect(

View File

@@ -16,6 +16,6 @@ export interface TestOptions {
export const test = base.extend<TestOptions>({
env: env.STUDIO_URL,
ref: 'default',
ref: env.PROJECT_REF ?? 'default',
apiUrl: env.API_URL,
})

View File

@@ -1,4 +1,4 @@
import { env } from '../env.config'
import { env } from '../env.config.js'
export function toUrl(path: `/${string}`) {
return `${env.STUDIO_URL}${path}`

View File

@@ -30,7 +30,8 @@
"test:studio": "turbo run test --filter=studio",
"test:studio:watch": "turbo run test --filter=studio -- watch",
"e2e:setup:cli": "supabase stop --all --no-backup ; supabase start --exclude studio && supabase db reset && supabase status --output json > keys.json && node scripts/generateLocalEnv.js",
"e2e:setup": "SKIP_ASSET_UPLOAD=1 pnpm e2e:setup:cli && NODE_ENV=test NODE_OPTIONS=\"--max-old-space-size=4096\" pnpm run build:studio && NODE_ENV=test pnpm --prefix ./apps/studio start --port 8082",
"e2e:setup:selfhosted": "SKIP_ASSET_UPLOAD=1 pnpm e2e:setup:cli && NODE_ENV=test NODE_OPTIONS=\"--max-old-space-size=4096\" pnpm run build:studio && NODE_ENV=test pnpm --prefix ./apps/studio start --port 8082",
"e2e:setup:platform": "SKIP_ASSET_UPLOAD=1 NODE_OPTIONS=\"--max-old-space-size=4096\" pnpm run build:studio && pnpm --prefix ./apps/studio start --port 8082",
"e2e": "pnpm --prefix e2e/studio run e2e",
"e2e:ui": "pnpm --prefix e2e/studio run e2e:ui",
"perf:kong": "ab -t 5 -c 20 -T application/json http://localhost:8000/",

76
pnpm-lock.yaml generated
View File

@@ -6,63 +6,12 @@ settings:
catalogs:
default:
'@sentry/nextjs':
specifier: ^10.26.0
version: 10.27.0
'@supabase/auth-js':
specifier: 2.86.0
version: 2.86.0
'@supabase/postgrest-js':
specifier: 2.86.0
version: 2.86.0
'@supabase/realtime-js':
specifier: 2.86.0
version: 2.86.0
'@supabase/supabase-js':
specifier: 2.86.0
version: 2.86.0
'@types/node':
specifier: ^22.0.0
version: 22.13.14
'@types/react':
specifier: ^18.3.0
version: 18.3.3
'@types/react-dom':
specifier: ^18.3.0
version: 18.3.0
next:
specifier: ^15.5.7
version: 15.5.7
react:
specifier: ^18.3.0
version: 18.3.1
react-dom:
specifier: ^18.3.0
version: 18.3.1
recharts:
specifier: ^2.15.4
version: 2.15.4
tailwindcss:
specifier: 3.4.1
version: 3.4.1
tsx:
specifier: 4.20.3
version: 4.20.3
typescript:
specifier: ~5.9.0
version: 5.9.2
valtio:
specifier: ^1.12.0
version: 1.12.0
vite:
specifier: ^7.1.11
version: 7.1.11
vitest:
specifier: ^3.2.0
version: 3.2.4
zod:
specifier: ^3.25.76
version: 3.25.76
overrides:
'@eslint/eslintrc>js-yaml': ^4.1.1
@@ -1881,9 +1830,25 @@ importers:
'@playwright/test':
specifier: ^1.52.0
version: 1.56.1
'@supabase/supabase-js':
specifier: 'catalog:'
version: 2.86.0
cross-fetch:
specifier: ^4.1.0
version: 4.1.0(encoding@0.1.13)
dotenv:
specifier: ^16.5.0
version: 16.5.0
devDependencies:
'@faker-js/faker':
specifier: ^9.9.0
version: 9.9.0
api-types:
specifier: workspace:*
version: link:../../packages/api-types
tsx:
specifier: 'catalog:'
version: 4.20.3
packages/ai-commands:
dependencies:
@@ -10854,6 +10819,9 @@ packages:
cross-fetch@3.2.0:
resolution: {integrity: sha512-Q+xVJLoGOeIMXZmbUK4HYk+69cQH6LudR0Vu/pRm2YlU/hDV9CiS0gKUMaWY5f2NeUH9C1nV3bsTlCo0FsTV1Q==}
cross-fetch@4.1.0:
resolution: {integrity: sha512-uKm5PU+MHTootlWEY+mZ4vvXoCn4fLQxT9dSc1sXVMSFkINTJVN8cAQROpwcKm8bJ/c7rgZVIBWzH5T78sNZZw==}
cross-inspect@1.0.1:
resolution: {integrity: sha512-Pcw1JTvZLSJH83iiGWt6fRcT+BjZlCDRVwYLbUcHzv/CRpB7r0MlSrGbIyQvVSNyGnbt7G4AXuyCiDR3POvZ1A==}
engines: {node: '>=16.0.0'}
@@ -30448,6 +30416,12 @@ snapshots:
transitivePeerDependencies:
- encoding
cross-fetch@4.1.0(encoding@0.1.13):
dependencies:
node-fetch: 2.7.0(encoding@0.1.13)
transitivePeerDependencies:
- encoding
cross-inspect@1.0.1:
dependencies:
tslib: 2.8.1

View File

@@ -7,9 +7,9 @@ packages:
catalog:
'@sentry/nextjs': ^10.26.0
'@supabase/auth-js': 2.86.0
'@supabase/postgrest-js': 2.86.0
'@supabase/realtime-js': 2.86.0
'@supabase/supabase-js': 2.86.0
'@supabase/postgrest-js': 2.86.0
'@types/node': ^22.0.0
'@types/react': ^18.3.0
'@types/react-dom': ^18.3.0
@@ -60,8 +60,8 @@ overrides:
'@redocly/respect-core>js-yaml': ^4.1.1
'@tanstack/directive-functions-plugin>vite': 'catalog:'
'@tanstack/react-start-plugin>vite': 'catalog:'
'vinxi>vite': 'catalog:'
'refractor>prismjs': ^1.30.0
esbuild: ^0.25.2
refractor>prismjs: ^1.30.0
tar: ^7.0.0
tmp: ^0.2.4
vinxi>vite: 'catalog:'