From fba0b0f7f90b4ee7cf7411621048dea78baca5b7 Mon Sep 17 00:00:00 2001 From: waleed Date: Thu, 8 Jan 2026 09:51:04 -0800 Subject: [PATCH 1/5] improvement(auth): added ability to inject secrets to kubernetes, server-side ff to disable email registration --- apps/sim/lib/auth/auth.ts | 7 + apps/sim/lib/core/config/env.ts | 1 + apps/sim/lib/core/config/feature-flags.ts | 5 + .../sim/lib/logs/execution/logging-session.ts | 80 +++++++- helm/sim/README.md | 107 ++++++++++ helm/sim/examples/values-existing-secret.yaml | 126 ++++++++++++ .../sim/examples/values-external-secrets.yaml | 192 ++++++++++++++++++ helm/sim/templates/_helpers.tpl | 112 +++++++++- helm/sim/templates/deployment-app.yaml | 26 +-- helm/sim/templates/deployment-realtime.yaml | 15 +- helm/sim/templates/external-db-secret.yaml | 3 +- helm/sim/templates/external-secret-app.yaml | 48 +++++ .../external-secret-external-db.yaml | 27 +++ .../templates/external-secret-postgresql.yaml | 26 +++ helm/sim/templates/secrets-app.yaml | 30 +++ .../sim/templates/statefulset-postgresql.yaml | 5 +- helm/sim/values.schema.json | 129 +++++++++++- helm/sim/values.yaml | 102 +++++++++- 18 files changed, 992 insertions(+), 49 deletions(-) create mode 100644 helm/sim/examples/values-existing-secret.yaml create mode 100644 helm/sim/examples/values-external-secrets.yaml create mode 100644 helm/sim/templates/external-secret-app.yaml create mode 100644 helm/sim/templates/external-secret-external-db.yaml create mode 100644 helm/sim/templates/external-secret-postgresql.yaml create mode 100644 helm/sim/templates/secrets-app.yaml diff --git a/apps/sim/lib/auth/auth.ts b/apps/sim/lib/auth/auth.ts index 926d833eb6..a16dee4451 100644 --- a/apps/sim/lib/auth/auth.ts +++ b/apps/sim/lib/auth/auth.ts @@ -47,6 +47,7 @@ import { env } from '@/lib/core/config/env' import { isAuthDisabled, isBillingEnabled, + isEmailPasswordEnabled, isEmailVerificationEnabled, isHosted, isRegistrationDisabled, @@ -461,6 +462,12 @@ export const auth = betterAuth({ if (ctx.path.startsWith('/sign-up') && isRegistrationDisabled) throw new Error('Registration is disabled, please contact your admin.') + if (!isEmailPasswordEnabled) { + const emailPasswordPaths = ['/sign-in/email', '/sign-up/email', '/email-otp'] + if (emailPasswordPaths.some((path) => ctx.path.startsWith(path))) + throw new Error('Email/password authentication is disabled. Please use SSO to sign in.') + } + if ( (ctx.path.startsWith('/sign-in') || ctx.path.startsWith('/sign-up')) && (env.ALLOWED_LOGIN_EMAILS || env.ALLOWED_LOGIN_DOMAINS) diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 56bc8ebee7..c879c3a775 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -20,6 +20,7 @@ export const env = createEnv({ BETTER_AUTH_URL: z.string().url(), // Base URL for Better Auth service BETTER_AUTH_SECRET: z.string().min(32), // Secret key for Better Auth JWT signing DISABLE_REGISTRATION: z.boolean().optional(), // Flag to disable new user registration + EMAIL_PASSWORD_SIGNUP_ENABLED: z.boolean().optional().default(true), // Enable email/password authentication (server-side enforcement) DISABLE_AUTH: z.boolean().optional(), // Bypass authentication entirely (self-hosted only, creates anonymous session) ALLOWED_LOGIN_EMAILS: z.string().optional(), // Comma-separated list of allowed email addresses for login ALLOWED_LOGIN_DOMAINS: z.string().optional(), // Comma-separated list of allowed email domains for login diff --git a/apps/sim/lib/core/config/feature-flags.ts b/apps/sim/lib/core/config/feature-flags.ts index 61e12732fa..0438f8074e 100644 --- a/apps/sim/lib/core/config/feature-flags.ts +++ b/apps/sim/lib/core/config/feature-flags.ts @@ -65,6 +65,11 @@ if (isTruthy(env.DISABLE_AUTH)) { */ export const isRegistrationDisabled = isTruthy(env.DISABLE_REGISTRATION) +/** + * Is email/password authentication enabled (defaults to true) + */ +export const isEmailPasswordEnabled = env.EMAIL_PASSWORD_SIGNUP_ENABLED !== false + /** * Is Trigger.dev enabled for async job processing */ diff --git a/apps/sim/lib/logs/execution/logging-session.ts b/apps/sim/lib/logs/execution/logging-session.ts index d618be12bc..de63aaf5e7 100644 --- a/apps/sim/lib/logs/execution/logging-session.ts +++ b/apps/sim/lib/logs/execution/logging-session.ts @@ -289,10 +289,12 @@ export class LoggingSession { this.completed = true - // Track workflow execution outcome + // Track workflow execution outcome and create trace spans if (traceSpans && traceSpans.length > 0) { try { - const { trackPlatformEvent } = await import('@/lib/core/telemetry') + const { trackPlatformEvent, createOTelSpansForWorkflowExecution } = await import( + '@/lib/core/telemetry' + ) // Determine status from trace spans const hasErrors = traceSpans.some((span: any) => { @@ -315,6 +317,20 @@ export class LoggingSession { 'execution.has_errors': hasErrors, 'execution.total_cost': costSummary.totalCost || 0, }) + + // Create OpenTelemetry trace spans for the workflow execution + const startTime = new Date(new Date(endTime).getTime() - duration).toISOString() + createOTelSpansForWorkflowExecution({ + workflowId: this.workflowId, + workflowName: this.workflowState?.metadata?.name, + executionId: this.executionId, + traceSpans, + trigger: this.triggerType, + startTime, + endTime, + totalDurationMs: duration, + status: hasErrors ? 'error' : 'success', + }) } catch (_e) { // Silently fail } @@ -404,9 +420,11 @@ export class LoggingSession { this.completed = true - // Track workflow execution error outcome + // Track workflow execution error outcome and create trace spans try { - const { trackPlatformEvent } = await import('@/lib/core/telemetry') + const { trackPlatformEvent, createOTelSpansForWorkflowExecution } = await import( + '@/lib/core/telemetry' + ) trackPlatformEvent('platform.workflow.executed', { 'workflow.id': this.workflowId, 'execution.duration_ms': Math.max(1, durationMs), @@ -416,6 +434,20 @@ export class LoggingSession { 'execution.has_errors': true, 'execution.error_message': message, }) + + // Create OpenTelemetry trace spans for the workflow execution + createOTelSpansForWorkflowExecution({ + workflowId: this.workflowId, + workflowName: this.workflowState?.metadata?.name, + executionId: this.executionId, + traceSpans: spans, + trigger: this.triggerType, + startTime: startTime.toISOString(), + endTime: endTime.toISOString(), + totalDurationMs: Math.max(1, durationMs), + status: 'error', + error: message, + }) } catch (_e) { // Silently fail } @@ -477,7 +509,9 @@ export class LoggingSession { this.completed = true try { - const { trackPlatformEvent } = await import('@/lib/core/telemetry') + const { trackPlatformEvent, createOTelSpansForWorkflowExecution } = await import( + '@/lib/core/telemetry' + ) trackPlatformEvent('platform.workflow.executed', { 'workflow.id': this.workflowId, 'execution.duration_ms': Math.max(1, durationMs), @@ -486,6 +520,22 @@ export class LoggingSession { 'execution.blocks_executed': traceSpans?.length || 0, 'execution.has_errors': false, }) + + // Create OpenTelemetry trace spans for the workflow execution + if (traceSpans && traceSpans.length > 0) { + const startTime = new Date(endTime.getTime() - Math.max(1, durationMs)) + createOTelSpansForWorkflowExecution({ + workflowId: this.workflowId, + workflowName: this.workflowState?.metadata?.name, + executionId: this.executionId, + traceSpans, + trigger: this.triggerType, + startTime: startTime.toISOString(), + endTime: endTime.toISOString(), + totalDurationMs: Math.max(1, durationMs), + status: 'success', // Cancelled executions are not errors + }) + } } catch (_e) { // Silently fail } @@ -540,7 +590,9 @@ export class LoggingSession { }) try { - const { trackPlatformEvent } = await import('@/lib/core/telemetry') + const { trackPlatformEvent, createOTelSpansForWorkflowExecution } = await import( + '@/lib/core/telemetry' + ) trackPlatformEvent('platform.workflow.executed', { 'workflow.id': this.workflowId, 'execution.duration_ms': Math.max(1, durationMs), @@ -550,6 +602,22 @@ export class LoggingSession { 'execution.has_errors': false, 'execution.total_cost': costSummary.totalCost || 0, }) + + // Create OpenTelemetry trace spans for the workflow execution + if (traceSpans && traceSpans.length > 0) { + const startTime = new Date(endTime.getTime() - Math.max(1, durationMs)) + createOTelSpansForWorkflowExecution({ + workflowId: this.workflowId, + workflowName: this.workflowState?.metadata?.name, + executionId: this.executionId, + traceSpans, + trigger: this.triggerType, + startTime: startTime.toISOString(), + endTime: endTime.toISOString(), + totalDurationMs: Math.max(1, durationMs), + status: 'success', // Paused executions are not errors + }) + } } catch (_e) {} if (this.requestId) { diff --git a/helm/sim/README.md b/helm/sim/README.md index fd6a452941..0c33120539 100644 --- a/helm/sim/README.md +++ b/helm/sim/README.md @@ -39,6 +39,8 @@ The chart includes several pre-configured values files for different scenarios: | `values-azure.yaml` | Azure AKS optimized | Azure Kubernetes Service | | `values-aws.yaml` | AWS EKS optimized | Amazon Elastic Kubernetes Service | | `values-gcp.yaml` | GCP GKE optimized | Google Kubernetes Engine | +| `values-external-secrets.yaml` | External Secrets Operator integration | Using Azure Key Vault, AWS Secrets Manager, Vault | +| `values-existing-secret.yaml` | Pre-existing Kubernetes secrets | GitOps, Sealed Secrets, manual secret management | ### Development Environment @@ -623,6 +625,111 @@ To uninstall/delete the release: helm uninstall sim ``` +## External Secret Management + +The chart supports integration with external secret management systems for production-grade secret handling. This enables you to store secrets in secure vaults and have them automatically synced to Kubernetes. + +### Option 1: External Secrets Operator (Recommended) + +[External Secrets Operator](https://external-secrets.io/) is the industry-standard solution for syncing secrets from external stores like Azure Key Vault, AWS Secrets Manager, HashiCorp Vault, and GCP Secret Manager. + +**Prerequisites:** +```bash +# Install External Secrets Operator +helm repo add external-secrets https://charts.external-secrets.io +helm install external-secrets external-secrets/external-secrets \ + -n external-secrets --create-namespace +``` + +**Configuration:** +```yaml +externalSecrets: + enabled: true + refreshInterval: "1h" + secretStoreRef: + name: "my-secret-store" + kind: "ClusterSecretStore" + remoteRefs: + app: + BETTER_AUTH_SECRET: "sim/app/better-auth-secret" + ENCRYPTION_KEY: "sim/app/encryption-key" + INTERNAL_API_SECRET: "sim/app/internal-api-secret" + postgresql: + password: "sim/postgresql/password" +``` + +See `examples/values-external-secrets.yaml` for complete examples including SecretStore configurations for Azure, AWS, GCP, and Vault. + +### Option 2: Pre-Existing Kubernetes Secrets + +Reference secrets you've created manually, via GitOps (Sealed Secrets, SOPS), or through other automation. + +**Configuration:** +```yaml +app: + secrets: + existingSecret: + enabled: true + name: "my-app-secrets" + +postgresql: + auth: + existingSecret: + enabled: true + name: "my-postgresql-secret" + passwordKey: "POSTGRES_PASSWORD" + +externalDatabase: + existingSecret: + enabled: true + name: "my-external-db-secret" + passwordKey: "password" +``` + +**Create secrets manually:** +```bash +# Generate secure values +BETTER_AUTH_SECRET=$(openssl rand -hex 32) +ENCRYPTION_KEY=$(openssl rand -hex 32) +INTERNAL_API_SECRET=$(openssl rand -hex 32) +POSTGRES_PASSWORD=$(openssl rand -base64 16 | tr -d '/+=') + +# Create app secrets +kubectl create secret generic my-app-secrets \ + --namespace sim \ + --from-literal=BETTER_AUTH_SECRET="$BETTER_AUTH_SECRET" \ + --from-literal=ENCRYPTION_KEY="$ENCRYPTION_KEY" \ + --from-literal=INTERNAL_API_SECRET="$INTERNAL_API_SECRET" + +# Create PostgreSQL secret +kubectl create secret generic my-postgresql-secret \ + --namespace sim \ + --from-literal=POSTGRES_PASSWORD="$POSTGRES_PASSWORD" +``` + +See `examples/values-existing-secret.yaml` for more details. + +### External Secrets Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `app.secrets.existingSecret.enabled` | Use existing secret for app credentials | `false` | +| `app.secrets.existingSecret.name` | Name of existing secret | `""` | +| `app.secrets.existingSecret.keys` | Key name mappings | See values.yaml | +| `postgresql.auth.existingSecret.enabled` | Use existing secret for PostgreSQL | `false` | +| `postgresql.auth.existingSecret.name` | Name of existing secret | `""` | +| `postgresql.auth.existingSecret.passwordKey` | Key containing password | `"POSTGRES_PASSWORD"` | +| `externalDatabase.existingSecret.enabled` | Use existing secret for external DB | `false` | +| `externalDatabase.existingSecret.name` | Name of existing secret | `""` | +| `externalDatabase.existingSecret.passwordKey` | Key containing password | `"EXTERNAL_DB_PASSWORD"` | +| `externalSecrets.enabled` | Enable External Secrets Operator integration | `false` | +| `externalSecrets.refreshInterval` | How often to sync secrets | `"1h"` | +| `externalSecrets.secretStoreRef.name` | Name of SecretStore/ClusterSecretStore | `""` | +| `externalSecrets.secretStoreRef.kind` | Kind of store | `"ClusterSecretStore"` | +| `externalSecrets.remoteRefs.app.*` | Remote paths for app secrets | See values.yaml | +| `externalSecrets.remoteRefs.postgresql.password` | Remote path for PostgreSQL password | `""` | +| `externalSecrets.remoteRefs.externalDatabase.password` | Remote path for external DB password | `""` | + ## Security Considerations ### Production Secrets diff --git a/helm/sim/examples/values-existing-secret.yaml b/helm/sim/examples/values-existing-secret.yaml new file mode 100644 index 0000000000..e0cf3fb656 --- /dev/null +++ b/helm/sim/examples/values-existing-secret.yaml @@ -0,0 +1,126 @@ +# Example: Using Pre-Existing Kubernetes Secrets +# +# This example shows how to use pre-existing Kubernetes secrets instead of +# having the Helm chart create them. This is useful when: +# +# 1. You manage secrets via GitOps tools (Flux, ArgoCD) with sealed secrets +# 2. You use External Secrets Operator to sync from external stores +# 3. You create secrets manually or via other automation +# 4. You want full control over secret lifecycle +# +# Prerequisites: +# Create your secrets before installing the Helm chart. + +# ============================================================================= +# SECRET CONFIGURATION +# ============================================================================= + +# Use existing secret for app credentials +app: + enabled: true + replicaCount: 2 + secrets: + existingSecret: + enabled: true + name: "sim-app-secrets" # Name of your pre-existing secret + # Optional: Customize key names if your secret uses different keys + keys: + BETTER_AUTH_SECRET: "BETTER_AUTH_SECRET" + ENCRYPTION_KEY: "ENCRYPTION_KEY" + INTERNAL_API_SECRET: "INTERNAL_API_SECRET" + CRON_SECRET: "CRON_SECRET" + API_ENCRYPTION_KEY: "API_ENCRYPTION_KEY" + env: + NEXT_PUBLIC_APP_URL: "https://sim.example.com" + BETTER_AUTH_URL: "https://sim.example.com" + NEXT_PUBLIC_SOCKET_URL: "wss://sim-ws.example.com" + NODE_ENV: "production" + +realtime: + enabled: true + replicaCount: 2 + env: + NEXT_PUBLIC_APP_URL: "https://sim.example.com" + BETTER_AUTH_URL: "https://sim.example.com" + NEXT_PUBLIC_SOCKET_URL: "wss://sim-ws.example.com" + ALLOWED_ORIGINS: "https://sim.example.com" + NODE_ENV: "production" + +# Use existing secret for PostgreSQL +postgresql: + enabled: true + auth: + username: postgres + database: sim + existingSecret: + enabled: true + name: "sim-postgresql-secret" # Name of your pre-existing secret + passwordKey: "POSTGRES_PASSWORD" # Key containing the password + +# ============================================================================= +# EXAMPLE: CREATING THE REQUIRED SECRETS +# ============================================================================= +# Run these commands before installing the Helm chart: +# +# # Generate secure values +# BETTER_AUTH_SECRET=$(openssl rand -hex 32) +# ENCRYPTION_KEY=$(openssl rand -hex 32) +# INTERNAL_API_SECRET=$(openssl rand -hex 32) +# CRON_SECRET=$(openssl rand -hex 32) +# API_ENCRYPTION_KEY=$(openssl rand -hex 32) +# POSTGRES_PASSWORD=$(openssl rand -base64 16 | tr -d '/+=') +# +# # Create app secrets +# kubectl create secret generic sim-app-secrets \ +# --namespace sim \ +# --from-literal=BETTER_AUTH_SECRET="$BETTER_AUTH_SECRET" \ +# --from-literal=ENCRYPTION_KEY="$ENCRYPTION_KEY" \ +# --from-literal=INTERNAL_API_SECRET="$INTERNAL_API_SECRET" \ +# --from-literal=CRON_SECRET="$CRON_SECRET" \ +# --from-literal=API_ENCRYPTION_KEY="$API_ENCRYPTION_KEY" +# +# # Create PostgreSQL secret +# kubectl create secret generic sim-postgresql-secret \ +# --namespace sim \ +# --from-literal=POSTGRES_PASSWORD="$POSTGRES_PASSWORD" + +# ============================================================================= +# EXAMPLE: USING SEALED SECRETS (GitOps) +# ============================================================================= +# If using Bitnami Sealed Secrets, create a SealedSecret: +# +# apiVersion: bitnami.com/v1alpha1 +# kind: SealedSecret +# metadata: +# name: sim-app-secrets +# namespace: sim +# spec: +# encryptedData: +# BETTER_AUTH_SECRET: +# ENCRYPTION_KEY: +# INTERNAL_API_SECRET: +# CRON_SECRET: +# API_ENCRYPTION_KEY: +# template: +# metadata: +# name: sim-app-secrets +# namespace: sim +# type: Opaque + +# ============================================================================= +# EXAMPLE: EXTERNAL DATABASE WITH EXISTING SECRET +# ============================================================================= +# If using an external database (e.g., AWS RDS, Azure Database), you can also +# use existing secrets: +# +# externalDatabase: +# enabled: true +# host: "mydb.cluster-xyz.us-east-1.rds.amazonaws.com" +# port: 5432 +# username: postgres +# database: sim +# sslMode: require +# existingSecret: +# enabled: true +# name: "sim-external-db-secret" +# passwordKey: "EXTERNAL_DB_PASSWORD" diff --git a/helm/sim/examples/values-external-secrets.yaml b/helm/sim/examples/values-external-secrets.yaml new file mode 100644 index 0000000000..27aec65043 --- /dev/null +++ b/helm/sim/examples/values-external-secrets.yaml @@ -0,0 +1,192 @@ +# Example: External Secrets Operator Integration +# +# This example shows how to integrate with External Secrets Operator (ESO) +# to automatically sync secrets from external secret management systems. +# +# Prerequisites: +# 1. Install External Secrets Operator in your cluster: +# helm repo add external-secrets https://charts.external-secrets.io +# helm install external-secrets external-secrets/external-secrets \ +# -n external-secrets --create-namespace +# +# 2. Create a SecretStore or ClusterSecretStore for your provider. +# See examples below for Azure Key Vault, AWS Secrets Manager, and HashiCorp Vault. +# +# Documentation: https://external-secrets.io/ + +# ============================================================================= +# EXTERNAL SECRETS OPERATOR CONFIGURATION +# ============================================================================= +externalSecrets: + enabled: true + apiVersion: "v1" # Use "v1" for ESO v0.17+ (default), "v1beta1" for older versions + refreshInterval: "1h" # How often to sync secrets + secretStoreRef: + name: "sim-secret-store" # Name of your SecretStore/ClusterSecretStore + kind: "ClusterSecretStore" # or "SecretStore" for namespace-scoped + remoteRefs: + app: + BETTER_AUTH_SECRET: "sim/app/better-auth-secret" + ENCRYPTION_KEY: "sim/app/encryption-key" + INTERNAL_API_SECRET: "sim/app/internal-api-secret" + CRON_SECRET: "sim/app/cron-secret" + API_ENCRYPTION_KEY: "sim/app/api-encryption-key" + postgresql: + password: "sim/postgresql/password" + +# ============================================================================= +# APPLICATION CONFIGURATION +# ============================================================================= +app: + enabled: true + replicaCount: 2 + env: + NEXT_PUBLIC_APP_URL: "https://sim.example.com" + BETTER_AUTH_URL: "https://sim.example.com" + NEXT_PUBLIC_SOCKET_URL: "wss://sim-ws.example.com" + NODE_ENV: "production" + # Note: Sensitive values (BETTER_AUTH_SECRET, ENCRYPTION_KEY, etc.) + # are injected from External Secrets and don't need to be set here + +realtime: + enabled: true + replicaCount: 2 + env: + NEXT_PUBLIC_APP_URL: "https://sim.example.com" + BETTER_AUTH_URL: "https://sim.example.com" + NEXT_PUBLIC_SOCKET_URL: "wss://sim-ws.example.com" + ALLOWED_ORIGINS: "https://sim.example.com" + NODE_ENV: "production" + +postgresql: + enabled: true + # Password is injected from External Secrets + auth: + username: postgres + database: sim + +# ============================================================================= +# EXAMPLE SECRETSTORE CONFIGURATIONS +# ============================================================================= +# Below are example SecretStore configurations for popular providers. +# Apply ONE of these to your cluster before installing the Sim chart. + +# ----------------------------------------------------------------------------- +# Azure Key Vault (with Workload Identity - Recommended) +# ----------------------------------------------------------------------------- +# apiVersion: external-secrets.io/v1beta1 +# kind: ClusterSecretStore +# metadata: +# name: sim-secret-store +# spec: +# provider: +# azurekv: +# authType: WorkloadIdentity +# vaultUrl: "https://your-keyvault-name.vault.azure.net" +# serviceAccountRef: +# name: external-secrets-sa +# namespace: external-secrets + +# ----------------------------------------------------------------------------- +# Azure Key Vault (with Service Principal) +# ----------------------------------------------------------------------------- +# apiVersion: external-secrets.io/v1beta1 +# kind: ClusterSecretStore +# metadata: +# name: sim-secret-store +# spec: +# provider: +# azurekv: +# tenantId: "your-tenant-id" +# vaultUrl: "https://your-keyvault-name.vault.azure.net" +# authSecretRef: +# clientId: +# name: azure-sp-credentials +# key: client-id +# namespace: external-secrets +# clientSecret: +# name: azure-sp-credentials +# key: client-secret +# namespace: external-secrets + +# ----------------------------------------------------------------------------- +# AWS Secrets Manager (with IRSA - Recommended) +# ----------------------------------------------------------------------------- +# apiVersion: external-secrets.io/v1beta1 +# kind: ClusterSecretStore +# metadata: +# name: sim-secret-store +# spec: +# provider: +# aws: +# service: SecretsManager +# region: us-east-1 +# role: arn:aws:iam::123456789012:role/external-secrets-role + +# ----------------------------------------------------------------------------- +# AWS Secrets Manager (with static credentials) +# ----------------------------------------------------------------------------- +# apiVersion: external-secrets.io/v1beta1 +# kind: ClusterSecretStore +# metadata: +# name: sim-secret-store +# spec: +# provider: +# aws: +# service: SecretsManager +# region: us-east-1 +# auth: +# secretRef: +# accessKeyIDSecretRef: +# name: aws-credentials +# key: access-key-id +# namespace: external-secrets +# secretAccessKeySecretRef: +# name: aws-credentials +# key: secret-access-key +# namespace: external-secrets + +# ----------------------------------------------------------------------------- +# HashiCorp Vault (with Kubernetes Auth - Recommended) +# ----------------------------------------------------------------------------- +# apiVersion: external-secrets.io/v1beta1 +# kind: ClusterSecretStore +# metadata: +# name: sim-secret-store +# spec: +# provider: +# vault: +# server: "https://vault.example.com" +# path: "secret" +# version: "v2" +# auth: +# kubernetes: +# mountPath: "kubernetes" +# role: "external-secrets" +# serviceAccountRef: +# name: external-secrets-sa +# namespace: external-secrets + +# ----------------------------------------------------------------------------- +# Google Cloud Secret Manager (with Workload Identity - Recommended) +# ----------------------------------------------------------------------------- +# apiVersion: external-secrets.io/v1beta1 +# kind: ClusterSecretStore +# metadata: +# name: sim-secret-store +# spec: +# provider: +# gcpsm: +# projectID: your-gcp-project-id + +# ============================================================================= +# SECRETS TO CREATE IN YOUR EXTERNAL SECRET STORE +# ============================================================================= +# Create the following secrets in your external store with these paths: +# +# sim/app/better-auth-secret - 64 char hex string (openssl rand -hex 32) +# sim/app/encryption-key - 64 char hex string (openssl rand -hex 32) +# sim/app/internal-api-secret - 64 char hex string (openssl rand -hex 32) +# sim/app/cron-secret - 64 char hex string (openssl rand -hex 32) +# sim/app/api-encryption-key - 64 char hex string (openssl rand -hex 32) +# sim/postgresql/password - Alphanumeric string (openssl rand -base64 16 | tr -d '/+=') diff --git a/helm/sim/templates/_helpers.tpl b/helm/sim/templates/_helpers.tpl index 9966b14937..e1bee30491 100644 --- a/helm/sim/templates/_helpers.tpl +++ b/helm/sim/templates/_helpers.tpl @@ -181,8 +181,15 @@ Database URL for internal PostgreSQL {{/* Validate required secrets and reject default placeholder values +Skip validation when using existing secrets or External Secrets Operator */}} {{- define "sim.validateSecrets" -}} +{{- $useExistingAppSecret := and .Values.app.secrets .Values.app.secrets.existingSecret .Values.app.secrets.existingSecret.enabled }} +{{- $useExternalSecrets := and .Values.externalSecrets .Values.externalSecrets.enabled }} +{{- $useExistingPostgresSecret := and .Values.postgresql.auth.existingSecret .Values.postgresql.auth.existingSecret.enabled }} +{{- $useExistingExternalDbSecret := and .Values.externalDatabase.existingSecret .Values.externalDatabase.existingSecret.enabled }} +{{- /* App secrets validation - skip if using existing secret or ESO */ -}} +{{- if not (or $useExistingAppSecret $useExternalSecrets) }} {{- if and .Values.app.enabled (not .Values.app.env.BETTER_AUTH_SECRET) }} {{- fail "app.env.BETTER_AUTH_SECRET is required for production deployment" }} {{- end }} @@ -198,15 +205,21 @@ Validate required secrets and reject default placeholder values {{- if and .Values.realtime.enabled (eq .Values.realtime.env.BETTER_AUTH_SECRET "CHANGE-ME-32-CHAR-SECRET-FOR-PRODUCTION-USE") }} {{- fail "realtime.env.BETTER_AUTH_SECRET must not use the default placeholder value. Generate a secure secret with: openssl rand -hex 32" }} {{- end }} +{{- end }} +{{- /* PostgreSQL password validation - skip if using existing secret or ESO */ -}} +{{- if not (or $useExistingPostgresSecret $useExternalSecrets) }} {{- if and .Values.postgresql.enabled (not .Values.postgresql.auth.password) }} {{- fail "postgresql.auth.password is required when using internal PostgreSQL" }} {{- end }} {{- if and .Values.postgresql.enabled (eq .Values.postgresql.auth.password "CHANGE-ME-SECURE-PASSWORD") }} {{- fail "postgresql.auth.password must not use the default placeholder value. Set a secure password for production" }} {{- end }} -{{- if and .Values.postgresql.enabled (not (regexMatch "^[a-zA-Z0-9._-]+$" .Values.postgresql.auth.password)) }} +{{- if and .Values.postgresql.enabled .Values.postgresql.auth.password (not (regexMatch "^[a-zA-Z0-9._-]+$" .Values.postgresql.auth.password)) }} {{- fail "postgresql.auth.password must only contain alphanumeric characters, hyphens, underscores, or periods to ensure DATABASE_URL compatibility. Generate with: openssl rand -base64 16 | tr -d '/+='" }} {{- end }} +{{- end }} +{{- /* External database password validation - skip if using existing secret or ESO */ -}} +{{- if not (or $useExistingExternalDbSecret $useExternalSecrets) }} {{- if and .Values.externalDatabase.enabled (not .Values.externalDatabase.password) }} {{- fail "externalDatabase.password is required when using external database" }} {{- end }} @@ -214,6 +227,103 @@ Validate required secrets and reject default placeholder values {{- fail "externalDatabase.password must only contain alphanumeric characters, hyphens, underscores, or periods to ensure DATABASE_URL compatibility." }} {{- end }} {{- end }} +{{- end }} + +{{/* +Get the app secrets name +Returns the name of the secret containing app credentials (auth, encryption keys) +*/}} +{{- define "sim.appSecretName" -}} +{{- if and .Values.app.secrets .Values.app.secrets.existingSecret .Values.app.secrets.existingSecret.enabled -}} +{{- .Values.app.secrets.existingSecret.name -}} +{{- else -}} +{{- printf "%s-app-secrets" (include "sim.fullname" .) -}} +{{- end -}} +{{- end }} + +{{/* +Get the PostgreSQL secret name +Returns the name of the secret containing PostgreSQL password +*/}} +{{- define "sim.postgresqlSecretName" -}} +{{- if and .Values.postgresql.auth.existingSecret .Values.postgresql.auth.existingSecret.enabled -}} +{{- .Values.postgresql.auth.existingSecret.name -}} +{{- else -}} +{{- printf "%s-postgresql-secret" (include "sim.fullname" .) -}} +{{- end -}} +{{- end }} + +{{/* +Get the PostgreSQL password key name +Returns the key name in the secret that contains the password +*/}} +{{- define "sim.postgresqlPasswordKey" -}} +{{- if and .Values.postgresql.auth.existingSecret .Values.postgresql.auth.existingSecret.enabled -}} +{{- .Values.postgresql.auth.existingSecret.passwordKey | default "POSTGRES_PASSWORD" -}} +{{- else -}} +{{- print "POSTGRES_PASSWORD" -}} +{{- end -}} +{{- end }} + +{{/* +Get the external database secret name +Returns the name of the secret containing external database password +*/}} +{{- define "sim.externalDbSecretName" -}} +{{- if and .Values.externalDatabase.existingSecret .Values.externalDatabase.existingSecret.enabled -}} +{{- .Values.externalDatabase.existingSecret.name -}} +{{- else -}} +{{- printf "%s-external-db-secret" (include "sim.fullname" .) -}} +{{- end -}} +{{- end }} + +{{/* +Get the external database password key name +Returns the key name in the secret that contains the password +*/}} +{{- define "sim.externalDbPasswordKey" -}} +{{- if and .Values.externalDatabase.existingSecret .Values.externalDatabase.existingSecret.enabled -}} +{{- .Values.externalDatabase.existingSecret.passwordKey | default "EXTERNAL_DB_PASSWORD" -}} +{{- else -}} +{{- print "EXTERNAL_DB_PASSWORD" -}} +{{- end -}} +{{- end }} + +{{/* +Check if app secrets should be created by the chart +Returns true if we should create the app secrets (not using existing or ESO) +*/}} +{{- define "sim.createAppSecrets" -}} +{{- $useExistingAppSecret := and .Values.app.secrets .Values.app.secrets.existingSecret .Values.app.secrets.existingSecret.enabled }} +{{- $useExternalSecrets := and .Values.externalSecrets .Values.externalSecrets.enabled }} +{{- if not (or $useExistingAppSecret $useExternalSecrets) -}} +true +{{- end -}} +{{- end }} + +{{/* +Check if PostgreSQL secret should be created by the chart +Returns true if we should create the PostgreSQL secret (not using existing or ESO) +*/}} +{{- define "sim.createPostgresqlSecret" -}} +{{- $useExistingSecret := and .Values.postgresql.auth.existingSecret .Values.postgresql.auth.existingSecret.enabled }} +{{- $useExternalSecrets := and .Values.externalSecrets .Values.externalSecrets.enabled }} +{{- if not (or $useExistingSecret $useExternalSecrets) -}} +true +{{- end -}} +{{- end }} + +{{/* +Check if external database secret should be created by the chart +Returns true if we should create the external database secret (not using existing or ESO) +*/}} +{{- define "sim.createExternalDbSecret" -}} +{{- $useExistingSecret := and .Values.externalDatabase.existingSecret .Values.externalDatabase.existingSecret.enabled }} +{{- $useExternalSecrets := and .Values.externalSecrets .Values.externalSecrets.enabled }} +{{- if not (or $useExistingSecret $useExternalSecrets) -}} +true +{{- end -}} +{{- end }} {{/* Ollama URL diff --git a/helm/sim/templates/deployment-app.yaml b/helm/sim/templates/deployment-app.yaml index 6433e82ea0..04eedd755d 100644 --- a/helm/sim/templates/deployment-app.yaml +++ b/helm/sim/templates/deployment-app.yaml @@ -44,15 +44,14 @@ spec: cd /app/packages/db export DATABASE_URL="{{ include "sim.databaseUrl" . }}" bun run db:migrate - {{- if .Values.postgresql.enabled }} envFrom: + {{- if .Values.postgresql.enabled }} - secretRef: - name: {{ include "sim.fullname" . }}-postgresql-secret - {{- else if .Values.externalDatabase.enabled }} - envFrom: + name: {{ include "sim.postgresqlSecretName" . }} + {{- else if .Values.externalDatabase.enabled }} - secretRef: - name: {{ include "sim.fullname" . }}-external-db-secret - {{- end }} + name: {{ include "sim.externalDbSecretName" . }} + {{- end }} {{- include "sim.resources" .Values.migrations | nindent 10 }} {{- include "sim.securityContext" .Values.migrations | nindent 10 }} {{- end }} @@ -89,15 +88,18 @@ spec: {{- with .Values.extraEnvVars }} {{- toYaml . | nindent 12 }} {{- end }} - {{- if .Values.postgresql.enabled }} envFrom: + # App secrets (authentication, encryption keys) - secretRef: - name: {{ include "sim.fullname" . }}-postgresql-secret - {{- else if .Values.externalDatabase.enabled }} - envFrom: + name: {{ include "sim.appSecretName" . }} + # Database secrets + {{- if .Values.postgresql.enabled }} - secretRef: - name: {{ include "sim.fullname" . }}-external-db-secret - {{- end }} + name: {{ include "sim.postgresqlSecretName" . }} + {{- else if .Values.externalDatabase.enabled }} + - secretRef: + name: {{ include "sim.externalDbSecretName" . }} + {{- end }} {{- if .Values.app.livenessProbe }} livenessProbe: {{- toYaml .Values.app.livenessProbe | nindent 12 }} diff --git a/helm/sim/templates/deployment-realtime.yaml b/helm/sim/templates/deployment-realtime.yaml index b951aee167..746516594e 100644 --- a/helm/sim/templates/deployment-realtime.yaml +++ b/helm/sim/templates/deployment-realtime.yaml @@ -62,15 +62,18 @@ spec: {{- with .Values.extraEnvVars }} {{- toYaml . | nindent 12 }} {{- end }} - {{- if .Values.postgresql.enabled }} envFrom: + # App secrets (authentication keys shared with main app) - secretRef: - name: {{ include "sim.fullname" . }}-postgresql-secret - {{- else if .Values.externalDatabase.enabled }} - envFrom: + name: {{ include "sim.appSecretName" . }} + # Database secrets + {{- if .Values.postgresql.enabled }} - secretRef: - name: {{ include "sim.fullname" . }}-external-db-secret - {{- end }} + name: {{ include "sim.postgresqlSecretName" . }} + {{- else if .Values.externalDatabase.enabled }} + - secretRef: + name: {{ include "sim.externalDbSecretName" . }} + {{- end }} {{- if .Values.realtime.livenessProbe }} livenessProbe: {{- toYaml .Values.realtime.livenessProbe | nindent 12 }} diff --git a/helm/sim/templates/external-db-secret.yaml b/helm/sim/templates/external-db-secret.yaml index 7cfa527f64..d4576e7c37 100644 --- a/helm/sim/templates/external-db-secret.yaml +++ b/helm/sim/templates/external-db-secret.yaml @@ -1,6 +1,7 @@ -{{- if .Values.externalDatabase.enabled }} +{{- if and .Values.externalDatabase.enabled (include "sim.createExternalDbSecret" .) }} --- # Secret for external database credentials +# Only created when not using existingSecret or External Secrets Operator apiVersion: v1 kind: Secret metadata: diff --git a/helm/sim/templates/external-secret-app.yaml b/helm/sim/templates/external-secret-app.yaml new file mode 100644 index 0000000000..fa74229c7a --- /dev/null +++ b/helm/sim/templates/external-secret-app.yaml @@ -0,0 +1,48 @@ +{{/* +ExternalSecret for App Secrets - Syncs from external secret managers +Only created when External Secrets Operator integration is enabled +Requires ESO to be installed: https://external-secrets.io/ +*/}} +{{- if and .Values.externalSecrets.enabled .Values.app.enabled }} +apiVersion: external-secrets.io/{{ .Values.externalSecrets.apiVersion | default "v1" }} +kind: ExternalSecret +metadata: + name: {{ include "sim.fullname" . }}-app-secrets + namespace: {{ .Release.Namespace }} + labels: + {{- include "sim.app.labels" . | nindent 4 }} +spec: + refreshInterval: {{ .Values.externalSecrets.refreshInterval | quote }} + secretStoreRef: + name: {{ required "externalSecrets.secretStoreRef.name is required when externalSecrets.enabled=true" .Values.externalSecrets.secretStoreRef.name }} + kind: {{ .Values.externalSecrets.secretStoreRef.kind | default "ClusterSecretStore" }} + target: + name: {{ include "sim.fullname" . }}-app-secrets + creationPolicy: Owner + data: + {{- if .Values.externalSecrets.remoteRefs.app.BETTER_AUTH_SECRET }} + - secretKey: BETTER_AUTH_SECRET + remoteRef: + key: {{ .Values.externalSecrets.remoteRefs.app.BETTER_AUTH_SECRET }} + {{- end }} + {{- if .Values.externalSecrets.remoteRefs.app.ENCRYPTION_KEY }} + - secretKey: ENCRYPTION_KEY + remoteRef: + key: {{ .Values.externalSecrets.remoteRefs.app.ENCRYPTION_KEY }} + {{- end }} + {{- if .Values.externalSecrets.remoteRefs.app.INTERNAL_API_SECRET }} + - secretKey: INTERNAL_API_SECRET + remoteRef: + key: {{ .Values.externalSecrets.remoteRefs.app.INTERNAL_API_SECRET }} + {{- end }} + {{- if .Values.externalSecrets.remoteRefs.app.CRON_SECRET }} + - secretKey: CRON_SECRET + remoteRef: + key: {{ .Values.externalSecrets.remoteRefs.app.CRON_SECRET }} + {{- end }} + {{- if .Values.externalSecrets.remoteRefs.app.API_ENCRYPTION_KEY }} + - secretKey: API_ENCRYPTION_KEY + remoteRef: + key: {{ .Values.externalSecrets.remoteRefs.app.API_ENCRYPTION_KEY }} + {{- end }} +{{- end }} diff --git a/helm/sim/templates/external-secret-external-db.yaml b/helm/sim/templates/external-secret-external-db.yaml new file mode 100644 index 0000000000..8480d152c7 --- /dev/null +++ b/helm/sim/templates/external-secret-external-db.yaml @@ -0,0 +1,27 @@ +{{/* +ExternalSecret for External Database - Syncs password from external secret managers +Only created when External Secrets Operator integration is enabled and external database is used +Requires ESO to be installed: https://external-secrets.io/ +*/}} +{{- if and .Values.externalSecrets.enabled .Values.externalDatabase.enabled .Values.externalSecrets.remoteRefs.externalDatabase.password }} +apiVersion: external-secrets.io/{{ .Values.externalSecrets.apiVersion | default "v1" }} +kind: ExternalSecret +metadata: + name: {{ include "sim.fullname" . }}-external-db-secret + namespace: {{ .Release.Namespace }} + labels: + {{- include "sim.labels" . | nindent 4 }} + app.kubernetes.io/component: external-database +spec: + refreshInterval: {{ .Values.externalSecrets.refreshInterval | quote }} + secretStoreRef: + name: {{ required "externalSecrets.secretStoreRef.name is required when externalSecrets.enabled=true" .Values.externalSecrets.secretStoreRef.name }} + kind: {{ .Values.externalSecrets.secretStoreRef.kind | default "ClusterSecretStore" }} + target: + name: {{ include "sim.fullname" . }}-external-db-secret + creationPolicy: Owner + data: + - secretKey: EXTERNAL_DB_PASSWORD + remoteRef: + key: {{ .Values.externalSecrets.remoteRefs.externalDatabase.password }} +{{- end }} diff --git a/helm/sim/templates/external-secret-postgresql.yaml b/helm/sim/templates/external-secret-postgresql.yaml new file mode 100644 index 0000000000..4ae0045b15 --- /dev/null +++ b/helm/sim/templates/external-secret-postgresql.yaml @@ -0,0 +1,26 @@ +{{/* +ExternalSecret for PostgreSQL - Syncs password from external secret managers +Only created when External Secrets Operator integration is enabled and internal PostgreSQL is used +Requires ESO to be installed: https://external-secrets.io/ +*/}} +{{- if and .Values.externalSecrets.enabled .Values.postgresql.enabled .Values.externalSecrets.remoteRefs.postgresql.password }} +apiVersion: external-secrets.io/{{ .Values.externalSecrets.apiVersion | default "v1" }} +kind: ExternalSecret +metadata: + name: {{ include "sim.fullname" . }}-postgresql-secret + namespace: {{ .Release.Namespace }} + labels: + {{- include "sim.postgresql.labels" . | nindent 4 }} +spec: + refreshInterval: {{ .Values.externalSecrets.refreshInterval | quote }} + secretStoreRef: + name: {{ required "externalSecrets.secretStoreRef.name is required when externalSecrets.enabled=true" .Values.externalSecrets.secretStoreRef.name }} + kind: {{ .Values.externalSecrets.secretStoreRef.kind | default "ClusterSecretStore" }} + target: + name: {{ include "sim.fullname" . }}-postgresql-secret + creationPolicy: Owner + data: + - secretKey: POSTGRES_PASSWORD + remoteRef: + key: {{ .Values.externalSecrets.remoteRefs.postgresql.password }} +{{- end }} diff --git a/helm/sim/templates/secrets-app.yaml b/helm/sim/templates/secrets-app.yaml new file mode 100644 index 0000000000..b031aa7659 --- /dev/null +++ b/helm/sim/templates/secrets-app.yaml @@ -0,0 +1,30 @@ +{{/* +App Secrets - Authentication and encryption keys +Only created when not using existingSecret or External Secrets Operator +*/}} +{{- if and .Values.app.enabled (include "sim.createAppSecrets" .) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "sim.fullname" . }}-app-secrets + namespace: {{ .Release.Namespace }} + labels: + {{- include "sim.app.labels" . | nindent 4 }} +type: Opaque +stringData: + {{- if .Values.app.env.BETTER_AUTH_SECRET }} + BETTER_AUTH_SECRET: {{ .Values.app.env.BETTER_AUTH_SECRET | quote }} + {{- end }} + {{- if .Values.app.env.ENCRYPTION_KEY }} + ENCRYPTION_KEY: {{ .Values.app.env.ENCRYPTION_KEY | quote }} + {{- end }} + {{- if .Values.app.env.INTERNAL_API_SECRET }} + INTERNAL_API_SECRET: {{ .Values.app.env.INTERNAL_API_SECRET | quote }} + {{- end }} + {{- if .Values.app.env.CRON_SECRET }} + CRON_SECRET: {{ .Values.app.env.CRON_SECRET | quote }} + {{- end }} + {{- if .Values.app.env.API_ENCRYPTION_KEY }} + API_ENCRYPTION_KEY: {{ .Values.app.env.API_ENCRYPTION_KEY | quote }} + {{- end }} +{{- end }} diff --git a/helm/sim/templates/statefulset-postgresql.yaml b/helm/sim/templates/statefulset-postgresql.yaml index f9283b6141..8422250dac 100644 --- a/helm/sim/templates/statefulset-postgresql.yaml +++ b/helm/sim/templates/statefulset-postgresql.yaml @@ -65,8 +65,10 @@ data: POSTGRES_USER: {{ .Values.postgresql.auth.username | quote }} PGDATA: "/var/lib/postgresql/data/pgdata" +{{- if (include "sim.createPostgresqlSecret" .) }} --- # Secret for PostgreSQL password +# Only created when not using existingSecret or External Secrets Operator apiVersion: v1 kind: Secret metadata: @@ -77,6 +79,7 @@ metadata: type: Opaque data: POSTGRES_PASSWORD: {{ .Values.postgresql.auth.password | b64enc }} +{{- end }} --- # StatefulSet for PostgreSQL @@ -128,7 +131,7 @@ spec: - configMapRef: name: {{ include "sim.fullname" . }}-postgresql-env - secretRef: - name: {{ include "sim.fullname" . }}-postgresql-secret + name: {{ include "sim.postgresqlSecretName" . }} {{- if .Values.postgresql.livenessProbe }} livenessProbe: {{- toYaml .Values.postgresql.livenessProbe | nindent 12 }} diff --git a/helm/sim/values.schema.json b/helm/sim/values.schema.json index 6aa96f1c2b..9eb8fe8ec9 100644 --- a/helm/sim/values.schema.json +++ b/helm/sim/values.schema.json @@ -81,18 +81,39 @@ } } }, + "secrets": { + "type": "object", + "description": "Secret management configuration", + "properties": { + "existingSecret": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Use an existing secret instead of creating one" + }, + "name": { + "type": "string", + "description": "Name of the existing Kubernetes secret" + }, + "keys": { + "type": "object", + "description": "Key name mappings in the existing secret" + } + } + } + } + }, "env": { "type": "object", "properties": { "BETTER_AUTH_SECRET": { "type": "string", - "minLength": 32, - "description": "Auth secret (minimum 32 characters required)" + "description": "Auth secret (minimum 32 characters required when not using existingSecret)" }, "ENCRYPTION_KEY": { "type": "string", - "minLength": 32, - "description": "Encryption key (minimum 32 characters required)" + "description": "Encryption key (minimum 32 characters required when not using existingSecret)" }, "NEXT_PUBLIC_APP_URL": { "type": "string", @@ -329,8 +350,7 @@ "properties": { "BETTER_AUTH_SECRET": { "type": "string", - "minLength": 32, - "description": "Auth secret (minimum 32 characters required)" + "description": "Auth secret (minimum 32 characters required when not using existingSecret)" }, "NEXT_PUBLIC_APP_URL": { "type": "string", @@ -431,11 +451,25 @@ }, "password": { "type": "string", - "minLength": 8, - "not": { - "const": "CHANGE-ME-SECURE-PASSWORD" - }, - "description": "PostgreSQL password (minimum 8 characters, must not be default placeholder)" + "description": "PostgreSQL password (minimum 8 characters when not using existingSecret)" + }, + "existingSecret": { + "type": "object", + "description": "Use an existing secret for PostgreSQL credentials", + "properties": { + "enabled": { + "type": "boolean", + "description": "Use an existing secret instead of creating one" + }, + "name": { + "type": "string", + "description": "Name of the existing Kubernetes secret" + }, + "passwordKey": { + "type": "string", + "description": "Key in the secret containing the password" + } + } } } } @@ -475,6 +509,24 @@ "type": "string", "enum": ["disable", "allow", "prefer", "require", "verify-ca", "verify-full"], "description": "SSL mode for database connection" + }, + "existingSecret": { + "type": "object", + "description": "Use an existing secret for external database credentials", + "properties": { + "enabled": { + "type": "boolean", + "description": "Use an existing secret instead of creating one" + }, + "name": { + "type": "string", + "description": "Name of the existing Kubernetes secret" + }, + "passwordKey": { + "type": "string", + "description": "Key in the secret containing the password" + } + } } }, "if": { @@ -821,6 +873,61 @@ } } }, + "externalSecrets": { + "type": "object", + "description": "External Secrets Operator integration", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable External Secrets Operator integration" + }, + "apiVersion": { + "type": "string", + "enum": ["v1", "v1beta1"], + "description": "ESO API version - use v1 for ESO v0.17+ (recommended), v1beta1 for older versions" + }, + "refreshInterval": { + "type": "string", + "description": "How often to sync secrets from external store" + }, + "secretStoreRef": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name of the SecretStore or ClusterSecretStore" + }, + "kind": { + "type": "string", + "enum": ["SecretStore", "ClusterSecretStore"], + "description": "Kind of the store" + } + } + }, + "remoteRefs": { + "type": "object", + "description": "Remote key paths in external secret store", + "properties": { + "app": { + "type": "object", + "additionalProperties": { "type": "string" } + }, + "postgresql": { + "type": "object", + "properties": { + "password": { "type": "string" } + } + }, + "externalDatabase": { + "type": "object", + "properties": { + "password": { "type": "string" } + } + } + } + } + } + }, "ingress": { "type": "object", "properties": { diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index 788d900cbd..24e794a9ca 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -16,16 +16,16 @@ global: app: # Enable/disable the main application enabled: true - + # Image configuration image: repository: simstudioai/simstudio tag: latest pullPolicy: Always - + # Number of replicas replicaCount: 1 - + # Resource limits and requests resources: limits: @@ -34,19 +34,37 @@ app: requests: memory: "2Gi" cpu: "1000m" - + # Node selector for pod scheduling (leave empty to allow scheduling on any node) nodeSelector: {} - + # Pod security context podSecurityContext: fsGroup: 1001 - + # Container security context securityContext: runAsNonRoot: true runAsUser: 1001 - + + # Secret management configuration + # Use this to reference pre-existing Kubernetes secrets instead of defining values directly + # This enables integration with External Secrets Operator, HashiCorp Vault, Azure Key Vault, etc. + secrets: + existingSecret: + # Set to true to use an existing secret instead of creating one from values + enabled: false + # Name of the existing Kubernetes secret containing app credentials + name: "" + # Key mappings - specify the key names in your existing secret + # Only needed if your secret uses different key names than the defaults + keys: + BETTER_AUTH_SECRET: "BETTER_AUTH_SECRET" + ENCRYPTION_KEY: "ENCRYPTION_KEY" + INTERNAL_API_SECRET: "INTERNAL_API_SECRET" + CRON_SECRET: "CRON_SECRET" + API_ENCRYPTION_KEY: "API_ENCRYPTION_KEY" + # Environment variables env: # Application URLs @@ -118,7 +136,9 @@ app: # Registration Control DISABLE_REGISTRATION: "" # Set to "true" to disable new user signups - + EMAIL_PASSWORD_SIGNUP_ENABLED: "" # Set to "false" to disable email/password login (SSO-only mode, server-side enforcement) + NEXT_PUBLIC_EMAIL_PASSWORD_SIGNUP_ENABLED: "" # Set to "false" to hide email/password login form (UI-side) + # Access Control (leave empty if not restricting login) ALLOWED_LOGIN_EMAILS: "" # Comma-separated list of allowed email addresses for login ALLOWED_LOGIN_DOMAINS: "" # Comma-separated list of allowed email domains for login @@ -305,6 +325,12 @@ postgresql: username: postgres password: "" # REQUIRED - set via --set flag or external secret manager database: sim + # Use an existing secret for PostgreSQL credentials + # This enables integration with External Secrets Operator, HashiCorp Vault, etc. + existingSecret: + enabled: false + name: "" # Name of existing Kubernetes secret + passwordKey: "POSTGRES_PASSWORD" # Key in the secret containing the password # Node selector for database pod scheduling (leave empty to allow scheduling on any node) nodeSelector: {} @@ -387,17 +413,24 @@ postgresql: externalDatabase: # Enable to use an external database instead of the internal PostgreSQL instance enabled: false - + # Database connection details host: "external-db.example.com" port: 5432 username: postgres password: "" database: sim - + # SSL configuration sslMode: require + # Use an existing secret for external database credentials + # This enables integration with External Secrets Operator, HashiCorp Vault, etc. + existingSecret: + enabled: false + name: "" # Name of existing Kubernetes secret + passwordKey: "EXTERNAL_DB_PASSWORD" # Key in the secret containing the password + # Ollama local AI models configuration ollama: # Enable/disable Ollama deployment @@ -1013,4 +1046,51 @@ copilot: # Job configuration backoffLimit: 3 - restartPolicy: OnFailure \ No newline at end of file + restartPolicy: OnFailure + +# External Secrets Operator integration +# Use this to automatically sync secrets from external secret managers (Azure Key Vault, AWS Secrets Manager, etc.) +# Prerequisites: Install External Secrets Operator in your cluster first +# See: https://external-secrets.io/latest/introduction/getting-started/ +externalSecrets: + # Enable External Secrets Operator integration + enabled: false + + # ESO API version - use "v1" for ESO v0.17+ (recommended), "v1beta1" for older versions + apiVersion: "v1" + + # How often to sync secrets from the external store + refreshInterval: "1h" + + # Reference to the SecretStore or ClusterSecretStore + secretStoreRef: + # Name of the SecretStore or ClusterSecretStore resource + name: "" + # Kind of the store: "SecretStore" (namespaced) or "ClusterSecretStore" (cluster-wide) + kind: "ClusterSecretStore" + + # Remote references - paths/keys in your external secret store + # These map to the secrets that will be created in Kubernetes + remoteRefs: + # App secrets (authentication, encryption keys) + app: + # Path to BETTER_AUTH_SECRET in external store (e.g., "sim/app/better-auth-secret") + BETTER_AUTH_SECRET: "" + # Path to ENCRYPTION_KEY in external store + ENCRYPTION_KEY: "" + # Path to INTERNAL_API_SECRET in external store + INTERNAL_API_SECRET: "" + # Path to CRON_SECRET in external store (optional) + CRON_SECRET: "" + # Path to API_ENCRYPTION_KEY in external store (optional) + API_ENCRYPTION_KEY: "" + + # PostgreSQL password (for internal PostgreSQL) + postgresql: + # Path to PostgreSQL password in external store (e.g., "sim/postgresql/password") + password: "" + + # External database password (when using managed database services) + externalDatabase: + # Path to external database password in external store + password: "" \ No newline at end of file From 307eee7929361319d9618202e82ec15fc31ce978 Mon Sep 17 00:00:00 2001 From: waleed Date: Thu, 8 Jan 2026 10:26:50 -0800 Subject: [PATCH 2/5] consolidated telemetry events --- apps/sim/app/api/chat/route.ts | 12 + .../app/api/knowledge/[id]/documents/route.ts | 30 +- apps/sim/app/api/knowledge/[id]/route.ts | 9 + apps/sim/app/api/knowledge/route.ts | 11 + apps/sim/app/api/knowledge/search/route.ts | 11 + apps/sim/app/api/mcp/servers/route.ts | 12 +- apps/sim/app/api/mcp/tools/execute/route.ts | 12 +- apps/sim/app/api/templates/[id]/use/route.ts | 17 +- apps/sim/app/api/webhooks/[id]/route.ts | 22 + apps/sim/app/api/webhooks/route.ts | 14 + .../app/api/workflows/[id]/deploy/route.ts | 6 +- .../app/api/workflows/[id]/duplicate/route.ts | 11 + apps/sim/app/api/workflows/[id]/route.ts | 10 + apps/sim/app/api/workflows/route.ts | 12 +- .../app/api/workspaces/[id]/api-keys/route.ts | 21 + .../app/api/workspaces/invitations/route.ts | 26 +- apps/sim/app/api/workspaces/route.ts | 30 +- .../utils/workflow-execution-utils.ts | 9 - apps/sim/instrumentation-node.ts | 61 ++- apps/sim/lib/auth/auth.ts | 19 + apps/sim/lib/core/telemetry.ts | 502 ++++++++++++++++++ .../sim/lib/logs/execution/logging-session.ts | 79 ++- apps/sim/lib/workflows/persistence/utils.ts | 21 +- helm/sim/examples/values-existing-secret.yaml | 108 +--- .../sim/examples/values-external-secrets.yaml | 130 +---- helm/sim/templates/external-db-secret.yaml | 1 - helm/sim/templates/external-secret-app.yaml | 6 +- .../external-secret-external-db.yaml | 6 +- .../templates/external-secret-postgresql.yaml | 6 +- helm/sim/templates/secrets-app.yaml | 5 +- .../sim/templates/statefulset-postgresql.yaml | 1 - 31 files changed, 842 insertions(+), 378 deletions(-) diff --git a/apps/sim/app/api/chat/route.ts b/apps/sim/app/api/chat/route.ts index dd736b6529..e9ad9c079a 100644 --- a/apps/sim/app/api/chat/route.ts +++ b/apps/sim/app/api/chat/route.ts @@ -212,6 +212,18 @@ export async function POST(request: NextRequest) { logger.info(`Chat "${title}" deployed successfully at ${chatUrl}`) + try { + const { PlatformEvents } = await import('@/lib/core/telemetry') + PlatformEvents.chatDeployed({ + chatId: id, + workflowId, + authType, + hasOutputConfigs: outputConfigs.length > 0, + }) + } catch (_e) { + // Silently fail + } + return createSuccessResponse({ id, chatUrl, diff --git a/apps/sim/app/api/knowledge/[id]/documents/route.ts b/apps/sim/app/api/knowledge/[id]/documents/route.ts index 7aba07d610..e15fa02209 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/route.ts @@ -198,15 +198,14 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id: `[${requestId}] Starting controlled async processing of ${createdDocuments.length} documents` ) - // Track bulk document upload try { - const { trackPlatformEvent } = await import('@/lib/core/telemetry') - trackPlatformEvent('platform.knowledge_base.documents_uploaded', { - 'knowledge_base.id': knowledgeBaseId, - 'documents.count': createdDocuments.length, - 'documents.upload_type': 'bulk', - 'processing.chunk_size': validatedData.processingOptions.chunkSize, - 'processing.recipe': validatedData.processingOptions.recipe, + const { PlatformEvents } = await import('@/lib/core/telemetry') + PlatformEvents.knowledgeBaseDocumentsUploaded({ + knowledgeBaseId, + documentsCount: createdDocuments.length, + uploadType: 'bulk', + chunkSize: validatedData.processingOptions.chunkSize, + recipe: validatedData.processingOptions.recipe, }) } catch (_e) { // Silently fail @@ -262,15 +261,14 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id: userId ) - // Track single document upload try { - const { trackPlatformEvent } = await import('@/lib/core/telemetry') - trackPlatformEvent('platform.knowledge_base.documents_uploaded', { - 'knowledge_base.id': knowledgeBaseId, - 'documents.count': 1, - 'documents.upload_type': 'single', - 'document.mime_type': validatedData.mimeType, - 'document.file_size': validatedData.fileSize, + const { PlatformEvents } = await import('@/lib/core/telemetry') + PlatformEvents.knowledgeBaseDocumentsUploaded({ + knowledgeBaseId, + documentsCount: 1, + uploadType: 'single', + mimeType: validatedData.mimeType, + fileSize: validatedData.fileSize, }) } catch (_e) { // Silently fail diff --git a/apps/sim/app/api/knowledge/[id]/route.ts b/apps/sim/app/api/knowledge/[id]/route.ts index a26273b4a4..778dd75905 100644 --- a/apps/sim/app/api/knowledge/[id]/route.ts +++ b/apps/sim/app/api/knowledge/[id]/route.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { getSession } from '@/lib/auth' +import { PlatformEvents } from '@/lib/core/telemetry' import { generateRequestId } from '@/lib/core/utils/request' import { deleteKnowledgeBase, @@ -183,6 +184,14 @@ export async function DELETE( await deleteKnowledgeBase(id, requestId) + try { + PlatformEvents.knowledgeBaseDeleted({ + knowledgeBaseId: id, + }) + } catch { + // Telemetry should not fail the operation + } + logger.info(`[${requestId}] Knowledge base deleted: ${id} for user ${session.user.id}`) return NextResponse.json({ diff --git a/apps/sim/app/api/knowledge/route.ts b/apps/sim/app/api/knowledge/route.ts index 3910fca333..07d439fe8c 100644 --- a/apps/sim/app/api/knowledge/route.ts +++ b/apps/sim/app/api/knowledge/route.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { getSession } from '@/lib/auth' +import { PlatformEvents } from '@/lib/core/telemetry' import { generateRequestId } from '@/lib/core/utils/request' import { createKnowledgeBase, getKnowledgeBases } from '@/lib/knowledge/service' @@ -94,6 +95,16 @@ export async function POST(req: NextRequest) { const newKnowledgeBase = await createKnowledgeBase(createData, requestId) + try { + PlatformEvents.knowledgeBaseCreated({ + knowledgeBaseId: newKnowledgeBase.id, + name: validatedData.name, + workspaceId: validatedData.workspaceId, + }) + } catch { + // Telemetry should not fail the operation + } + logger.info( `[${requestId}] Knowledge base created: ${newKnowledgeBase.id} for user ${session.user.id}` ) diff --git a/apps/sim/app/api/knowledge/search/route.ts b/apps/sim/app/api/knowledge/search/route.ts index 6e3f584029..fdfce836b5 100644 --- a/apps/sim/app/api/knowledge/search/route.ts +++ b/apps/sim/app/api/knowledge/search/route.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' +import { PlatformEvents } from '@/lib/core/telemetry' import { generateRequestId } from '@/lib/core/utils/request' import { ALL_TAG_SLOTS } from '@/lib/knowledge/constants' import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service' @@ -294,6 +295,16 @@ export async function POST(request: NextRequest) { const documentIds = results.map((result) => result.documentId) const documentNameMap = await getDocumentNamesByIds(documentIds) + try { + PlatformEvents.knowledgeBaseSearched({ + knowledgeBaseId: accessibleKbIds[0], + resultsCount: results.length, + workspaceId: workspaceId || undefined, + }) + } catch { + // Telemetry should not fail the operation + } + return NextResponse.json({ success: true, data: { diff --git a/apps/sim/app/api/mcp/servers/route.ts b/apps/sim/app/api/mcp/servers/route.ts index 4ba367d133..8f304035b9 100644 --- a/apps/sim/app/api/mcp/servers/route.ts +++ b/apps/sim/app/api/mcp/servers/route.ts @@ -140,12 +140,12 @@ export const POST = withMcpAuth('write')( ) try { - const { trackPlatformEvent } = await import('@/lib/core/telemetry') - trackPlatformEvent('platform.mcp.server_added', { - 'mcp.server_id': serverId, - 'mcp.server_name': body.name, - 'mcp.transport': body.transport, - 'workspace.id': workspaceId, + const { PlatformEvents } = await import('@/lib/core/telemetry') + PlatformEvents.mcpServerAdded({ + serverId, + serverName: body.name, + transport: body.transport, + workspaceId, }) } catch (_e) { // Silently fail diff --git a/apps/sim/app/api/mcp/tools/execute/route.ts b/apps/sim/app/api/mcp/tools/execute/route.ts index 1bcdf6488e..fe0736ba14 100644 --- a/apps/sim/app/api/mcp/tools/execute/route.ts +++ b/apps/sim/app/api/mcp/tools/execute/route.ts @@ -194,12 +194,12 @@ export const POST = withMcpAuth('read')( logger.info(`[${requestId}] Successfully executed tool ${toolName} on server ${serverId}`) try { - const { trackPlatformEvent } = await import('@/lib/core/telemetry') - trackPlatformEvent('platform.mcp.tool_executed', { - 'mcp.server_id': serverId, - 'mcp.tool_name': toolName, - 'mcp.execution_status': 'success', - 'workspace.id': workspaceId, + const { PlatformEvents } = await import('@/lib/core/telemetry') + PlatformEvents.mcpToolExecuted({ + serverId, + toolName, + status: 'success', + workspaceId, }) } catch { // Telemetry failure is non-critical diff --git a/apps/sim/app/api/templates/[id]/use/route.ts b/apps/sim/app/api/templates/[id]/use/route.ts index 26ab63a65a..4ad3bda21e 100644 --- a/apps/sim/app/api/templates/[id]/use/route.ts +++ b/apps/sim/app/api/templates/[id]/use/route.ts @@ -168,18 +168,15 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{ `[${requestId}] Successfully used template: ${id}, created workflow: ${newWorkflowId}` ) - // Track template usage try { - const { trackPlatformEvent } = await import('@/lib/core/telemetry') + const { PlatformEvents } = await import('@/lib/core/telemetry') const templateState = templateData.state as any - trackPlatformEvent('platform.template.used', { - 'template.id': id, - 'template.name': templateData.name, - 'workflow.created_id': newWorkflowId, - 'workflow.blocks_count': templateState?.blocks - ? Object.keys(templateState.blocks).length - : 0, - 'workspace.id': workspaceId, + PlatformEvents.templateUsed({ + templateId: id, + templateName: templateData.name, + newWorkflowId, + blocksCount: templateState?.blocks ? Object.keys(templateState.blocks).length : 0, + workspaceId, }) } catch (_e) { // Silently fail diff --git a/apps/sim/app/api/webhooks/[id]/route.ts b/apps/sim/app/api/webhooks/[id]/route.ts index f5674ff36a..0cd31402df 100644 --- a/apps/sim/app/api/webhooks/[id]/route.ts +++ b/apps/sim/app/api/webhooks/[id]/route.ts @@ -5,6 +5,7 @@ import { and, eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { getSession } from '@/lib/auth' import { validateInteger } from '@/lib/core/security/input-validation' +import { PlatformEvents } from '@/lib/core/telemetry' import { generateRequestId } from '@/lib/core/utils/request' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' @@ -314,6 +315,17 @@ export async function DELETE( await db.delete(webhook).where(eq(webhook.id, wId)) } + try { + for (const wId of idsToDelete) { + PlatformEvents.webhookDeleted({ + webhookId: wId, + workflowId: webhookData.workflow.id, + }) + } + } catch { + // Telemetry should not fail the operation + } + logger.info( `[${requestId}] Successfully deleted ${idsToDelete.length} webhooks for credential set`, { @@ -325,6 +337,16 @@ export async function DELETE( } else { await cleanupExternalWebhook(foundWebhook, webhookData.workflow, requestId) await db.delete(webhook).where(eq(webhook.id, id)) + + try { + PlatformEvents.webhookDeleted({ + webhookId: id, + workflowId: webhookData.workflow.id, + }) + } catch { + // Telemetry should not fail the operation + } + logger.info(`[${requestId}] Successfully deleted webhook: ${id}`) } diff --git a/apps/sim/app/api/webhooks/route.ts b/apps/sim/app/api/webhooks/route.ts index 28f3180b3a..e300294a21 100644 --- a/apps/sim/app/api/webhooks/route.ts +++ b/apps/sim/app/api/webhooks/route.ts @@ -5,6 +5,7 @@ import { and, desc, eq } from 'drizzle-orm' import { nanoid } from 'nanoid' import { type NextRequest, NextResponse } from 'next/server' import { getSession } from '@/lib/auth' +import { PlatformEvents } from '@/lib/core/telemetry' import { generateRequestId } from '@/lib/core/utils/request' import { getBaseUrl } from '@/lib/core/utils/urls' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' @@ -790,6 +791,19 @@ export async function POST(request: NextRequest) { } // --- End Grain specific logic --- + if (!targetWebhookId && savedWebhook) { + try { + PlatformEvents.webhookCreated({ + webhookId: savedWebhook.id, + workflowId: workflowId, + provider: provider || 'generic', + workspaceId: workflowRecord.workspaceId || undefined, + }) + } catch { + // Telemetry should not fail the operation + } + } + const status = targetWebhookId ? 200 : 201 return NextResponse.json({ webhook: savedWebhook }, { status }) } catch (error: any) { diff --git a/apps/sim/app/api/workflows/[id]/deploy/route.ts b/apps/sim/app/api/workflows/[id]/deploy/route.ts index 09fc458f9b..1ba7647955 100644 --- a/apps/sim/app/api/workflows/[id]/deploy/route.ts +++ b/apps/sim/app/api/workflows/[id]/deploy/route.ts @@ -217,10 +217,8 @@ export async function DELETE( logger.info(`[${requestId}] Workflow undeployed successfully: ${id}`) try { - const { trackPlatformEvent } = await import('@/lib/core/telemetry') - trackPlatformEvent('platform.workflow.undeployed', { - 'workflow.id': id, - }) + const { PlatformEvents } = await import('@/lib/core/telemetry') + PlatformEvents.workflowUndeployed({ workflowId: id }) } catch (_e) { // Silently fail } diff --git a/apps/sim/app/api/workflows/[id]/duplicate/route.ts b/apps/sim/app/api/workflows/[id]/duplicate/route.ts index 41ce249d0c..935dc4ed0f 100644 --- a/apps/sim/app/api/workflows/[id]/duplicate/route.ts +++ b/apps/sim/app/api/workflows/[id]/duplicate/route.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { getSession } from '@/lib/auth' +import { PlatformEvents } from '@/lib/core/telemetry' import { generateRequestId } from '@/lib/core/utils/request' import { duplicateWorkflow } from '@/lib/workflows/persistence/duplicate' @@ -46,6 +47,16 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id: requestId, }) + try { + PlatformEvents.workflowDuplicated({ + sourceWorkflowId, + newWorkflowId: result.id, + workspaceId, + }) + } catch { + // Telemetry should not fail the operation + } + const elapsed = Date.now() - startTime logger.info( `[${requestId}] Successfully duplicated workflow ${sourceWorkflowId} to ${result.id} in ${elapsed}ms` diff --git a/apps/sim/app/api/workflows/[id]/route.ts b/apps/sim/app/api/workflows/[id]/route.ts index 92a19d41c7..968593e94f 100644 --- a/apps/sim/app/api/workflows/[id]/route.ts +++ b/apps/sim/app/api/workflows/[id]/route.ts @@ -8,6 +8,7 @@ import { authenticateApiKeyFromHeader, updateApiKeyLastUsed } from '@/lib/api-ke import { getSession } from '@/lib/auth' import { verifyInternalToken } from '@/lib/auth/internal' import { env } from '@/lib/core/config/env' +import { PlatformEvents } from '@/lib/core/telemetry' import { generateRequestId } from '@/lib/core/utils/request' import { loadWorkflowFromNormalizedTables } from '@/lib/workflows/persistence/utils' import { getWorkflowAccessContext, getWorkflowById } from '@/lib/workflows/utils' @@ -335,6 +336,15 @@ export async function DELETE( await db.delete(workflow).where(eq(workflow.id, workflowId)) + try { + PlatformEvents.workflowDeleted({ + workflowId, + workspaceId: workflowData.workspaceId || undefined, + }) + } catch { + // Telemetry should not fail the operation + } + const elapsed = Date.now() - startTime logger.info(`[${requestId}] Successfully deleted workflow ${workflowId} in ${elapsed}ms`) diff --git a/apps/sim/app/api/workflows/route.ts b/apps/sim/app/api/workflows/route.ts index 4ff9d99acc..7c905ab7e6 100644 --- a/apps/sim/app/api/workflows/route.ts +++ b/apps/sim/app/api/workflows/route.ts @@ -119,12 +119,12 @@ export async function POST(req: NextRequest) { logger.info(`[${requestId}] Creating workflow ${workflowId} for user ${session.user.id}`) import('@/lib/core/telemetry') - .then(({ trackPlatformEvent }) => { - trackPlatformEvent('platform.workflow.created', { - 'workflow.id': workflowId, - 'workflow.name': name, - 'workflow.has_workspace': !!workspaceId, - 'workflow.has_folder': !!folderId, + .then(({ PlatformEvents }) => { + PlatformEvents.workflowCreated({ + workflowId, + name, + workspaceId: workspaceId || undefined, + folderId: folderId || undefined, }) }) .catch(() => { diff --git a/apps/sim/app/api/workspaces/[id]/api-keys/route.ts b/apps/sim/app/api/workspaces/[id]/api-keys/route.ts index 0944b15fe2..1232272366 100644 --- a/apps/sim/app/api/workspaces/[id]/api-keys/route.ts +++ b/apps/sim/app/api/workspaces/[id]/api-keys/route.ts @@ -7,6 +7,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { z } from 'zod' import { createApiKey, getApiKeyDisplayFormat } from '@/lib/api-key/auth' import { getSession } from '@/lib/auth' +import { PlatformEvents } from '@/lib/core/telemetry' import { generateRequestId } from '@/lib/core/utils/request' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' @@ -147,6 +148,15 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{ createdAt: apiKey.createdAt, }) + try { + PlatformEvents.apiKeyGenerated({ + userId: userId, + keyName: name, + }) + } catch { + // Telemetry should not fail the operation + } + logger.info(`[${requestId}] Created workspace API key: ${name} in workspace ${workspaceId}`) return NextResponse.json({ @@ -198,6 +208,17 @@ export async function DELETE( ) ) + try { + for (const keyId of keys) { + PlatformEvents.apiKeyRevoked({ + userId: userId, + keyId: keyId, + }) + } + } catch { + // Telemetry should not fail the operation + } + logger.info( `[${requestId}] Deleted ${deletedCount} workspace API keys from workspace ${workspaceId}` ) diff --git a/apps/sim/app/api/workspaces/invitations/route.ts b/apps/sim/app/api/workspaces/invitations/route.ts index 19ee610878..06ad14d34d 100644 --- a/apps/sim/app/api/workspaces/invitations/route.ts +++ b/apps/sim/app/api/workspaces/invitations/route.ts @@ -14,6 +14,7 @@ import { and, eq, inArray } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { WorkspaceInvitationEmail } from '@/components/emails' import { getSession } from '@/lib/auth' +import { PlatformEvents } from '@/lib/core/telemetry' import { getBaseUrl } from '@/lib/core/utils/urls' import { sendEmail } from '@/lib/messaging/email/mailer' import { getFromEmailAddress } from '@/lib/messaging/email/utils' @@ -81,7 +82,6 @@ export async function POST(req: NextRequest) { return NextResponse.json({ error: 'Workspace ID and email are required' }, { status: 400 }) } - // Validate permission type const validPermissions: PermissionType[] = ['admin', 'write', 'read'] if (!validPermissions.includes(permission)) { return NextResponse.json( @@ -90,7 +90,6 @@ export async function POST(req: NextRequest) { ) } - // Check if user has admin permissions for this workspace const userPermission = await db .select() .from(permissions) @@ -111,7 +110,6 @@ export async function POST(req: NextRequest) { ) } - // Get the workspace details for the email const workspaceDetails = await db .select() .from(workspace) @@ -122,8 +120,6 @@ export async function POST(req: NextRequest) { return NextResponse.json({ error: 'Workspace not found' }, { status: 404 }) } - // Check if the user is already a member - // First find if a user with this email exists const existingUser = await db .select() .from(user) @@ -131,7 +127,6 @@ export async function POST(req: NextRequest) { .then((rows) => rows[0]) if (existingUser) { - // Check if the user already has permissions for this workspace const existingPermission = await db .select() .from(permissions) @@ -155,7 +150,6 @@ export async function POST(req: NextRequest) { } } - // Check if there's already a pending invitation const existingInvitation = await db .select() .from(workspaceInvitation) @@ -178,12 +172,10 @@ export async function POST(req: NextRequest) { ) } - // Generate a unique token and set expiry date (1 week from now) const token = randomUUID() const expiresAt = new Date() expiresAt.setDate(expiresAt.getDate() + 7) // 7 days expiry - // Create the invitation const invitationData = { id: randomUUID(), workspaceId, @@ -198,10 +190,19 @@ export async function POST(req: NextRequest) { updatedAt: new Date(), } - // Create invitation await db.insert(workspaceInvitation).values(invitationData) - // Send the invitation email + try { + PlatformEvents.workspaceMemberInvited({ + workspaceId, + invitedBy: session.user.id, + inviteeEmail: email, + role: permission, + }) + } catch { + // Telemetry should not fail the operation + } + await sendInvitationEmail({ to: email, inviterName: session.user.name || session.user.email || 'A user', @@ -217,7 +218,6 @@ export async function POST(req: NextRequest) { } } -// Helper function to send invitation email using the Resend API async function sendInvitationEmail({ to, inviterName, @@ -233,7 +233,6 @@ async function sendInvitationEmail({ }) { try { const baseUrl = getBaseUrl() - // Use invitation ID in path, token in query parameter for security const invitationLink = `${baseUrl}/invite/${invitationId}?token=${token}` const emailHtml = await render( @@ -263,6 +262,5 @@ async function sendInvitationEmail({ } } catch (error) { logger.error('Error sending invitation email:', error) - // Continue even if email fails - the invitation is still created } } diff --git a/apps/sim/app/api/workspaces/route.ts b/apps/sim/app/api/workspaces/route.ts index 6b8c36ba31..f9172d9c30 100644 --- a/apps/sim/app/api/workspaces/route.ts +++ b/apps/sim/app/api/workspaces/route.ts @@ -5,6 +5,7 @@ import { and, desc, eq, isNull } from 'drizzle-orm' import { NextResponse } from 'next/server' import { z } from 'zod' import { getSession } from '@/lib/auth' +import { PlatformEvents } from '@/lib/core/telemetry' import { buildDefaultWorkflowArtifacts } from '@/lib/workflows/defaults' import { saveWorkflowToNormalizedTables } from '@/lib/workflows/persistence/utils' @@ -22,7 +23,6 @@ export async function GET() { return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) } - // Get all workspaces where the user has permissions const userWorkspaces = await db .select({ workspace: workspace, @@ -34,19 +34,15 @@ export async function GET() { .orderBy(desc(workspace.createdAt)) if (userWorkspaces.length === 0) { - // Create a default workspace for the user const defaultWorkspace = await createDefaultWorkspace(session.user.id, session.user.name) - // Migrate existing workflows to the default workspace await migrateExistingWorkflows(session.user.id, defaultWorkspace.id) return NextResponse.json({ workspaces: [defaultWorkspace] }) } - // If user has workspaces but might have orphaned workflows, migrate them await ensureWorkflowsHaveWorkspace(session.user.id, userWorkspaces[0].workspace.id) - // Format the response with permission information const workspacesWithPermissions = userWorkspaces.map( ({ workspace: workspaceDetails, permissionType }) => ({ ...workspaceDetails, @@ -78,24 +74,19 @@ export async function POST(req: Request) { } } -// Helper function to create a default workspace async function createDefaultWorkspace(userId: string, userName?: string | null) { - // Extract first name only by splitting on spaces and taking the first part const firstName = userName?.split(' ')[0] || null const workspaceName = firstName ? `${firstName}'s Workspace` : 'My Workspace' return createWorkspace(userId, workspaceName) } -// Helper function to create a workspace async function createWorkspace(userId: string, name: string) { const workspaceId = crypto.randomUUID() const workflowId = crypto.randomUUID() const now = new Date() - // Create the workspace and initial workflow in a transaction try { await db.transaction(async (tx) => { - // Create the workspace await tx.insert(workspace).values({ id: workspaceId, name, @@ -135,8 +126,6 @@ async function createWorkspace(userId: string, name: string) { variables: {}, }) - // No blocks are inserted - empty canvas - logger.info( `Created workspace ${workspaceId} with initial workflow ${workflowId} for user ${userId}` ) @@ -153,7 +142,16 @@ async function createWorkspace(userId: string, name: string) { throw error } - // Return the workspace data directly instead of querying again + try { + PlatformEvents.workspaceCreated({ + workspaceId, + userId, + name, + }) + } catch { + // Telemetry should not fail the operation + } + return { id: workspaceId, name, @@ -166,9 +164,7 @@ async function createWorkspace(userId: string, name: string) { } } -// Helper function to migrate existing workflows to a workspace async function migrateExistingWorkflows(userId: string, workspaceId: string) { - // Find all workflows that have no workspace ID const orphanedWorkflows = await db .select({ id: workflow.id }) .from(workflow) @@ -182,7 +178,6 @@ async function migrateExistingWorkflows(userId: string, workspaceId: string) { `Migrating ${orphanedWorkflows.length} workflows to workspace ${workspaceId} for user ${userId}` ) - // Bulk update all orphaned workflows at once await db .update(workflow) .set({ @@ -192,16 +187,13 @@ async function migrateExistingWorkflows(userId: string, workspaceId: string) { .where(and(eq(workflow.userId, userId), isNull(workflow.workspaceId))) } -// Helper function to ensure all workflows have a workspace async function ensureWorkflowsHaveWorkspace(userId: string, defaultWorkspaceId: string) { - // First check if there are any orphaned workflows const orphanedWorkflows = await db .select() .from(workflow) .where(and(eq(workflow.userId, userId), isNull(workflow.workspaceId))) if (orphanedWorkflows.length > 0) { - // Directly update any workflows that don't have a workspace ID in a single query await db .update(workflow) .set({ diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.ts index e19068fac3..c072628253 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.ts @@ -31,7 +31,6 @@ export async function executeWorkflowWithFullLogging( const { setActiveBlocks, setBlockRunStatus, setEdgeRunStatus } = useExecutionStore.getState() const workflowEdges = useWorkflowStore.getState().edges - // Track active blocks for pulsing animation const activeBlocksSet = new Set() const payload: any = { @@ -59,7 +58,6 @@ export async function executeWorkflowWithFullLogging( throw new Error('No response body') } - // Parse SSE stream const reader = response.body.getReader() const decoder = new TextDecoder() let buffer = '' @@ -89,11 +87,9 @@ export async function executeWorkflowWithFullLogging( switch (event.type) { case 'block:started': { - // Add block to active set for pulsing animation activeBlocksSet.add(event.data.blockId) setActiveBlocks(new Set(activeBlocksSet)) - // Track edges that led to this block as soon as execution starts const incomingEdges = workflowEdges.filter( (edge) => edge.target === event.data.blockId ) @@ -104,11 +100,9 @@ export async function executeWorkflowWithFullLogging( } case 'block:completed': - // Remove block from active set activeBlocksSet.delete(event.data.blockId) setActiveBlocks(new Set(activeBlocksSet)) - // Track successful block execution in run path setBlockRunStatus(event.data.blockId, 'success') addConsole({ @@ -134,11 +128,9 @@ export async function executeWorkflowWithFullLogging( break case 'block:error': - // Remove block from active set activeBlocksSet.delete(event.data.blockId) setActiveBlocks(new Set(activeBlocksSet)) - // Track failed block execution in run path setBlockRunStatus(event.data.blockId, 'error') addConsole({ @@ -183,7 +175,6 @@ export async function executeWorkflowWithFullLogging( } } finally { reader.releaseLock() - // Clear active blocks when execution ends setActiveBlocks(new Set()) } diff --git a/apps/sim/instrumentation-node.ts b/apps/sim/instrumentation-node.ts index 5ac6d02f61..c1d1f4bad8 100644 --- a/apps/sim/instrumentation-node.ts +++ b/apps/sim/instrumentation-node.ts @@ -2,7 +2,9 @@ * Sim OpenTelemetry - Server-side Instrumentation */ +import type { Attributes, Context, Link, SpanKind } from '@opentelemetry/api' import { DiagConsoleLogger, DiagLogLevel, diag } from '@opentelemetry/api' +import type { Sampler, SamplingResult } from '@opentelemetry/sdk-trace-base' import { createLogger } from '@sim/logger' import { env } from './lib/core/config/env' @@ -24,8 +26,25 @@ const DEFAULT_TELEMETRY_CONFIG = { } /** - * Initialize OpenTelemetry SDK with proper configuration + * Span name prefixes we want to KEEP */ +const ALLOWED_SPAN_PREFIXES = [ + 'platform.', // Our platform events + 'gen_ai.', // GenAI semantic convention spans + 'workflow.', // Workflow execution spans + 'block.', // Block execution spans + 'http.client.', // Our API block HTTP calls + 'function.', // Function block execution + 'router.', // Router block evaluation + 'condition.', // Condition block evaluation + 'loop.', // Loop block execution + 'parallel.', // Parallel block execution +] + +function isBusinessSpan(spanName: string): boolean { + return ALLOWED_SPAN_PREFIXES.some((prefix) => spanName.startsWith(prefix)) +} + async function initializeOpenTelemetry() { try { if (env.NEXT_TELEMETRY_DISABLED === '1') { @@ -52,18 +71,43 @@ async function initializeOpenTelemetry() { ) const { OTLPTraceExporter } = await import('@opentelemetry/exporter-trace-otlp-http') const { BatchSpanProcessor } = await import('@opentelemetry/sdk-trace-node') - const { ParentBasedSampler, TraceIdRatioBasedSampler } = await import( + const { ParentBasedSampler, TraceIdRatioBasedSampler, SamplingDecision } = await import( '@opentelemetry/sdk-trace-base' ) + const createBusinessSpanSampler = (baseSampler: Sampler): Sampler => ({ + shouldSample( + context: Context, + traceId: string, + spanName: string, + spanKind: SpanKind, + attributes: Attributes, + links: Link[] + ): SamplingResult { + if (attributes['next.span_type']) { + return { decision: SamplingDecision.NOT_RECORD } + } + + if (isBusinessSpan(spanName)) { + return baseSampler.shouldSample(context, traceId, spanName, spanKind, attributes, links) + } + + return { decision: SamplingDecision.NOT_RECORD } + }, + + toString(): string { + return `BusinessSpanSampler{baseSampler=${baseSampler.toString()}}` + }, + }) + const exporter = new OTLPTraceExporter({ url: telemetryConfig.endpoint, headers: {}, - timeoutMillis: Math.min(telemetryConfig.batchSettings.exportTimeoutMillis, 10000), // Max 10s + timeoutMillis: Math.min(telemetryConfig.batchSettings.exportTimeoutMillis, 10000), keepAlive: false, }) - const spanProcessor = new BatchSpanProcessor(exporter, { + const batchProcessor = new BatchSpanProcessor(exporter, { maxQueueSize: telemetryConfig.batchSettings.maxQueueSize, maxExportBatchSize: telemetryConfig.batchSettings.maxExportBatchSize, scheduledDelayMillis: telemetryConfig.batchSettings.scheduledDelayMillis, @@ -82,13 +126,14 @@ async function initializeOpenTelemetry() { }) ) - const sampler = new ParentBasedSampler({ - root: new TraceIdRatioBasedSampler(0.1), // 10% sampling for root spans + const baseSampler = new ParentBasedSampler({ + root: new TraceIdRatioBasedSampler(0.1), }) + const sampler = createBusinessSpanSampler(baseSampler) const sdk = new NodeSDK({ resource, - spanProcessor, + spanProcessor: batchProcessor, sampler, traceExporter: exporter, }) @@ -107,7 +152,7 @@ async function initializeOpenTelemetry() { process.on('SIGTERM', shutdownHandler) process.on('SIGINT', shutdownHandler) - logger.info('OpenTelemetry instrumentation initialized') + logger.info('OpenTelemetry instrumentation initialized with business span filtering') } catch (error) { logger.error('Failed to initialize OpenTelemetry instrumentation', error) } diff --git a/apps/sim/lib/auth/auth.ts b/apps/sim/lib/auth/auth.ts index a16dee4451..43e4c919ad 100644 --- a/apps/sim/lib/auth/auth.ts +++ b/apps/sim/lib/auth/auth.ts @@ -52,6 +52,7 @@ import { isHosted, isRegistrationDisabled, } from '@/lib/core/config/feature-flags' +import { PlatformEvents } from '@/lib/core/telemetry' import { getBaseUrl } from '@/lib/core/utils/urls' import { sendEmail } from '@/lib/messaging/email/mailer' import { getFromEmailAddress, getPersonalEmailFrom } from '@/lib/messaging/email/utils' @@ -98,6 +99,15 @@ export const auth = betterAuth({ userId: user.id, }) + try { + PlatformEvents.userSignedUp({ + userId: user.id, + authMethod: 'email', + }) + } catch { + // Telemetry should not fail the operation + } + try { await handleNewUser(user.id) } catch (error) { @@ -320,6 +330,15 @@ export const auth = betterAuth({ } } } + + try { + PlatformEvents.oauthConnected({ + userId: account.userId, + provider: account.providerId, + }) + } catch { + // Telemetry should not fail the operation + } }, }, }, diff --git a/apps/sim/lib/core/telemetry.ts b/apps/sim/lib/core/telemetry.ts index dc2c60e3a2..c12fe1303a 100644 --- a/apps/sim/lib/core/telemetry.ts +++ b/apps/sim/lib/core/telemetry.ts @@ -449,3 +449,505 @@ export function trackPlatformEvent( // Silently fail } } + +// ============================================================================ +// PLATFORM TELEMETRY EVENTS +// ============================================================================ +// +// Naming Convention: +// Event: platform.{resource}.{past_tense_action} +// Attribute: {resource}.{attribute_name} +// +// Examples: +// Event: platform.user.signed_up +// Attribute: user.id, user.auth_method, workspace.id +// +// Categories: +// - User/Auth: platform.user.* +// - Workspace: platform.workspace.* +// - Workflow: platform.workflow.* +// - Knowledge Base: platform.knowledge_base.* +// - MCP: platform.mcp.* +// - API Keys: platform.api_key.* +// - OAuth: platform.oauth.* +// - Webhook: platform.webhook.* +// - Billing: platform.billing.* +// - Template: platform.template.* +// ============================================================================ + +/** + * Platform Events - Typed event tracking helpers + * These provide type-safe, consistent telemetry across the platform + */ +export const PlatformEvents = { + /** + * Track user sign up + */ + userSignedUp: (attrs: { + userId: string + authMethod: 'email' | 'oauth' | 'sso' + provider?: string + }) => { + trackPlatformEvent('platform.user.signed_up', { + 'user.id': attrs.userId, + 'user.auth_method': attrs.authMethod, + ...(attrs.provider && { 'user.auth_provider': attrs.provider }), + }) + }, + + /** + * Track user sign in + */ + userSignedIn: (attrs: { + userId: string + authMethod: 'email' | 'oauth' | 'sso' + provider?: string + }) => { + trackPlatformEvent('platform.user.signed_in', { + 'user.id': attrs.userId, + 'user.auth_method': attrs.authMethod, + ...(attrs.provider && { 'user.auth_provider': attrs.provider }), + }) + }, + + /** + * Track password reset requested + */ + passwordResetRequested: (attrs: { userId: string }) => { + trackPlatformEvent('platform.user.password_reset_requested', { + 'user.id': attrs.userId, + }) + }, + + /** + * Track workspace created + */ + workspaceCreated: (attrs: { workspaceId: string; userId: string; name: string }) => { + trackPlatformEvent('platform.workspace.created', { + 'workspace.id': attrs.workspaceId, + 'workspace.name': attrs.name, + 'user.id': attrs.userId, + }) + }, + + /** + * Track member invited to workspace + */ + workspaceMemberInvited: (attrs: { + workspaceId: string + invitedBy: string + inviteeEmail: string + role: string + }) => { + trackPlatformEvent('platform.workspace.member_invited', { + 'workspace.id': attrs.workspaceId, + 'user.id': attrs.invitedBy, + 'invitation.role': attrs.role, + }) + }, + + /** + * Track member joined workspace + */ + workspaceMemberJoined: (attrs: { workspaceId: string; userId: string; role: string }) => { + trackPlatformEvent('platform.workspace.member_joined', { + 'workspace.id': attrs.workspaceId, + 'user.id': attrs.userId, + 'member.role': attrs.role, + }) + }, + + /** + * Track workflow created + */ + workflowCreated: (attrs: { + workflowId: string + name: string + workspaceId?: string + folderId?: string + }) => { + trackPlatformEvent('platform.workflow.created', { + 'workflow.id': attrs.workflowId, + 'workflow.name': attrs.name, + 'workflow.has_workspace': !!attrs.workspaceId, + 'workflow.has_folder': !!attrs.folderId, + }) + }, + + /** + * Track workflow deleted + */ + workflowDeleted: (attrs: { workflowId: string; workspaceId?: string }) => { + trackPlatformEvent('platform.workflow.deleted', { + 'workflow.id': attrs.workflowId, + ...(attrs.workspaceId && { 'workspace.id': attrs.workspaceId }), + }) + }, + + /** + * Track workflow duplicated + */ + workflowDuplicated: (attrs: { + sourceWorkflowId: string + newWorkflowId: string + workspaceId?: string + }) => { + trackPlatformEvent('platform.workflow.duplicated', { + 'workflow.source_id': attrs.sourceWorkflowId, + 'workflow.new_id': attrs.newWorkflowId, + ...(attrs.workspaceId && { 'workspace.id': attrs.workspaceId }), + }) + }, + + /** + * Track workflow deployed + */ + workflowDeployed: (attrs: { + workflowId: string + workflowName: string + blocksCount: number + edgesCount: number + version: number + loopsCount?: number + parallelsCount?: number + blockTypes?: string + }) => { + trackPlatformEvent('platform.workflow.deployed', { + 'workflow.id': attrs.workflowId, + 'workflow.name': attrs.workflowName, + 'workflow.blocks_count': attrs.blocksCount, + 'workflow.edges_count': attrs.edgesCount, + 'deployment.version': attrs.version, + ...(attrs.loopsCount !== undefined && { 'workflow.loops_count': attrs.loopsCount }), + ...(attrs.parallelsCount !== undefined && { + 'workflow.parallels_count': attrs.parallelsCount, + }), + ...(attrs.blockTypes && { 'workflow.block_types': attrs.blockTypes }), + }) + }, + + /** + * Track workflow undeployed + */ + workflowUndeployed: (attrs: { workflowId: string }) => { + trackPlatformEvent('platform.workflow.undeployed', { + 'workflow.id': attrs.workflowId, + }) + }, + + /** + * Track workflow executed + */ + workflowExecuted: (attrs: { + workflowId: string + durationMs: number + status: 'success' | 'error' | 'cancelled' | 'paused' + trigger: string + blocksExecuted: number + hasErrors: boolean + totalCost?: number + errorMessage?: string + }) => { + trackPlatformEvent('platform.workflow.executed', { + 'workflow.id': attrs.workflowId, + 'execution.duration_ms': attrs.durationMs, + 'execution.status': attrs.status, + 'execution.trigger': attrs.trigger, + 'execution.blocks_executed': attrs.blocksExecuted, + 'execution.has_errors': attrs.hasErrors, + ...(attrs.totalCost !== undefined && { 'execution.total_cost': attrs.totalCost }), + ...(attrs.errorMessage && { 'execution.error_message': attrs.errorMessage }), + }) + }, + + /** + * Track knowledge base created + */ + knowledgeBaseCreated: (attrs: { + knowledgeBaseId: string + name: string + workspaceId?: string + }) => { + trackPlatformEvent('platform.knowledge_base.created', { + 'knowledge_base.id': attrs.knowledgeBaseId, + 'knowledge_base.name': attrs.name, + ...(attrs.workspaceId && { 'workspace.id': attrs.workspaceId }), + }) + }, + + /** + * Track knowledge base deleted + */ + knowledgeBaseDeleted: (attrs: { knowledgeBaseId: string }) => { + trackPlatformEvent('platform.knowledge_base.deleted', { + 'knowledge_base.id': attrs.knowledgeBaseId, + }) + }, + + /** + * Track documents uploaded to knowledge base + */ + knowledgeBaseDocumentsUploaded: (attrs: { + knowledgeBaseId: string + documentsCount: number + uploadType: 'single' | 'bulk' + chunkSize?: number + recipe?: string + mimeType?: string + fileSize?: number + }) => { + trackPlatformEvent('platform.knowledge_base.documents_uploaded', { + 'knowledge_base.id': attrs.knowledgeBaseId, + 'documents.count': attrs.documentsCount, + 'documents.upload_type': attrs.uploadType, + ...(attrs.chunkSize !== undefined && { 'processing.chunk_size': attrs.chunkSize }), + ...(attrs.recipe && { 'processing.recipe': attrs.recipe }), + ...(attrs.mimeType && { 'document.mime_type': attrs.mimeType }), + ...(attrs.fileSize !== undefined && { 'document.file_size': attrs.fileSize }), + }) + }, + + /** + * Track knowledge base searched + */ + knowledgeBaseSearched: (attrs: { + knowledgeBaseId: string + resultsCount: number + workspaceId?: string + }) => { + trackPlatformEvent('platform.knowledge_base.searched', { + 'knowledge_base.id': attrs.knowledgeBaseId, + 'search.results_count': attrs.resultsCount, + ...(attrs.workspaceId && { 'workspace.id': attrs.workspaceId }), + }) + }, + + /** + * Track API key generated + */ + apiKeyGenerated: (attrs: { userId: string; keyName?: string }) => { + trackPlatformEvent('platform.api_key.generated', { + 'user.id': attrs.userId, + ...(attrs.keyName && { 'api_key.name': attrs.keyName }), + }) + }, + + /** + * Track API key revoked + */ + apiKeyRevoked: (attrs: { userId: string; keyId: string }) => { + trackPlatformEvent('platform.api_key.revoked', { + 'user.id': attrs.userId, + 'api_key.id': attrs.keyId, + }) + }, + + /** + * Track OAuth provider connected + */ + oauthConnected: (attrs: { userId: string; provider: string; workspaceId?: string }) => { + trackPlatformEvent('platform.oauth.connected', { + 'user.id': attrs.userId, + 'oauth.provider': attrs.provider, + ...(attrs.workspaceId && { 'workspace.id': attrs.workspaceId }), + }) + }, + + /** + * Track OAuth provider disconnected + */ + oauthDisconnected: (attrs: { userId: string; provider: string }) => { + trackPlatformEvent('platform.oauth.disconnected', { + 'user.id': attrs.userId, + 'oauth.provider': attrs.provider, + }) + }, + + /** + * Track credential set created + */ + credentialSetCreated: (attrs: { credentialSetId: string; userId: string; name: string }) => { + trackPlatformEvent('platform.credential_set.created', { + 'credential_set.id': attrs.credentialSetId, + 'credential_set.name': attrs.name, + 'user.id': attrs.userId, + }) + }, + + /** + * Track webhook created + */ + webhookCreated: (attrs: { + webhookId: string + workflowId: string + provider: string + workspaceId?: string + }) => { + trackPlatformEvent('platform.webhook.created', { + 'webhook.id': attrs.webhookId, + 'workflow.id': attrs.workflowId, + 'webhook.provider': attrs.provider, + ...(attrs.workspaceId && { 'workspace.id': attrs.workspaceId }), + }) + }, + + /** + * Track webhook deleted + */ + webhookDeleted: (attrs: { webhookId: string; workflowId: string }) => { + trackPlatformEvent('platform.webhook.deleted', { + 'webhook.id': attrs.webhookId, + 'workflow.id': attrs.workflowId, + }) + }, + + /** + * Track webhook triggered + */ + webhookTriggered: (attrs: { + webhookId: string + workflowId: string + provider: string + success: boolean + }) => { + trackPlatformEvent('platform.webhook.triggered', { + 'webhook.id': attrs.webhookId, + 'workflow.id': attrs.workflowId, + 'webhook.provider': attrs.provider, + 'webhook.trigger_success': attrs.success, + }) + }, + + /** + * Track MCP server added + */ + mcpServerAdded: (attrs: { + serverId: string + serverName: string + transport: string + workspaceId: string + }) => { + trackPlatformEvent('platform.mcp.server_added', { + 'mcp.server_id': attrs.serverId, + 'mcp.server_name': attrs.serverName, + 'mcp.transport': attrs.transport, + 'workspace.id': attrs.workspaceId, + }) + }, + + /** + * Track MCP tool executed + */ + mcpToolExecuted: (attrs: { + serverId: string + toolName: string + status: 'success' | 'error' + workspaceId: string + }) => { + trackPlatformEvent('platform.mcp.tool_executed', { + 'mcp.server_id': attrs.serverId, + 'mcp.tool_name': attrs.toolName, + 'mcp.execution_status': attrs.status, + 'workspace.id': attrs.workspaceId, + }) + }, + + /** + * Track template used + */ + templateUsed: (attrs: { + templateId: string + templateName: string + newWorkflowId: string + blocksCount: number + workspaceId: string + }) => { + trackPlatformEvent('platform.template.used', { + 'template.id': attrs.templateId, + 'template.name': attrs.templateName, + 'workflow.created_id': attrs.newWorkflowId, + 'workflow.blocks_count': attrs.blocksCount, + 'workspace.id': attrs.workspaceId, + }) + }, + + /** + * Track subscription created + */ + subscriptionCreated: (attrs: { + userId: string + plan: string + interval: 'monthly' | 'yearly' + }) => { + trackPlatformEvent('platform.billing.subscription_created', { + 'user.id': attrs.userId, + 'billing.plan': attrs.plan, + 'billing.interval': attrs.interval, + }) + }, + + /** + * Track subscription changed + */ + subscriptionChanged: (attrs: { + userId: string + previousPlan: string + newPlan: string + changeType: 'upgrade' | 'downgrade' + }) => { + trackPlatformEvent('platform.billing.subscription_changed', { + 'user.id': attrs.userId, + 'billing.previous_plan': attrs.previousPlan, + 'billing.new_plan': attrs.newPlan, + 'billing.change_type': attrs.changeType, + }) + }, + + /** + * Track subscription cancelled + */ + subscriptionCancelled: (attrs: { userId: string; plan: string }) => { + trackPlatformEvent('platform.billing.subscription_cancelled', { + 'user.id': attrs.userId, + 'billing.plan': attrs.plan, + }) + }, + + /** + * Track folder created + */ + folderCreated: (attrs: { folderId: string; name: string; workspaceId: string }) => { + trackPlatformEvent('platform.folder.created', { + 'folder.id': attrs.folderId, + 'folder.name': attrs.name, + 'workspace.id': attrs.workspaceId, + }) + }, + + /** + * Track folder deleted + */ + folderDeleted: (attrs: { folderId: string; workspaceId: string }) => { + trackPlatformEvent('platform.folder.deleted', { + 'folder.id': attrs.folderId, + 'workspace.id': attrs.workspaceId, + }) + }, + + /** + * Track chat deployed (workflow deployed as chat interface) + */ + chatDeployed: (attrs: { + chatId: string + workflowId: string + authType: 'public' | 'password' | 'email' | 'sso' + hasOutputConfigs: boolean + }) => { + trackPlatformEvent('platform.chat.deployed', { + 'chat.id': attrs.chatId, + 'workflow.id': attrs.workflowId, + 'chat.auth_type': attrs.authType, + 'chat.has_output_configs': attrs.hasOutputConfigs, + }) + }, +} diff --git a/apps/sim/lib/logs/execution/logging-session.ts b/apps/sim/lib/logs/execution/logging-session.ts index de63aaf5e7..1761217fec 100644 --- a/apps/sim/lib/logs/execution/logging-session.ts +++ b/apps/sim/lib/logs/execution/logging-session.ts @@ -289,14 +289,12 @@ export class LoggingSession { this.completed = true - // Track workflow execution outcome and create trace spans if (traceSpans && traceSpans.length > 0) { try { - const { trackPlatformEvent, createOTelSpansForWorkflowExecution } = await import( + const { PlatformEvents, createOTelSpansForWorkflowExecution } = await import( '@/lib/core/telemetry' ) - // Determine status from trace spans const hasErrors = traceSpans.some((span: any) => { const checkForErrors = (s: any): boolean => { if (s.status === 'error') return true @@ -308,17 +306,16 @@ export class LoggingSession { return checkForErrors(span) }) - trackPlatformEvent('platform.workflow.executed', { - 'workflow.id': this.workflowId, - 'execution.duration_ms': duration, - 'execution.status': hasErrors ? 'error' : 'success', - 'execution.trigger': this.triggerType, - 'execution.blocks_executed': traceSpans.length, - 'execution.has_errors': hasErrors, - 'execution.total_cost': costSummary.totalCost || 0, + PlatformEvents.workflowExecuted({ + workflowId: this.workflowId, + durationMs: duration, + status: hasErrors ? 'error' : 'success', + trigger: this.triggerType, + blocksExecuted: traceSpans.length, + hasErrors, + totalCost: costSummary.totalCost || 0, }) - // Create OpenTelemetry trace spans for the workflow execution const startTime = new Date(new Date(endTime).getTime() - duration).toISOString() createOTelSpansForWorkflowExecution({ workflowId: this.workflowId, @@ -340,7 +337,6 @@ export class LoggingSession { logger.debug(`[${this.requestId}] Completed logging for execution ${this.executionId}`) } } catch (error) { - // Always log completion failures with full details - these should not be silent logger.error(`Failed to complete logging for execution ${this.executionId}:`, { requestId: this.requestId, workflowId: this.workflowId, @@ -348,7 +344,6 @@ export class LoggingSession { error: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined, }) - // Rethrow so safeComplete can decide what to do throw error } } @@ -420,22 +415,20 @@ export class LoggingSession { this.completed = true - // Track workflow execution error outcome and create trace spans try { - const { trackPlatformEvent, createOTelSpansForWorkflowExecution } = await import( + const { PlatformEvents, createOTelSpansForWorkflowExecution } = await import( '@/lib/core/telemetry' ) - trackPlatformEvent('platform.workflow.executed', { - 'workflow.id': this.workflowId, - 'execution.duration_ms': Math.max(1, durationMs), - 'execution.status': 'error', - 'execution.trigger': this.triggerType, - 'execution.blocks_executed': spans.length, - 'execution.has_errors': true, - 'execution.error_message': message, + PlatformEvents.workflowExecuted({ + workflowId: this.workflowId, + durationMs: Math.max(1, durationMs), + status: 'error', + trigger: this.triggerType, + blocksExecuted: spans.length, + hasErrors: true, + errorMessage: message, }) - // Create OpenTelemetry trace spans for the workflow execution createOTelSpansForWorkflowExecution({ workflowId: this.workflowId, workflowName: this.workflowState?.metadata?.name, @@ -458,7 +451,6 @@ export class LoggingSession { ) } } catch (enhancedError) { - // Always log completion failures with full details logger.error(`Failed to complete error logging for execution ${this.executionId}:`, { requestId: this.requestId, workflowId: this.workflowId, @@ -466,7 +458,6 @@ export class LoggingSession { error: enhancedError instanceof Error ? enhancedError.message : String(enhancedError), stack: enhancedError instanceof Error ? enhancedError.stack : undefined, }) - // Rethrow so safeCompleteWithError can decide what to do throw enhancedError } } @@ -509,16 +500,16 @@ export class LoggingSession { this.completed = true try { - const { trackPlatformEvent, createOTelSpansForWorkflowExecution } = await import( + const { PlatformEvents, createOTelSpansForWorkflowExecution } = await import( '@/lib/core/telemetry' ) - trackPlatformEvent('platform.workflow.executed', { - 'workflow.id': this.workflowId, - 'execution.duration_ms': Math.max(1, durationMs), - 'execution.status': 'cancelled', - 'execution.trigger': this.triggerType, - 'execution.blocks_executed': traceSpans?.length || 0, - 'execution.has_errors': false, + PlatformEvents.workflowExecuted({ + workflowId: this.workflowId, + durationMs: Math.max(1, durationMs), + status: 'cancelled', + trigger: this.triggerType, + blocksExecuted: traceSpans?.length || 0, + hasErrors: false, }) // Create OpenTelemetry trace spans for the workflow execution @@ -590,17 +581,17 @@ export class LoggingSession { }) try { - const { trackPlatformEvent, createOTelSpansForWorkflowExecution } = await import( + const { PlatformEvents, createOTelSpansForWorkflowExecution } = await import( '@/lib/core/telemetry' ) - trackPlatformEvent('platform.workflow.executed', { - 'workflow.id': this.workflowId, - 'execution.duration_ms': Math.max(1, durationMs), - 'execution.status': 'paused', - 'execution.trigger': this.triggerType, - 'execution.blocks_executed': traceSpans?.length || 0, - 'execution.has_errors': false, - 'execution.total_cost': costSummary.totalCost || 0, + PlatformEvents.workflowExecuted({ + workflowId: this.workflowId, + durationMs: Math.max(1, durationMs), + status: 'paused', + trigger: this.triggerType, + blocksExecuted: traceSpans?.length || 0, + hasErrors: false, + totalCost: costSummary.totalCost || 0, }) // Create OpenTelemetry trace spans for the workflow execution diff --git a/apps/sim/lib/workflows/persistence/utils.ts b/apps/sim/lib/workflows/persistence/utils.ts index 34bc5969f6..b115321202 100644 --- a/apps/sim/lib/workflows/persistence/utils.ts +++ b/apps/sim/lib/workflows/persistence/utils.ts @@ -565,10 +565,9 @@ export async function deployWorkflow(params: { logger.info(`Deployed workflow ${workflowId} as v${deployedVersion}`) - // Track deployment telemetry if workflow name is provided if (workflowName) { try { - const { trackPlatformEvent } = await import('@/lib/core/telemetry') + const { PlatformEvents } = await import('@/lib/core/telemetry') const blockTypeCounts: Record = {} for (const block of Object.values(currentState.blocks)) { @@ -576,15 +575,15 @@ export async function deployWorkflow(params: { blockTypeCounts[blockType] = (blockTypeCounts[blockType] || 0) + 1 } - trackPlatformEvent('platform.workflow.deployed', { - 'workflow.id': workflowId, - 'workflow.name': workflowName, - 'workflow.blocks_count': Object.keys(currentState.blocks).length, - 'workflow.edges_count': currentState.edges.length, - 'workflow.loops_count': Object.keys(currentState.loops).length, - 'workflow.parallels_count': Object.keys(currentState.parallels).length, - 'workflow.block_types': JSON.stringify(blockTypeCounts), - 'deployment.version': deployedVersion, + PlatformEvents.workflowDeployed({ + workflowId, + workflowName, + blocksCount: Object.keys(currentState.blocks).length, + edgesCount: currentState.edges.length, + version: deployedVersion, + loopsCount: Object.keys(currentState.loops).length, + parallelsCount: Object.keys(currentState.parallels).length, + blockTypes: JSON.stringify(blockTypeCounts), }) } catch (telemetryError) { logger.warn(`Failed to track deployment telemetry for ${workflowId}`, telemetryError) diff --git a/helm/sim/examples/values-existing-secret.yaml b/helm/sim/examples/values-existing-secret.yaml index e0cf3fb656..4f686256cf 100644 --- a/helm/sim/examples/values-existing-secret.yaml +++ b/helm/sim/examples/values-existing-secret.yaml @@ -1,35 +1,16 @@ -# Example: Using Pre-Existing Kubernetes Secrets -# -# This example shows how to use pre-existing Kubernetes secrets instead of -# having the Helm chart create them. This is useful when: -# -# 1. You manage secrets via GitOps tools (Flux, ArgoCD) with sealed secrets -# 2. You use External Secrets Operator to sync from external stores -# 3. You create secrets manually or via other automation -# 4. You want full control over secret lifecycle -# -# Prerequisites: -# Create your secrets before installing the Helm chart. +# Using pre-existing Kubernetes secrets for Sim +# For GitOps, Sealed Secrets, or manual secret management -# ============================================================================= -# SECRET CONFIGURATION -# ============================================================================= +# Prerequisites: +# Create your secrets before installing (see examples at bottom of file) -# Use existing secret for app credentials app: enabled: true replicaCount: 2 secrets: existingSecret: enabled: true - name: "sim-app-secrets" # Name of your pre-existing secret - # Optional: Customize key names if your secret uses different keys - keys: - BETTER_AUTH_SECRET: "BETTER_AUTH_SECRET" - ENCRYPTION_KEY: "ENCRYPTION_KEY" - INTERNAL_API_SECRET: "INTERNAL_API_SECRET" - CRON_SECRET: "CRON_SECRET" - API_ENCRYPTION_KEY: "API_ENCRYPTION_KEY" + name: "sim-app-secrets" env: NEXT_PUBLIC_APP_URL: "https://sim.example.com" BETTER_AUTH_URL: "https://sim.example.com" @@ -46,7 +27,6 @@ realtime: ALLOWED_ORIGINS: "https://sim.example.com" NODE_ENV: "production" -# Use existing secret for PostgreSQL postgresql: enabled: true auth: @@ -54,73 +34,21 @@ postgresql: database: sim existingSecret: enabled: true - name: "sim-postgresql-secret" # Name of your pre-existing secret - passwordKey: "POSTGRES_PASSWORD" # Key containing the password + name: "sim-postgresql-secret" + passwordKey: "POSTGRES_PASSWORD" + +# --- +# Create secrets before installing: +# --- -# ============================================================================= -# EXAMPLE: CREATING THE REQUIRED SECRETS -# ============================================================================= -# Run these commands before installing the Helm chart: -# -# # Generate secure values -# BETTER_AUTH_SECRET=$(openssl rand -hex 32) -# ENCRYPTION_KEY=$(openssl rand -hex 32) -# INTERNAL_API_SECRET=$(openssl rand -hex 32) -# CRON_SECRET=$(openssl rand -hex 32) -# API_ENCRYPTION_KEY=$(openssl rand -hex 32) -# POSTGRES_PASSWORD=$(openssl rand -base64 16 | tr -d '/+=') -# -# # Create app secrets # kubectl create secret generic sim-app-secrets \ # --namespace sim \ -# --from-literal=BETTER_AUTH_SECRET="$BETTER_AUTH_SECRET" \ -# --from-literal=ENCRYPTION_KEY="$ENCRYPTION_KEY" \ -# --from-literal=INTERNAL_API_SECRET="$INTERNAL_API_SECRET" \ -# --from-literal=CRON_SECRET="$CRON_SECRET" \ -# --from-literal=API_ENCRYPTION_KEY="$API_ENCRYPTION_KEY" -# -# # Create PostgreSQL secret +# --from-literal=BETTER_AUTH_SECRET="$(openssl rand -hex 32)" \ +# --from-literal=ENCRYPTION_KEY="$(openssl rand -hex 32)" \ +# --from-literal=INTERNAL_API_SECRET="$(openssl rand -hex 32)" \ +# --from-literal=CRON_SECRET="$(openssl rand -hex 32)" \ +# --from-literal=API_ENCRYPTION_KEY="$(openssl rand -hex 32)" + # kubectl create secret generic sim-postgresql-secret \ # --namespace sim \ -# --from-literal=POSTGRES_PASSWORD="$POSTGRES_PASSWORD" - -# ============================================================================= -# EXAMPLE: USING SEALED SECRETS (GitOps) -# ============================================================================= -# If using Bitnami Sealed Secrets, create a SealedSecret: -# -# apiVersion: bitnami.com/v1alpha1 -# kind: SealedSecret -# metadata: -# name: sim-app-secrets -# namespace: sim -# spec: -# encryptedData: -# BETTER_AUTH_SECRET: -# ENCRYPTION_KEY: -# INTERNAL_API_SECRET: -# CRON_SECRET: -# API_ENCRYPTION_KEY: -# template: -# metadata: -# name: sim-app-secrets -# namespace: sim -# type: Opaque - -# ============================================================================= -# EXAMPLE: EXTERNAL DATABASE WITH EXISTING SECRET -# ============================================================================= -# If using an external database (e.g., AWS RDS, Azure Database), you can also -# use existing secrets: -# -# externalDatabase: -# enabled: true -# host: "mydb.cluster-xyz.us-east-1.rds.amazonaws.com" -# port: 5432 -# username: postgres -# database: sim -# sslMode: require -# existingSecret: -# enabled: true -# name: "sim-external-db-secret" -# passwordKey: "EXTERNAL_DB_PASSWORD" +# --from-literal=POSTGRES_PASSWORD="$(openssl rand -base64 16 | tr -d '/+=')" diff --git a/helm/sim/examples/values-external-secrets.yaml b/helm/sim/examples/values-external-secrets.yaml index 27aec65043..5aa36bc3ac 100644 --- a/helm/sim/examples/values-external-secrets.yaml +++ b/helm/sim/examples/values-external-secrets.yaml @@ -1,29 +1,17 @@ -# Example: External Secrets Operator Integration -# -# This example shows how to integrate with External Secrets Operator (ESO) -# to automatically sync secrets from external secret management systems. -# +# External Secrets Operator integration for Sim +# Syncs secrets from Azure Key Vault, AWS Secrets Manager, HashiCorp Vault, etc. + # Prerequisites: -# 1. Install External Secrets Operator in your cluster: -# helm repo add external-secrets https://charts.external-secrets.io -# helm install external-secrets external-secrets/external-secrets \ -# -n external-secrets --create-namespace -# -# 2. Create a SecretStore or ClusterSecretStore for your provider. -# See examples below for Azure Key Vault, AWS Secrets Manager, and HashiCorp Vault. -# -# Documentation: https://external-secrets.io/ +# 1. Install ESO: helm install external-secrets external-secrets/external-secrets -n external-secrets --create-namespace +# 2. Create a SecretStore/ClusterSecretStore for your provider (see examples at bottom of file) -# ============================================================================= -# EXTERNAL SECRETS OPERATOR CONFIGURATION -# ============================================================================= externalSecrets: enabled: true - apiVersion: "v1" # Use "v1" for ESO v0.17+ (default), "v1beta1" for older versions - refreshInterval: "1h" # How often to sync secrets + apiVersion: "v1" + refreshInterval: "1h" secretStoreRef: - name: "sim-secret-store" # Name of your SecretStore/ClusterSecretStore - kind: "ClusterSecretStore" # or "SecretStore" for namespace-scoped + name: "sim-secret-store" + kind: "ClusterSecretStore" remoteRefs: app: BETTER_AUTH_SECRET: "sim/app/better-auth-secret" @@ -34,9 +22,6 @@ externalSecrets: postgresql: password: "sim/postgresql/password" -# ============================================================================= -# APPLICATION CONFIGURATION -# ============================================================================= app: enabled: true replicaCount: 2 @@ -45,8 +30,6 @@ app: BETTER_AUTH_URL: "https://sim.example.com" NEXT_PUBLIC_SOCKET_URL: "wss://sim-ws.example.com" NODE_ENV: "production" - # Note: Sensitive values (BETTER_AUTH_SECRET, ENCRYPTION_KEY, etc.) - # are injected from External Secrets and don't need to be set here realtime: enabled: true @@ -60,20 +43,15 @@ realtime: postgresql: enabled: true - # Password is injected from External Secrets auth: username: postgres database: sim -# ============================================================================= -# EXAMPLE SECRETSTORE CONFIGURATIONS -# ============================================================================= -# Below are example SecretStore configurations for popular providers. -# Apply ONE of these to your cluster before installing the Sim chart. +# --- +# SecretStore Examples (apply one of these to your cluster before installing) +# --- -# ----------------------------------------------------------------------------- -# Azure Key Vault (with Workload Identity - Recommended) -# ----------------------------------------------------------------------------- +# Azure Key Vault (Workload Identity): # apiVersion: external-secrets.io/v1beta1 # kind: ClusterSecretStore # metadata: @@ -82,36 +60,12 @@ postgresql: # provider: # azurekv: # authType: WorkloadIdentity -# vaultUrl: "https://your-keyvault-name.vault.azure.net" +# vaultUrl: "https://your-keyvault.vault.azure.net" # serviceAccountRef: # name: external-secrets-sa # namespace: external-secrets -# ----------------------------------------------------------------------------- -# Azure Key Vault (with Service Principal) -# ----------------------------------------------------------------------------- -# apiVersion: external-secrets.io/v1beta1 -# kind: ClusterSecretStore -# metadata: -# name: sim-secret-store -# spec: -# provider: -# azurekv: -# tenantId: "your-tenant-id" -# vaultUrl: "https://your-keyvault-name.vault.azure.net" -# authSecretRef: -# clientId: -# name: azure-sp-credentials -# key: client-id -# namespace: external-secrets -# clientSecret: -# name: azure-sp-credentials -# key: client-secret -# namespace: external-secrets - -# ----------------------------------------------------------------------------- -# AWS Secrets Manager (with IRSA - Recommended) -# ----------------------------------------------------------------------------- +# AWS Secrets Manager (IRSA): # apiVersion: external-secrets.io/v1beta1 # kind: ClusterSecretStore # metadata: @@ -123,32 +77,7 @@ postgresql: # region: us-east-1 # role: arn:aws:iam::123456789012:role/external-secrets-role -# ----------------------------------------------------------------------------- -# AWS Secrets Manager (with static credentials) -# ----------------------------------------------------------------------------- -# apiVersion: external-secrets.io/v1beta1 -# kind: ClusterSecretStore -# metadata: -# name: sim-secret-store -# spec: -# provider: -# aws: -# service: SecretsManager -# region: us-east-1 -# auth: -# secretRef: -# accessKeyIDSecretRef: -# name: aws-credentials -# key: access-key-id -# namespace: external-secrets -# secretAccessKeySecretRef: -# name: aws-credentials -# key: secret-access-key -# namespace: external-secrets - -# ----------------------------------------------------------------------------- -# HashiCorp Vault (with Kubernetes Auth - Recommended) -# ----------------------------------------------------------------------------- +# HashiCorp Vault (Kubernetes Auth): # apiVersion: external-secrets.io/v1beta1 # kind: ClusterSecretStore # metadata: @@ -163,30 +92,3 @@ postgresql: # kubernetes: # mountPath: "kubernetes" # role: "external-secrets" -# serviceAccountRef: -# name: external-secrets-sa -# namespace: external-secrets - -# ----------------------------------------------------------------------------- -# Google Cloud Secret Manager (with Workload Identity - Recommended) -# ----------------------------------------------------------------------------- -# apiVersion: external-secrets.io/v1beta1 -# kind: ClusterSecretStore -# metadata: -# name: sim-secret-store -# spec: -# provider: -# gcpsm: -# projectID: your-gcp-project-id - -# ============================================================================= -# SECRETS TO CREATE IN YOUR EXTERNAL SECRET STORE -# ============================================================================= -# Create the following secrets in your external store with these paths: -# -# sim/app/better-auth-secret - 64 char hex string (openssl rand -hex 32) -# sim/app/encryption-key - 64 char hex string (openssl rand -hex 32) -# sim/app/internal-api-secret - 64 char hex string (openssl rand -hex 32) -# sim/app/cron-secret - 64 char hex string (openssl rand -hex 32) -# sim/app/api-encryption-key - 64 char hex string (openssl rand -hex 32) -# sim/postgresql/password - Alphanumeric string (openssl rand -base64 16 | tr -d '/+=') diff --git a/helm/sim/templates/external-db-secret.yaml b/helm/sim/templates/external-db-secret.yaml index d4576e7c37..4a7e3c5f76 100644 --- a/helm/sim/templates/external-db-secret.yaml +++ b/helm/sim/templates/external-db-secret.yaml @@ -1,7 +1,6 @@ {{- if and .Values.externalDatabase.enabled (include "sim.createExternalDbSecret" .) }} --- # Secret for external database credentials -# Only created when not using existingSecret or External Secrets Operator apiVersion: v1 kind: Secret metadata: diff --git a/helm/sim/templates/external-secret-app.yaml b/helm/sim/templates/external-secret-app.yaml index fa74229c7a..3377901fcc 100644 --- a/helm/sim/templates/external-secret-app.yaml +++ b/helm/sim/templates/external-secret-app.yaml @@ -1,9 +1,5 @@ -{{/* -ExternalSecret for App Secrets - Syncs from external secret managers -Only created when External Secrets Operator integration is enabled -Requires ESO to be installed: https://external-secrets.io/ -*/}} {{- if and .Values.externalSecrets.enabled .Values.app.enabled }} +# ExternalSecret for app credentials (syncs from external secret managers) apiVersion: external-secrets.io/{{ .Values.externalSecrets.apiVersion | default "v1" }} kind: ExternalSecret metadata: diff --git a/helm/sim/templates/external-secret-external-db.yaml b/helm/sim/templates/external-secret-external-db.yaml index 8480d152c7..d1cde291aa 100644 --- a/helm/sim/templates/external-secret-external-db.yaml +++ b/helm/sim/templates/external-secret-external-db.yaml @@ -1,9 +1,5 @@ -{{/* -ExternalSecret for External Database - Syncs password from external secret managers -Only created when External Secrets Operator integration is enabled and external database is used -Requires ESO to be installed: https://external-secrets.io/ -*/}} {{- if and .Values.externalSecrets.enabled .Values.externalDatabase.enabled .Values.externalSecrets.remoteRefs.externalDatabase.password }} +# ExternalSecret for external database password (syncs from external secret managers) apiVersion: external-secrets.io/{{ .Values.externalSecrets.apiVersion | default "v1" }} kind: ExternalSecret metadata: diff --git a/helm/sim/templates/external-secret-postgresql.yaml b/helm/sim/templates/external-secret-postgresql.yaml index 4ae0045b15..d8796077d9 100644 --- a/helm/sim/templates/external-secret-postgresql.yaml +++ b/helm/sim/templates/external-secret-postgresql.yaml @@ -1,9 +1,5 @@ -{{/* -ExternalSecret for PostgreSQL - Syncs password from external secret managers -Only created when External Secrets Operator integration is enabled and internal PostgreSQL is used -Requires ESO to be installed: https://external-secrets.io/ -*/}} {{- if and .Values.externalSecrets.enabled .Values.postgresql.enabled .Values.externalSecrets.remoteRefs.postgresql.password }} +# ExternalSecret for PostgreSQL password (syncs from external secret managers) apiVersion: external-secrets.io/{{ .Values.externalSecrets.apiVersion | default "v1" }} kind: ExternalSecret metadata: diff --git a/helm/sim/templates/secrets-app.yaml b/helm/sim/templates/secrets-app.yaml index b031aa7659..29a9d065f2 100644 --- a/helm/sim/templates/secrets-app.yaml +++ b/helm/sim/templates/secrets-app.yaml @@ -1,8 +1,5 @@ -{{/* -App Secrets - Authentication and encryption keys -Only created when not using existingSecret or External Secrets Operator -*/}} {{- if and .Values.app.enabled (include "sim.createAppSecrets" .) }} +# Secret for app credentials (authentication, encryption keys) apiVersion: v1 kind: Secret metadata: diff --git a/helm/sim/templates/statefulset-postgresql.yaml b/helm/sim/templates/statefulset-postgresql.yaml index 8422250dac..b9f25d9211 100644 --- a/helm/sim/templates/statefulset-postgresql.yaml +++ b/helm/sim/templates/statefulset-postgresql.yaml @@ -68,7 +68,6 @@ data: {{- if (include "sim.createPostgresqlSecret" .) }} --- # Secret for PostgreSQL password -# Only created when not using existingSecret or External Secrets Operator apiVersion: v1 kind: Secret metadata: From 4cabc4be219d5c4e4b7e4ce62862ce155658af7b Mon Sep 17 00:00:00 2001 From: waleed Date: Thu, 8 Jan 2026 10:28:44 -0800 Subject: [PATCH 3/5] comments cleanup --- apps/sim/lib/logs/execution/logging-session.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/sim/lib/logs/execution/logging-session.ts b/apps/sim/lib/logs/execution/logging-session.ts index 1761217fec..fd3ae55bab 100644 --- a/apps/sim/lib/logs/execution/logging-session.ts +++ b/apps/sim/lib/logs/execution/logging-session.ts @@ -512,7 +512,6 @@ export class LoggingSession { hasErrors: false, }) - // Create OpenTelemetry trace spans for the workflow execution if (traceSpans && traceSpans.length > 0) { const startTime = new Date(endTime.getTime() - Math.max(1, durationMs)) createOTelSpansForWorkflowExecution({ @@ -594,7 +593,6 @@ export class LoggingSession { totalCost: costSummary.totalCost || 0, }) - // Create OpenTelemetry trace spans for the workflow execution if (traceSpans && traceSpans.length > 0) { const startTime = new Date(endTime.getTime() - Math.max(1, durationMs)) createOTelSpansForWorkflowExecution({ From 8cbe082b2b6b9af219e57aa4da360d73e33a52c3 Mon Sep 17 00:00:00 2001 From: waleed Date: Thu, 8 Jan 2026 10:37:31 -0800 Subject: [PATCH 4/5] ack PR comment --- apps/sim/lib/core/config/feature-flags.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/sim/lib/core/config/feature-flags.ts b/apps/sim/lib/core/config/feature-flags.ts index 0438f8074e..7b52ba8f31 100644 --- a/apps/sim/lib/core/config/feature-flags.ts +++ b/apps/sim/lib/core/config/feature-flags.ts @@ -1,7 +1,7 @@ /** * Environment utility functions for consistent environment detection across the application */ -import { env, getEnv, isTruthy } from './env' +import { env, getEnv, isFalsy, isTruthy } from './env' /** * Is the application running in production mode @@ -68,7 +68,7 @@ export const isRegistrationDisabled = isTruthy(env.DISABLE_REGISTRATION) /** * Is email/password authentication enabled (defaults to true) */ -export const isEmailPasswordEnabled = env.EMAIL_PASSWORD_SIGNUP_ENABLED !== false +export const isEmailPasswordEnabled = !isFalsy(env.EMAIL_PASSWORD_SIGNUP_ENABLED) /** * Is Trigger.dev enabled for async job processing From 4bfb3c7d779e24fe619bcfaf162c1711ed04656b Mon Sep 17 00:00:00 2001 From: waleed Date: Thu, 8 Jan 2026 10:47:53 -0800 Subject: [PATCH 5/5] refactor to use createEnvMock helper instead of local mocks --- .../app/api/knowledge/search/route.test.ts | 9 ++------ .../app/api/knowledge/search/utils.test.ts | 8 ++----- apps/sim/app/api/knowledge/utils.test.ts | 8 ++----- apps/sim/lib/core/security/csp.test.ts | 21 +++++-------------- apps/sim/lib/core/security/encryption.test.ts | 4 ++++ apps/sim/lib/messaging/email/mailer.test.ts | 9 ++++---- .../lib/messaging/email/unsubscribe.test.ts | 10 ++------- apps/sim/lib/messaging/email/utils.test.ts | 9 ++++---- apps/sim/lib/oauth/oauth.test.ts | 10 ++++----- 9 files changed, 32 insertions(+), 56 deletions(-) diff --git a/apps/sim/app/api/knowledge/search/route.test.ts b/apps/sim/app/api/knowledge/search/route.test.ts index 94f8a0a2b6..04259062e7 100644 --- a/apps/sim/app/api/knowledge/search/route.test.ts +++ b/apps/sim/app/api/knowledge/search/route.test.ts @@ -5,6 +5,7 @@ * * @vitest-environment node */ +import { createEnvMock } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { createMockRequest, @@ -26,13 +27,7 @@ vi.mock('drizzle-orm', () => ({ mockKnowledgeSchemas() -vi.mock('@/lib/core/config/env', () => ({ - env: { - OPENAI_API_KEY: 'test-api-key', - }, - isTruthy: (value: string | boolean | number | undefined) => - typeof value === 'string' ? value === 'true' || value === '1' : Boolean(value), -})) +vi.mock('@/lib/core/config/env', () => createEnvMock({ OPENAI_API_KEY: 'test-api-key' })) vi.mock('@/lib/core/utils/request', () => ({ generateRequestId: vi.fn(() => 'test-request-id'), diff --git a/apps/sim/app/api/knowledge/search/utils.test.ts b/apps/sim/app/api/knowledge/search/utils.test.ts index 53ceeaa0ae..e5ebe22a8e 100644 --- a/apps/sim/app/api/knowledge/search/utils.test.ts +++ b/apps/sim/app/api/knowledge/search/utils.test.ts @@ -4,6 +4,7 @@ * * @vitest-environment node */ +import { createEnvMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('drizzle-orm') @@ -30,12 +31,7 @@ vi.stubGlobal( }) ) -vi.mock('@/lib/core/config/env', () => ({ - env: {}, - getEnv: (key: string) => process.env[key], - isTruthy: (value: string | boolean | number | undefined) => - typeof value === 'string' ? value === 'true' || value === '1' : Boolean(value), -})) +vi.mock('@/lib/core/config/env', () => createEnvMock()) import { generateSearchEmbedding, diff --git a/apps/sim/app/api/knowledge/utils.test.ts b/apps/sim/app/api/knowledge/utils.test.ts index 9ae1372c0f..0e8debe701 100644 --- a/apps/sim/app/api/knowledge/utils.test.ts +++ b/apps/sim/app/api/knowledge/utils.test.ts @@ -6,6 +6,7 @@ * This file contains unit tests for the knowledge base utility functions, * including access checks, document processing, and embedding generation. */ +import { createEnvMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('drizzle-orm', () => ({ @@ -15,12 +16,7 @@ vi.mock('drizzle-orm', () => ({ sql: (strings: TemplateStringsArray, ...expr: any[]) => ({ strings, expr }), })) -vi.mock('@/lib/core/config/env', () => ({ - env: { OPENAI_API_KEY: 'test-key' }, - getEnv: (key: string) => process.env[key], - isTruthy: (value: string | boolean | number | undefined) => - typeof value === 'string' ? value === 'true' || value === '1' : Boolean(value), -})) +vi.mock('@/lib/core/config/env', () => createEnvMock({ OPENAI_API_KEY: 'test-key' })) vi.mock('@/lib/knowledge/documents/utils', () => ({ retryWithExponentialBackoff: (fn: any) => fn(), diff --git a/apps/sim/lib/core/security/csp.test.ts b/apps/sim/lib/core/security/csp.test.ts index bd51015471..e36344eb4d 100644 --- a/apps/sim/lib/core/security/csp.test.ts +++ b/apps/sim/lib/core/security/csp.test.ts @@ -1,7 +1,8 @@ +import { createEnvMock } from '@sim/testing' import { afterEach, describe, expect, it, vi } from 'vitest' -vi.mock('@/lib/core/config/env', () => ({ - env: { +vi.mock('@/lib/core/config/env', () => + createEnvMock({ NEXT_PUBLIC_APP_URL: 'https://example.com', NEXT_PUBLIC_SOCKET_URL: 'https://socket.example.com', OLLAMA_URL: 'http://localhost:11434', @@ -13,20 +14,8 @@ vi.mock('@/lib/core/config/env', () => ({ NEXT_PUBLIC_BRAND_FAVICON_URL: 'https://brand.example.com/favicon.ico', NEXT_PUBLIC_PRIVACY_URL: 'https://legal.example.com/privacy', NEXT_PUBLIC_TERMS_URL: 'https://legal.example.com/terms', - }, - getEnv: vi.fn((key: string) => { - const envMap: Record = { - NEXT_PUBLIC_APP_URL: 'https://example.com', - NEXT_PUBLIC_SOCKET_URL: 'https://socket.example.com', - OLLAMA_URL: 'http://localhost:11434', - NEXT_PUBLIC_BRAND_LOGO_URL: 'https://brand.example.com/logo.png', - NEXT_PUBLIC_BRAND_FAVICON_URL: 'https://brand.example.com/favicon.ico', - NEXT_PUBLIC_PRIVACY_URL: 'https://legal.example.com/privacy', - NEXT_PUBLIC_TERMS_URL: 'https://legal.example.com/terms', - } - return envMap[key] || '' - }), -})) + }) +) vi.mock('@/lib/core/config/feature-flags', () => ({ isDev: false, diff --git a/apps/sim/lib/core/security/encryption.test.ts b/apps/sim/lib/core/security/encryption.test.ts index 0e54d21dec..67540e5cac 100644 --- a/apps/sim/lib/core/security/encryption.test.ts +++ b/apps/sim/lib/core/security/encryption.test.ts @@ -6,6 +6,10 @@ const mockEnv = vi.hoisted(() => ({ vi.mock('@/lib/core/config/env', () => ({ env: mockEnv, + isTruthy: (value: string | boolean | number | undefined) => + typeof value === 'string' ? value.toLowerCase() === 'true' || value === '1' : Boolean(value), + isFalsy: (value: string | boolean | number | undefined) => + typeof value === 'string' ? value.toLowerCase() === 'false' || value === '0' : value === false, })) vi.mock('@sim/logger', () => ({ diff --git a/apps/sim/lib/messaging/email/mailer.test.ts b/apps/sim/lib/messaging/email/mailer.test.ts index a5921eb008..9ea8e5d871 100644 --- a/apps/sim/lib/messaging/email/mailer.test.ts +++ b/apps/sim/lib/messaging/email/mailer.test.ts @@ -1,3 +1,4 @@ +import { createEnvMock } from '@sim/testing' import { beforeEach, describe, expect, it, type Mock, vi } from 'vitest' /** @@ -44,15 +45,15 @@ vi.mock('@/lib/messaging/email/unsubscribe', () => ({ })) // Mock env with valid API keys so the clients get initialized -vi.mock('@/lib/core/config/env', () => ({ - env: { +vi.mock('@/lib/core/config/env', () => + createEnvMock({ RESEND_API_KEY: 'test-api-key', AZURE_ACS_CONNECTION_STRING: 'test-azure-connection-string', AZURE_COMMUNICATION_EMAIL_DOMAIN: 'test.azurecomm.net', NEXT_PUBLIC_APP_URL: 'https://test.sim.ai', FROM_EMAIL_ADDRESS: 'Sim ', - }, -})) + }) +) // Mock URL utilities vi.mock('@/lib/core/utils/urls', () => ({ diff --git a/apps/sim/lib/messaging/email/unsubscribe.test.ts b/apps/sim/lib/messaging/email/unsubscribe.test.ts index b456e79c05..578976da57 100644 --- a/apps/sim/lib/messaging/email/unsubscribe.test.ts +++ b/apps/sim/lib/messaging/email/unsubscribe.test.ts @@ -1,3 +1,4 @@ +import { createEnvMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { EmailType } from '@/lib/messaging/email/mailer' @@ -25,14 +26,7 @@ vi.mock('drizzle-orm', () => ({ eq: vi.fn((a, b) => ({ type: 'eq', left: a, right: b })), })) -vi.mock('@/lib/core/config/env', () => ({ - env: { - BETTER_AUTH_SECRET: 'test-secret-key', - }, - isTruthy: (value: string | boolean | number | undefined) => - typeof value === 'string' ? value === 'true' || value === '1' : Boolean(value), - getEnv: (variable: string) => process.env[variable], -})) +vi.mock('@/lib/core/config/env', () => createEnvMock({ BETTER_AUTH_SECRET: 'test-secret-key' })) vi.mock('@sim/logger', () => ({ createLogger: () => ({ diff --git a/apps/sim/lib/messaging/email/utils.test.ts b/apps/sim/lib/messaging/email/utils.test.ts index c58010be34..2d398a5492 100644 --- a/apps/sim/lib/messaging/email/utils.test.ts +++ b/apps/sim/lib/messaging/email/utils.test.ts @@ -1,3 +1,4 @@ +import { createEnvMock } from '@sim/testing' import { describe, expect, it, vi } from 'vitest' /** @@ -8,12 +9,12 @@ import { describe, expect, it, vi } from 'vitest' */ // Set up mocks at module level - these will be used for all tests in this file -vi.mock('@/lib/core/config/env', () => ({ - env: { +vi.mock('@/lib/core/config/env', () => + createEnvMock({ FROM_EMAIL_ADDRESS: 'Sim ', EMAIL_DOMAIN: 'example.com', - }, -})) + }) +) vi.mock('@/lib/core/utils/urls', () => ({ getEmailDomain: vi.fn().mockReturnValue('fallback.com'), diff --git a/apps/sim/lib/oauth/oauth.test.ts b/apps/sim/lib/oauth/oauth.test.ts index 85a31d5286..5373ccccf2 100644 --- a/apps/sim/lib/oauth/oauth.test.ts +++ b/apps/sim/lib/oauth/oauth.test.ts @@ -1,8 +1,8 @@ -import { createMockFetch, loggerMock } from '@sim/testing' +import { createEnvMock, createMockFetch, loggerMock } from '@sim/testing' import { describe, expect, it, vi } from 'vitest' -vi.mock('@/lib/core/config/env', () => ({ - env: { +vi.mock('@/lib/core/config/env', () => + createEnvMock({ GOOGLE_CLIENT_ID: 'google_client_id', GOOGLE_CLIENT_SECRET: 'google_client_secret', GITHUB_CLIENT_ID: 'github_client_id', @@ -49,8 +49,8 @@ vi.mock('@/lib/core/config/env', () => ({ WORDPRESS_CLIENT_SECRET: 'wordpress_client_secret', SPOTIFY_CLIENT_ID: 'spotify_client_id', SPOTIFY_CLIENT_SECRET: 'spotify_client_secret', - }, -})) + }) +) vi.mock('@sim/logger', () => loggerMock)