From 5f8d769749426a0eb7fbfdb17e829903a8398ce9 Mon Sep 17 00:00:00 2001 From: waleed Date: Fri, 2 Jan 2026 01:03:04 -0800 Subject: [PATCH 1/4] feat(tools): added support for imap trigger --- apps/docs/components/icons.tsx | 69 +- apps/docs/components/ui/icon-mapping.ts | 2 + apps/docs/content/docs/en/tools/imap.mdx | 40 ++ apps/docs/content/docs/en/tools/meta.json | 1 + .../sim/app/api/tools/imap/mailboxes/route.ts | 99 +++ apps/sim/app/api/webhooks/poll/imap/route.ts | 68 ++ apps/sim/blocks/blocks/imap.ts | 56 ++ apps/sim/blocks/registry.ts | 2 + apps/sim/components/icons.tsx | 51 ++ apps/sim/lib/webhooks/imap-polling-service.ts | 615 ++++++++++++++++++ apps/sim/lib/webhooks/utils.server.ts | 61 ++ apps/sim/package.json | 3 +- apps/sim/triggers/imap/index.ts | 1 + apps/sim/triggers/imap/poller.ts | 250 +++++++ apps/sim/triggers/registry.ts | 2 + bun.lock | 21 + helm/sim/values.yaml | 9 + 17 files changed, 1336 insertions(+), 14 deletions(-) create mode 100644 apps/docs/content/docs/en/tools/imap.mdx create mode 100644 apps/sim/app/api/tools/imap/mailboxes/route.ts create mode 100644 apps/sim/app/api/webhooks/poll/imap/route.ts create mode 100644 apps/sim/blocks/blocks/imap.ts create mode 100644 apps/sim/lib/webhooks/imap-polling-service.ts create mode 100644 apps/sim/triggers/imap/index.ts create mode 100644 apps/sim/triggers/imap/poller.ts diff --git a/apps/docs/components/icons.tsx b/apps/docs/components/icons.tsx index 8a449bf6af..751494d6e2 100644 --- a/apps/docs/components/icons.tsx +++ b/apps/docs/components/icons.tsx @@ -295,6 +295,57 @@ export function MailIcon(props: SVGProps) { ) } +export function MailServerIcon(props: SVGProps) { + return ( + + {/* Server/inbox icon with mail symbol */} + + + + + + ) +} + export function CodeIcon(props: SVGProps) { return ( ) { export function SpotifyIcon(props: SVGProps) { return ( - - + + ) } diff --git a/apps/docs/components/ui/icon-mapping.ts b/apps/docs/components/ui/icon-mapping.ts index b2b57527a9..4b4e966bc1 100644 --- a/apps/docs/components/ui/icon-mapping.ts +++ b/apps/docs/components/ui/icon-mapping.ts @@ -58,6 +58,7 @@ import { LinkupIcon, MailchimpIcon, MailgunIcon, + MailServerIcon, Mem0Icon, MicrosoftExcelIcon, MicrosoftOneDriveIcon, @@ -165,6 +166,7 @@ export const blockTypeToIconMap: Record = { huggingface: HuggingFaceIcon, hunter: HunterIOIcon, image_generator: ImageIcon, + imap: MailServerIcon, incidentio: IncidentioIcon, intercom: IntercomIcon, jina: JinaAIIcon, diff --git a/apps/docs/content/docs/en/tools/imap.mdx b/apps/docs/content/docs/en/tools/imap.mdx new file mode 100644 index 0000000000..af53c8200f --- /dev/null +++ b/apps/docs/content/docs/en/tools/imap.mdx @@ -0,0 +1,40 @@ +--- +title: IMAP Email +description: Trigger workflows when new emails arrive via IMAP (works with any email provider) +--- + +import { BlockInfoCard } from "@/components/ui/block-info-card" + + + +{/* MANUAL-CONTENT-START:intro */} +The IMAP Email trigger allows your Sim workflows to start automatically whenever a new email is received in any mailbox that supports the IMAP protocol. This works with Gmail, Outlook, Yahoo, and most other email providers. + +With the IMAP trigger, you can: + +- **Automate email processing**: Start workflows in real time when new messages arrive in your inbox. +- **Filter by sender, subject, or folder**: Configure your trigger to react only to emails that match certain conditions. +- **Extract and process attachments**: Automatically download and use file attachments in your automated flows. +- **Parse and use email content**: Access the subject, sender, recipients, full body, and other metadata in downstream workflow steps. +- **Integrate with any email provider**: Works with any service that provides standard IMAP access, without vendor lock-in. +- **Trigger on unread, flagged, or custom criteria**: Set up advanced filters for the kinds of emails that start your workflows. + +With Sim, the IMAP integration gives you the power to turn email into an actionable source of automation. Respond to customer inquiries, process notifications, kick off data pipelines, and more—directly from your email inbox, with no manual intervention. +{/* MANUAL-CONTENT-END */} + + +## Usage Instructions + +Connect to any email server via IMAP protocol to trigger workflows when new emails are received. Supports Gmail, Outlook, Yahoo, and any other IMAP-compatible email provider. + + + + + +## Notes + +- Category: `triggers` +- Type: `imap` diff --git a/apps/docs/content/docs/en/tools/meta.json b/apps/docs/content/docs/en/tools/meta.json index 7ee6738b7a..aaffe099d3 100644 --- a/apps/docs/content/docs/en/tools/meta.json +++ b/apps/docs/content/docs/en/tools/meta.json @@ -42,6 +42,7 @@ "huggingface", "hunter", "image_generator", + "imap", "incidentio", "intercom", "jina", diff --git a/apps/sim/app/api/tools/imap/mailboxes/route.ts b/apps/sim/app/api/tools/imap/mailboxes/route.ts new file mode 100644 index 0000000000..f807f11bfb --- /dev/null +++ b/apps/sim/app/api/tools/imap/mailboxes/route.ts @@ -0,0 +1,99 @@ +import { createLogger } from '@sim/logger' +import { ImapFlow } from 'imapflow' +import { type NextRequest, NextResponse } from 'next/server' + +const logger = createLogger('ImapMailboxesAPI') + +interface ImapMailboxRequest { + host: string + port: number + secure: boolean + rejectUnauthorized: boolean + username: string + password: string +} + +export async function POST(request: NextRequest) { + try { + const body = (await request.json()) as ImapMailboxRequest + const { host, port, secure, rejectUnauthorized, username, password } = body + + if (!host || !username || !password) { + return NextResponse.json( + { success: false, message: 'Missing required fields: host, username, password' }, + { status: 400 } + ) + } + + const client = new ImapFlow({ + host, + port: port || 993, + secure: secure ?? true, + auth: { + user: username, + pass: password, + }, + tls: { + rejectUnauthorized: rejectUnauthorized ?? true, + }, + logger: false, + }) + + try { + await client.connect() + + // List all mailboxes + const listResult = await client.list() + const mailboxes = listResult.map((mailbox) => ({ + path: mailbox.path, + name: mailbox.name, + delimiter: mailbox.delimiter, + })) + + await client.logout() + + // Sort mailboxes: INBOX first, then alphabetically + mailboxes.sort((a, b) => { + if (a.path === 'INBOX') return -1 + if (b.path === 'INBOX') return 1 + return a.path.localeCompare(b.path) + }) + + return NextResponse.json({ + success: true, + mailboxes, + }) + } catch (error) { + // Make sure to close connection on error + try { + await client.logout() + } catch { + // Ignore logout errors + } + throw error + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error' + logger.error('Error fetching IMAP mailboxes:', errorMessage) + + // Provide user-friendly error messages + let userMessage = 'Failed to connect to IMAP server' + if ( + errorMessage.includes('AUTHENTICATIONFAILED') || + errorMessage.includes('Invalid credentials') + ) { + userMessage = 'Invalid username or password. For Gmail, use an App Password.' + } else if (errorMessage.includes('ENOTFOUND') || errorMessage.includes('getaddrinfo')) { + userMessage = 'Could not find IMAP server. Please check the hostname.' + } else if (errorMessage.includes('ECONNREFUSED')) { + userMessage = 'Connection refused. Please check the port and SSL settings.' + } else if (errorMessage.includes('certificate') || errorMessage.includes('SSL')) { + userMessage = + 'TLS/SSL error. Try disabling "Verify TLS Certificate" for self-signed certificates.' + } else if (errorMessage.includes('timeout')) { + userMessage = 'Connection timed out. Please check your network and server settings.' + } + + return NextResponse.json({ success: false, message: userMessage }, { status: 500 }) + } +} diff --git a/apps/sim/app/api/webhooks/poll/imap/route.ts b/apps/sim/app/api/webhooks/poll/imap/route.ts new file mode 100644 index 0000000000..1759f43c0b --- /dev/null +++ b/apps/sim/app/api/webhooks/poll/imap/route.ts @@ -0,0 +1,68 @@ +import { createLogger } from '@sim/logger' +import { nanoid } from 'nanoid' +import { type NextRequest, NextResponse } from 'next/server' +import { verifyCronAuth } from '@/lib/auth/internal' +import { acquireLock, releaseLock } from '@/lib/core/config/redis' +import { pollImapWebhooks } from '@/lib/webhooks/imap-polling-service' + +const logger = createLogger('ImapPollingAPI') + +export const dynamic = 'force-dynamic' +export const maxDuration = 180 // Allow up to 3 minutes for polling to complete + +const LOCK_KEY = 'imap-polling-lock' +const LOCK_TTL_SECONDS = 180 // Same as maxDuration (3 min) + +export async function GET(request: NextRequest) { + const requestId = nanoid() + logger.info(`IMAP webhook polling triggered (${requestId})`) + + let lockValue: string | undefined + + try { + const authError = verifyCronAuth(request, 'IMAP webhook polling') + if (authError) { + return authError + } + + lockValue = requestId // unique value to identify the holder + const locked = await acquireLock(LOCK_KEY, lockValue, LOCK_TTL_SECONDS) + + if (!locked) { + return NextResponse.json( + { + success: true, + message: 'Polling already in progress – skipped', + requestId, + status: 'skip', + }, + { status: 202 } + ) + } + + const results = await pollImapWebhooks() + + return NextResponse.json({ + success: true, + message: 'IMAP polling completed', + requestId, + status: 'completed', + ...results, + }) + } catch (error) { + logger.error(`Error during IMAP polling (${requestId}):`, error) + return NextResponse.json( + { + success: false, + message: 'IMAP polling failed', + error: error instanceof Error ? error.message : 'Unknown error', + requestId, + }, + { status: 500 } + ) + } finally { + if (lockValue) { + await releaseLock(LOCK_KEY, lockValue).catch(() => {}) + } + } +} diff --git a/apps/sim/blocks/blocks/imap.ts b/apps/sim/blocks/blocks/imap.ts new file mode 100644 index 0000000000..398bdadf78 --- /dev/null +++ b/apps/sim/blocks/blocks/imap.ts @@ -0,0 +1,56 @@ +import { MailServerIcon } from '@/components/icons' +import type { BlockConfig } from '@/blocks/types' +import { getTrigger } from '@/triggers' + +export const ImapBlock: BlockConfig = { + type: 'imap', + name: 'IMAP Email', + description: 'Trigger workflows when new emails arrive via IMAP (works with any email provider)', + longDescription: + 'Connect to any email server via IMAP protocol to trigger workflows when new emails are received. Supports Gmail, Outlook, Yahoo, and any other IMAP-compatible email provider.', + category: 'triggers', + bgColor: '#6366F1', + icon: MailServerIcon, + triggerAllowed: true, + hideFromToolbar: false, + subBlocks: [...getTrigger('imap_poller').subBlocks], + tools: { + access: [], + config: { + tool: () => '', + }, + }, + inputs: { + // Connection settings (stored in providerConfig) + host: { type: 'string', description: 'IMAP server hostname' }, + port: { type: 'string', description: 'IMAP server port' }, + secure: { type: 'boolean', description: 'Use SSL/TLS encryption' }, + rejectUnauthorized: { type: 'boolean', description: 'Verify TLS certificate' }, + username: { type: 'string', description: 'Email username' }, + password: { type: 'string', description: 'Email password' }, + mailbox: { type: 'string', description: 'Mailbox to monitor' }, + searchCriteria: { type: 'string', description: 'IMAP search criteria' }, + markAsRead: { type: 'boolean', description: 'Mark emails as read after processing' }, + includeAttachments: { type: 'boolean', description: 'Include email attachments' }, + }, + outputs: { + // Trigger outputs (from email) + uid: { type: 'string', description: 'IMAP message UID' }, + messageId: { type: 'string', description: 'RFC Message-ID header' }, + subject: { type: 'string', description: 'Email subject line' }, + from: { type: 'string', description: 'Sender email address' }, + to: { type: 'string', description: 'Recipient email address' }, + cc: { type: 'string', description: 'CC recipients' }, + date: { type: 'string', description: 'Email date in ISO format' }, + bodyText: { type: 'string', description: 'Plain text email body' }, + bodyHtml: { type: 'string', description: 'HTML email body' }, + mailbox: { type: 'string', description: 'Mailbox/folder where email was received' }, + hasAttachments: { type: 'boolean', description: 'Whether email has attachments' }, + attachments: { type: 'json', description: 'Array of email attachments' }, + timestamp: { type: 'string', description: 'Event timestamp' }, + }, + triggers: { + enabled: true, + available: ['imap_poller'], + }, +} diff --git a/apps/sim/blocks/registry.ts b/apps/sim/blocks/registry.ts index 16cd7d08e1..2bf7392669 100644 --- a/apps/sim/blocks/registry.ts +++ b/apps/sim/blocks/registry.ts @@ -50,6 +50,7 @@ import { HuggingFaceBlock } from '@/blocks/blocks/huggingface' import { HumanInTheLoopBlock } from '@/blocks/blocks/human_in_the_loop' import { HunterBlock } from '@/blocks/blocks/hunter' import { ImageGeneratorBlock } from '@/blocks/blocks/image_generator' +import { ImapBlock } from '@/blocks/blocks/imap' import { IncidentioBlock } from '@/blocks/blocks/incidentio' import { InputTriggerBlock } from '@/blocks/blocks/input_trigger' import { IntercomBlock } from '@/blocks/blocks/intercom' @@ -195,6 +196,7 @@ export const registry: Record = { huggingface: HuggingFaceBlock, human_in_the_loop: HumanInTheLoopBlock, hunter: HunterBlock, + imap: ImapBlock, image_generator: ImageGeneratorBlock, incidentio: IncidentioBlock, input_trigger: InputTriggerBlock, diff --git a/apps/sim/components/icons.tsx b/apps/sim/components/icons.tsx index e1f1a317a5..751494d6e2 100644 --- a/apps/sim/components/icons.tsx +++ b/apps/sim/components/icons.tsx @@ -295,6 +295,57 @@ export function MailIcon(props: SVGProps) { ) } +export function MailServerIcon(props: SVGProps) { + return ( + + {/* Server/inbox icon with mail symbol */} + + + + + + ) +} + export function CodeIcon(props: SVGProps) { return ( = MAX_CONSECUTIVE_FAILURES + + if (shouldDisable) { + await db + .update(webhook) + .set({ + isActive: false, + updatedAt: new Date(), + }) + .where(eq(webhook.id, webhookId)) + + logger.warn( + `Webhook ${webhookId} auto-disabled after ${MAX_CONSECUTIVE_FAILURES} consecutive failures` + ) + } + } catch (err) { + logger.error(`Failed to mark webhook ${webhookId} as failed:`, err) + } +} + +async function markWebhookSuccess(webhookId: string) { + try { + await db + .update(webhook) + .set({ + failedCount: 0, + updatedAt: new Date(), + }) + .where(eq(webhook.id, webhookId)) + } catch (err) { + logger.error(`Failed to mark webhook ${webhookId} as successful:`, err) + } +} + +export async function pollImapWebhooks() { + logger.info('Starting IMAP webhook polling') + + try { + const activeWebhooksResult = await db + .select({ webhook }) + .from(webhook) + .innerJoin(workflow, eq(webhook.workflowId, workflow.id)) + .where( + and(eq(webhook.provider, 'imap'), eq(webhook.isActive, true), eq(workflow.isDeployed, true)) + ) + + const activeWebhooks = activeWebhooksResult.map((r) => r.webhook) + + if (!activeWebhooks.length) { + logger.info('No active IMAP webhooks found') + return { total: 0, successful: 0, failed: 0, details: [] } + } + + logger.info(`Found ${activeWebhooks.length} active IMAP webhooks`) + + // Limit concurrency to avoid overwhelming IMAP servers + const CONCURRENCY = 5 + + const running: Promise[] = [] + let successCount = 0 + let failureCount = 0 + + const enqueue = async (webhookData: (typeof activeWebhooks)[number]) => { + const webhookId = webhookData.id + const requestId = nanoid() + + try { + const config = webhookData.providerConfig as unknown as ImapWebhookConfig + + if (!config.host || !config.username || !config.password) { + logger.error(`[${requestId}] Missing IMAP credentials for webhook ${webhookId}`) + await markWebhookFailed(webhookId) + failureCount++ + return + } + + const fetchResult = await fetchNewEmails(config, requestId) + const { emails, latestUid } = fetchResult + + if (!emails || !emails.length) { + if (latestUid && latestUid !== config.lastProcessedUid) { + await updateWebhookLastProcessedUid(webhookId, latestUid) + } + await markWebhookSuccess(webhookId) + logger.info(`[${requestId}] No new emails found for webhook ${webhookId}`) + successCount++ + return + } + + logger.info(`[${requestId}] Found ${emails.length} new emails for webhook ${webhookId}`) + + const { processedCount, failedCount: emailFailedCount } = await processEmails( + emails, + webhookData, + config, + requestId + ) + + if (latestUid) { + await updateWebhookLastProcessedUid(webhookId, latestUid) + } + + if (emailFailedCount > 0 && processedCount === 0) { + await markWebhookFailed(webhookId) + failureCount++ + logger.warn( + `[${requestId}] All ${emailFailedCount} emails failed to process for webhook ${webhookId}` + ) + } else { + await markWebhookSuccess(webhookId) + successCount++ + logger.info( + `[${requestId}] Successfully processed ${processedCount} emails for webhook ${webhookId}${emailFailedCount > 0 ? ` (${emailFailedCount} failed)` : ''}` + ) + } + } catch (error) { + logger.error(`[${requestId}] Error processing IMAP webhook ${webhookId}:`, error) + await markWebhookFailed(webhookId) + failureCount++ + } + } + + for (const webhookData of activeWebhooks) { + const promise = enqueue(webhookData) + .then(() => {}) + .catch((err) => { + logger.error('Unexpected error in webhook processing:', err) + failureCount++ + }) + + running.push(promise) + + if (running.length >= CONCURRENCY) { + const completedIdx = await Promise.race(running.map((p, i) => p.then(() => i))) + running.splice(completedIdx, 1) + } + } + + await Promise.allSettled(running) + + const summary = { + total: activeWebhooks.length, + successful: successCount, + failed: failureCount, + details: [], + } + + logger.info('IMAP polling completed', { + total: summary.total, + successful: summary.successful, + failed: summary.failed, + }) + + return summary + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error' + logger.error('Error in IMAP polling service:', errorMessage) + throw error + } +} + +async function fetchNewEmails(config: ImapWebhookConfig, requestId: string) { + const client = new ImapFlow({ + host: config.host, + port: config.port || 993, + secure: config.secure ?? true, + auth: { + user: config.username, + pass: config.password, + }, + tls: { + rejectUnauthorized: config.rejectUnauthorized ?? true, + }, + logger: false, + }) + + const emails: Array<{ + uid: number + envelope: FetchMessageObject['envelope'] + bodyStructure: FetchMessageObject['bodyStructure'] + source?: Buffer + }> = [] + let latestUid = config.lastProcessedUid + + try { + await client.connect() + logger.debug(`[${requestId}] Connected to IMAP server ${config.host}`) + + const mailbox = await client.mailboxOpen(config.mailbox || 'INBOX') + logger.debug(`[${requestId}] Opened mailbox: ${mailbox.path}, exists: ${mailbox.exists}`) + + // Build search criteria + let searchCriteria: any = config.searchCriteria || 'UNSEEN' + + // If we have a last processed UID, add UID filter + if (config.lastProcessedUid) { + // Search for messages with UID greater than last processed + const uidCriteria = { uid: `${config.lastProcessedUid + 1}:*` } + if (typeof searchCriteria === 'string') { + searchCriteria = { [searchCriteria]: true, ...uidCriteria } + } else { + searchCriteria = { ...searchCriteria, ...uidCriteria } + } + } + + // Search for matching messages + const messageUids: number[] = [] + try { + for await (const msg of client.fetch(searchCriteria, { uid: true })) { + messageUids.push(msg.uid) + } + } catch (fetchError) { + // If search fails (e.g., no messages match), return empty + logger.debug(`[${requestId}] Fetch returned no messages or failed: ${fetchError}`) + await client.logout() + return { emails: [], latestUid } + } + + if (messageUids.length === 0) { + logger.debug(`[${requestId}] No messages matching criteria`) + await client.logout() + return { emails: [], latestUid } + } + + // Sort UIDs and take the most recent ones + messageUids.sort((a, b) => b - a) + const maxEmails = config.maxEmailsPerPoll || 25 + const uidsToProcess = messageUids.slice(0, maxEmails) + latestUid = Math.max(...uidsToProcess, config.lastProcessedUid || 0) + + logger.info(`[${requestId}] Processing ${uidsToProcess.length} emails from ${config.mailbox}`) + + // Fetch full message details + for await (const msg of client.fetch(uidsToProcess, { + uid: true, + envelope: true, + bodyStructure: true, + source: true, + })) { + emails.push({ + uid: msg.uid, + envelope: msg.envelope, + bodyStructure: msg.bodyStructure, + source: msg.source, + }) + } + + await client.logout() + logger.debug(`[${requestId}] Disconnected from IMAP server`) + + return { emails, latestUid } + } catch (error) { + try { + await client.logout() + } catch { + // Ignore logout errors + } + throw error + } +} + +function parseEmailAddress( + addr: { name?: string; address?: string } | { name?: string; address?: string }[] | undefined +): string { + if (!addr) return '' + if (Array.isArray(addr)) { + return addr + .map((a) => (a.name ? `${a.name} <${a.address}>` : a.address || '')) + .filter(Boolean) + .join(', ') + } + return addr.name ? `${addr.name} <${addr.address}>` : addr.address || '' +} + +function extractTextFromSource(source: Buffer): { text: string; html: string } { + const content = source.toString('utf-8') + let text = '' + let html = '' + + // Simple extraction - look for Content-Type boundaries + const parts = content.split(/--[^\r\n]+/) + + for (const part of parts) { + const lowerPart = part.toLowerCase() + + if (lowerPart.includes('content-type: text/plain')) { + // Extract text content after headers (double newline) + const match = part.match(/\r?\n\r?\n([\s\S]*?)(?=\r?\n--|\r?\n\.\r?\n|$)/i) + if (match) { + text = match[1].trim() + // Handle quoted-printable decoding + if (lowerPart.includes('quoted-printable')) { + text = text + .replace(/=\r?\n/g, '') + .replace(/=([0-9A-F]{2})/gi, (_, hex) => String.fromCharCode(Number.parseInt(hex, 16))) + } + // Handle base64 decoding + if (lowerPart.includes('base64')) { + try { + text = Buffer.from(text.replace(/\s/g, ''), 'base64').toString('utf-8') + } catch { + // Keep as-is if base64 decode fails + } + } + } + } else if (lowerPart.includes('content-type: text/html')) { + const match = part.match(/\r?\n\r?\n([\s\S]*?)(?=\r?\n--|\r?\n\.\r?\n|$)/i) + if (match) { + html = match[1].trim() + if (lowerPart.includes('quoted-printable')) { + html = html + .replace(/=\r?\n/g, '') + .replace(/=([0-9A-F]{2})/gi, (_, hex) => String.fromCharCode(Number.parseInt(hex, 16))) + } + if (lowerPart.includes('base64')) { + try { + html = Buffer.from(html.replace(/\s/g, ''), 'base64').toString('utf-8') + } catch { + // Keep as-is if base64 decode fails + } + } + } + } + } + + // If no multipart, try to get the body directly + if (!text && !html) { + const bodyMatch = content.match(/\r?\n\r?\n([\s\S]+)$/) + if (bodyMatch) { + text = bodyMatch[1].trim() + } + } + + return { text, html } +} + +function extractAttachmentsFromSource( + source: Buffer, + bodyStructure: FetchMessageObject['bodyStructure'] +): ImapAttachment[] { + const attachments: ImapAttachment[] = [] + + if (!bodyStructure) return attachments + + const content = source.toString('utf-8') + const parts = content.split(/--[^\r\n]+/) + + for (const part of parts) { + const lowerPart = part.toLowerCase() + + // Look for attachment dispositions or non-text content types + const dispositionMatch = part.match( + /content-disposition:\s*attachment[^;]*;\s*filename="?([^"\r\n]+)"?/i + ) + const filenameMatch = part.match(/name="?([^"\r\n]+)"?/i) + const contentTypeMatch = part.match(/content-type:\s*([^;\r\n]+)/i) + + if ( + dispositionMatch || + (filenameMatch && !lowerPart.includes('text/plain') && !lowerPart.includes('text/html')) + ) { + const filename = dispositionMatch?.[1] || filenameMatch?.[1] || 'attachment' + const mimeType = contentTypeMatch?.[1]?.trim() || 'application/octet-stream' + + // Extract the attachment data + const dataMatch = part.match(/\r?\n\r?\n([\s\S]*?)$/i) + if (dataMatch) { + const data = dataMatch[1].trim() + + // Most attachments are base64 encoded + if (lowerPart.includes('base64')) { + try { + const buffer = Buffer.from(data.replace(/\s/g, ''), 'base64') + attachments.push({ + name: filename, + data: buffer, + mimeType, + size: buffer.length, + }) + } catch { + // Skip if decode fails + } + } + } + } + } + + return attachments +} + +async function processEmails( + emails: Array<{ + uid: number + envelope: FetchMessageObject['envelope'] + bodyStructure: FetchMessageObject['bodyStructure'] + source?: Buffer + }>, + webhookData: any, + config: ImapWebhookConfig, + requestId: string +) { + let processedCount = 0 + let failedCount = 0 + + // Create a new client for marking messages + const client = new ImapFlow({ + host: config.host, + port: config.port || 993, + secure: config.secure ?? true, + auth: { + user: config.username, + pass: config.password, + }, + tls: { + rejectUnauthorized: config.rejectUnauthorized ?? true, + }, + logger: false, + }) + + try { + if (config.markAsRead) { + await client.connect() + await client.mailboxOpen(config.mailbox || 'INBOX') + } + + for (const email of emails) { + try { + await pollingIdempotency.executeWithIdempotency( + 'imap', + `${webhookData.id}:${email.uid}`, + async () => { + const envelope = email.envelope + + // Extract body content + const { text: bodyText, html: bodyHtml } = email.source + ? extractTextFromSource(email.source) + : { text: '', html: '' } + + // Extract attachments if enabled + let attachments: ImapAttachment[] = [] + const hasAttachments = email.bodyStructure + ? JSON.stringify(email.bodyStructure).toLowerCase().includes('attachment') + : false + + if (config.includeAttachments && hasAttachments && email.source) { + attachments = extractAttachmentsFromSource(email.source, email.bodyStructure) + } + + const simplifiedEmail: SimplifiedImapEmail = { + uid: String(email.uid), + messageId: envelope?.messageId || '', + subject: envelope?.subject || '[No Subject]', + from: parseEmailAddress(envelope?.from), + to: parseEmailAddress(envelope?.to), + cc: parseEmailAddress(envelope?.cc), + date: envelope?.date ? new Date(envelope.date).toISOString() : null, + bodyText, + bodyHtml, + mailbox: config.mailbox || 'INBOX', + hasAttachments, + attachments, + } + + const payload: ImapWebhookPayload = { + email: simplifiedEmail, + timestamp: new Date().toISOString(), + } + + const webhookUrl = `${getBaseUrl()}/api/webhooks/trigger/${webhookData.path}` + + const response = await fetch(webhookUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Webhook-Secret': webhookData.secret || '', + 'User-Agent': 'Sim/1.0', + }, + body: JSON.stringify(payload), + }) + + if (!response.ok) { + const errorText = await response.text() + logger.error( + `[${requestId}] Failed to trigger webhook for email ${email.uid}:`, + response.status, + errorText + ) + throw new Error(`Webhook request failed: ${response.status} - ${errorText}`) + } + + // Mark as read if configured + if (config.markAsRead) { + try { + await client.messageFlagsAdd({ uid: email.uid }, ['\\Seen']) + } catch (flagError) { + logger.warn( + `[${requestId}] Failed to mark message ${email.uid} as read:`, + flagError + ) + } + } + + return { + emailUid: email.uid, + webhookStatus: response.status, + processed: true, + } + } + ) + + logger.info( + `[${requestId}] Successfully processed email ${email.uid} for webhook ${webhookData.id}` + ) + processedCount++ + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error' + logger.error(`[${requestId}] Error processing email ${email.uid}:`, errorMessage) + failedCount++ + } + } + } finally { + if (config.markAsRead) { + try { + await client.logout() + } catch { + // Ignore logout errors + } + } + } + + return { processedCount, failedCount } +} + +async function updateWebhookLastProcessedUid(webhookId: string, uid: number) { + const result = await db.select().from(webhook).where(eq(webhook.id, webhookId)) + const existingConfig = (result[0]?.providerConfig as Record) || {} + await db + .update(webhook) + .set({ + providerConfig: { + ...existingConfig, + lastProcessedUid: uid, + } as any, + updatedAt: new Date(), + }) + .where(eq(webhook.id, webhookId)) +} diff --git a/apps/sim/lib/webhooks/utils.server.ts b/apps/sim/lib/webhooks/utils.server.ts index 49e69649df..8f538c2690 100644 --- a/apps/sim/lib/webhooks/utils.server.ts +++ b/apps/sim/lib/webhooks/utils.server.ts @@ -869,6 +869,14 @@ export async function formatWebhookInput( return body } + if (foundWebhook.provider === 'imap') { + // IMAP email data is already formatted by the polling service + if (body && typeof body === 'object' && 'email' in body) { + return body + } + return body + } + if (foundWebhook.provider === 'hubspot') { const events = Array.isArray(body) ? body : [body] const event = events[0] @@ -2559,6 +2567,59 @@ export async function configureRssPolling(webhookData: any, requestId: string): } } +/** + * Configure IMAP polling for a webhook + */ +export async function configureImapPolling(webhookData: any, requestId: string): Promise { + const logger = createLogger('ImapWebhookSetup') + logger.info(`[${requestId}] Setting up IMAP polling for webhook ${webhookData.id}`) + + try { + const providerConfig = (webhookData.providerConfig as Record) || {} + const now = new Date() + + // Validate required IMAP connection settings + if (!providerConfig.host || !providerConfig.username || !providerConfig.password) { + logger.error( + `[${requestId}] Missing required IMAP connection settings for webhook ${webhookData.id}` + ) + return false + } + + await db + .update(webhook) + .set({ + providerConfig: { + ...providerConfig, + // Connection defaults + port: providerConfig.port || '993', + secure: providerConfig.secure !== false, + rejectUnauthorized: providerConfig.rejectUnauthorized !== false, + // Mailbox defaults + mailbox: providerConfig.mailbox || 'INBOX', + searchCriteria: providerConfig.searchCriteria || 'UNSEEN', + // Processing options + markAsRead: providerConfig.markAsRead || false, + includeAttachments: providerConfig.includeAttachments !== false, + // Polling state + lastCheckedTimestamp: now.toISOString(), + setupCompleted: true, + }, + updatedAt: now, + }) + .where(eq(webhook.id, webhookData.id)) + + logger.info(`[${requestId}] Successfully configured IMAP polling for webhook ${webhookData.id}`) + return true + } catch (error: any) { + logger.error(`[${requestId}] Failed to configure IMAP polling`, { + webhookId: webhookData.id, + error: error.message, + }) + return false + } +} + export function convertSquareBracketsToTwiML(twiml: string | undefined): string | undefined { if (!twiml) { return twiml diff --git a/apps/sim/package.json b/apps/sim/package.json index 93dcd5d52b..f968fe0f04 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -102,6 +102,7 @@ "groq-sdk": "^0.15.0", "html-to-image": "1.11.13", "html-to-text": "^9.0.5", + "imapflow": "1.2.4", "input-otp": "^1.4.2", "ioredis": "^5.6.0", "isolated-vm": "6.0.2", @@ -126,9 +127,9 @@ "onedollarstats": "0.0.10", "openai": "^4.91.1", "papaparse": "5.5.3", + "postgres": "^3.4.5", "posthog-js": "1.268.9", "posthog-node": "5.9.2", - "postgres": "^3.4.5", "prismjs": "^1.30.0", "react": "19.2.1", "react-colorful": "5.6.1", diff --git a/apps/sim/triggers/imap/index.ts b/apps/sim/triggers/imap/index.ts new file mode 100644 index 0000000000..d88b4d3924 --- /dev/null +++ b/apps/sim/triggers/imap/index.ts @@ -0,0 +1 @@ +export { imapPollingTrigger } from '@/triggers/imap/poller' diff --git a/apps/sim/triggers/imap/poller.ts b/apps/sim/triggers/imap/poller.ts new file mode 100644 index 0000000000..4adf5383da --- /dev/null +++ b/apps/sim/triggers/imap/poller.ts @@ -0,0 +1,250 @@ +import { createLogger } from '@sim/logger' +import { MailServerIcon } from '@/components/icons' +import { useSubBlockStore } from '@/stores/workflows/subblock/store' +import type { TriggerConfig } from '@/triggers/types' + +const logger = createLogger('ImapPollingTrigger') + +export const imapPollingTrigger: TriggerConfig = { + id: 'imap_poller', + name: 'IMAP Email Trigger', + provider: 'imap', + description: 'Triggers when new emails are received via IMAP (works with any email provider)', + version: '1.0.0', + icon: MailServerIcon, + + subBlocks: [ + // Connection settings + { + id: 'host', + title: 'IMAP Server', + type: 'short-input', + placeholder: 'imap.example.com', + description: 'IMAP server hostname (e.g., imap.gmail.com, outlook.office365.com)', + required: true, + mode: 'trigger', + }, + { + id: 'port', + title: 'Port', + type: 'short-input', + placeholder: '993', + description: 'IMAP port (993 for SSL/TLS, 143 for STARTTLS)', + defaultValue: '993', + required: true, + mode: 'trigger', + }, + { + id: 'secure', + title: 'Use SSL/TLS', + type: 'switch', + defaultValue: true, + description: 'Enable SSL/TLS encryption (recommended for port 993)', + required: false, + mode: 'trigger', + }, + { + id: 'rejectUnauthorized', + title: 'Verify TLS Certificate', + type: 'switch', + defaultValue: true, + description: 'Verify server TLS certificate. Disable for self-signed certificates.', + required: false, + mode: 'trigger', + }, + // Authentication + { + id: 'username', + title: 'Username', + type: 'short-input', + placeholder: 'user@example.com', + description: 'Email address or username for authentication', + required: true, + mode: 'trigger', + }, + { + id: 'password', + title: 'Password', + type: 'short-input', + password: true, + placeholder: 'App password or email password', + description: 'Password or app-specific password (for Gmail, use an App Password)', + required: true, + mode: 'trigger', + }, + // Mailbox selection + { + id: 'mailbox', + title: 'Mailbox', + type: 'dropdown', + placeholder: 'Select mailbox to monitor', + description: 'Choose which mailbox/folder to monitor for new emails', + required: true, + options: [{ id: 'INBOX', label: 'INBOX' }], + fetchOptions: async (blockId: string, _subBlockId: string) => { + const store = useSubBlockStore.getState() + const host = store.getValue(blockId, 'host') as string | null + const port = store.getValue(blockId, 'port') as string | null + const secure = store.getValue(blockId, 'secure') as boolean | null + const rejectUnauthorized = store.getValue(blockId, 'rejectUnauthorized') as boolean | null + const username = store.getValue(blockId, 'username') as string | null + const password = store.getValue(blockId, 'password') as string | null + + if (!host || !username || !password) { + throw new Error('Please enter IMAP server, username, and password first') + } + + try { + const response = await fetch('/api/tools/imap/mailboxes', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + host, + port: port ? Number.parseInt(port, 10) : 993, + secure: secure ?? true, + rejectUnauthorized: rejectUnauthorized ?? true, + username, + password, + }), + }) + + if (!response.ok) { + const error = await response.json() + throw new Error(error.message || 'Failed to fetch mailboxes') + } + + const data = await response.json() + if (data.mailboxes && Array.isArray(data.mailboxes)) { + return data.mailboxes.map((mailbox: { path: string; name: string }) => ({ + id: mailbox.path, + label: mailbox.name, + })) + } + return [{ id: 'INBOX', label: 'INBOX' }] + } catch (error) { + logger.error('Error fetching IMAP mailboxes:', error) + throw error + } + }, + dependsOn: ['host', 'port', 'secure', 'rejectUnauthorized', 'username', 'password'], + mode: 'trigger', + }, + // Email filtering + { + id: 'searchCriteria', + title: 'Search Criteria', + type: 'short-input', + placeholder: 'UNSEEN', + description: + 'IMAP search criteria (e.g., UNSEEN, FROM "sender@example.com", SUBJECT "report"). Default: UNSEEN', + defaultValue: 'UNSEEN', + required: false, + mode: 'trigger', + }, + // Processing options + { + id: 'markAsRead', + title: 'Mark as Read', + type: 'switch', + defaultValue: true, + description: 'Automatically mark emails as read (SEEN) after processing', + required: false, + mode: 'trigger', + }, + { + id: 'includeAttachments', + title: 'Include Attachments', + type: 'switch', + defaultValue: false, + description: 'Download and include email attachments in the trigger payload', + required: false, + mode: 'trigger', + }, + // Instructions + { + id: 'triggerInstructions', + title: 'Setup Instructions', + hideFromPreview: true, + type: 'text', + defaultValue: [ + 'Enter your IMAP server details (host, port, SSL settings)', + 'Enter your email credentials (username and password)', + 'For Gmail: Use an App Password instead of your regular password', + 'Select the mailbox to monitor (INBOX is most common)', + 'Optionally configure search criteria and processing options', + 'The system will automatically check for new emails and trigger your workflow', + ] + .map( + (instruction, index) => + `
${index + 1}. ${instruction}
` + ) + .join(''), + mode: 'trigger', + }, + { + id: 'triggerSave', + title: '', + type: 'trigger-save', + hideFromPreview: true, + mode: 'trigger', + triggerId: 'imap_poller', + }, + ], + + outputs: { + email: { + uid: { + type: 'string', + description: 'IMAP message UID', + }, + messageId: { + type: 'string', + description: 'RFC Message-ID header', + }, + subject: { + type: 'string', + description: 'Email subject line', + }, + from: { + type: 'string', + description: 'Sender email address', + }, + to: { + type: 'string', + description: 'Recipient email address', + }, + cc: { + type: 'string', + description: 'CC recipients', + }, + date: { + type: 'string', + description: 'Email date in ISO format', + }, + bodyText: { + type: 'string', + description: 'Plain text email body', + }, + bodyHtml: { + type: 'string', + description: 'HTML email body', + }, + mailbox: { + type: 'string', + description: 'Mailbox/folder where email was received', + }, + hasAttachments: { + type: 'boolean', + description: 'Whether email has attachments', + }, + attachments: { + type: 'file[]', + description: 'Array of email attachments as files (if includeAttachments is enabled)', + }, + }, + timestamp: { + type: 'string', + description: 'Event timestamp', + }, + }, +} diff --git a/apps/sim/triggers/registry.ts b/apps/sim/triggers/registry.ts index 942ff0e018..7d2fc552be 100644 --- a/apps/sim/triggers/registry.ts +++ b/apps/sim/triggers/registry.ts @@ -55,6 +55,7 @@ import { hubspotTicketDeletedTrigger, hubspotTicketPropertyChangedTrigger, } from '@/triggers/hubspot' +import { imapPollingTrigger } from '@/triggers/imap' import { jiraIssueCommentedTrigger, jiraIssueCreatedTrigger, @@ -183,4 +184,5 @@ export const TRIGGER_REGISTRY: TriggerRegistry = { hubspot_ticket_created: hubspotTicketCreatedTrigger, hubspot_ticket_deleted: hubspotTicketDeletedTrigger, hubspot_ticket_property_changed: hubspotTicketPropertyChangedTrigger, + imap_poller: imapPollingTrigger, } diff --git a/bun.lock b/bun.lock index 23513563b2..5e24a3f2cc 100644 --- a/bun.lock +++ b/bun.lock @@ -132,6 +132,7 @@ "groq-sdk": "^0.15.0", "html-to-image": "1.11.13", "html-to-text": "^9.0.5", + "imapflow": "1.2.4", "input-otp": "^1.4.2", "ioredis": "^5.6.0", "isolated-vm": "6.0.2", @@ -1556,6 +1557,8 @@ "@xmldom/xmldom": ["@xmldom/xmldom@0.8.11", "", {}, "sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw=="], + "@zone-eu/mailsplit": ["@zone-eu/mailsplit@5.4.8", "", { "dependencies": { "libbase64": "1.3.0", "libmime": "5.3.7", "libqp": "2.1.1" } }, "sha512-eEyACj4JZ7sjzRvy26QhLgKEMWwQbsw1+QZnlLX+/gihcNH07lVPOcnwf5U6UAL7gkc//J3jVd76o/WS+taUiA=="], + "abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="], "accepts": ["accepts@1.3.8", "", { "dependencies": { "mime-types": "~2.1.34", "negotiator": "0.6.3" } }, "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw=="], @@ -1998,6 +2001,8 @@ "encoding": ["encoding@0.1.13", "", { "dependencies": { "iconv-lite": "^0.6.2" } }, "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A=="], + "encoding-japanese": ["encoding-japanese@2.2.0", "", {}, "sha512-EuJWwlHPZ1LbADuKTClvHtwbaFn4rOD+dRAbWysqEOXRc2Uui0hJInNJrsdH0c+OhJA4nrCBdSkW4DD5YxAo6A=="], + "encoding-sniffer": ["encoding-sniffer@0.2.1", "", { "dependencies": { "iconv-lite": "^0.6.3", "whatwg-encoding": "^3.1.1" } }, "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw=="], "end-of-stream": ["end-of-stream@1.4.5", "", { "dependencies": { "once": "^1.4.0" } }, "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg=="], @@ -2292,6 +2297,8 @@ "image-size": ["image-size@2.0.2", "", { "bin": { "image-size": "bin/image-size.js" } }, "sha512-IRqXKlaXwgSMAMtpNzZa1ZAe8m+Sa1770Dhk8VkSsP9LS+iHD62Zd8FQKs8fbPiagBE7BzoFX23cxFnwshpV6w=="], + "imapflow": ["imapflow@1.2.4", "", { "dependencies": { "@zone-eu/mailsplit": "5.4.8", "encoding-japanese": "2.2.0", "iconv-lite": "0.7.1", "libbase64": "1.3.0", "libmime": "5.3.7", "libqp": "2.1.1", "nodemailer": "7.0.12", "pino": "10.1.0", "socks": "2.8.7" } }, "sha512-X/eRQeje33uZycfopjwoQKKbya+bBIaqpviOFxhPOD24DXU2hMfXwYe9e8j1+ADwFVgTvKq4G2/ljjZK3Y8mvg=="], + "immediate": ["immediate@3.0.6", "", {}, "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ=="], "import-in-the-middle": ["import-in-the-middle@1.15.0", "", { "dependencies": { "acorn": "^8.14.0", "acorn-import-attributes": "^1.9.5", "cjs-module-lexer": "^1.2.2", "module-details-from-path": "^1.0.3" } }, "sha512-bpQy+CrsRmYmoPMAE/0G33iwRqwW4ouqdRg8jgbH3aKuCtOc8lxgmYXg2dMM92CRiGP660EtBcymH/eVUpCSaA=="], @@ -2424,6 +2431,12 @@ "leac": ["leac@0.6.0", "", {}, "sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg=="], + "libbase64": ["libbase64@1.3.0", "", {}, "sha512-GgOXd0Eo6phYgh0DJtjQ2tO8dc0IVINtZJeARPeiIJqge+HdsWSuaDTe8ztQ7j/cONByDZ3zeB325AHiv5O0dg=="], + + "libmime": ["libmime@5.3.7", "", { "dependencies": { "encoding-japanese": "2.2.0", "iconv-lite": "0.6.3", "libbase64": "1.3.0", "libqp": "2.1.1" } }, "sha512-FlDb3Wtha8P01kTL3P9M+ZDNDWPKPmKHWaU/cG/lg5pfuAwdflVpZE+wm9m7pKmC5ww6s+zTxBKS1p6yl3KpSw=="], + + "libqp": ["libqp@2.1.1", "", {}, "sha512-0Wd+GPz1O134cP62YU2GTOPNA7Qgl09XwCqM5zpBv87ERCXdfDtyKXvV7c9U22yWJh44QZqBocFnXN11K96qow=="], + "lie": ["lie@3.3.0", "", { "dependencies": { "immediate": "~3.0.5" } }, "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ=="], "lighthouse-logger": ["lighthouse-logger@2.0.2", "", { "dependencies": { "debug": "^4.4.1", "marky": "^1.2.2" } }, "sha512-vWl2+u5jgOQuZR55Z1WM0XDdrJT6mzMP8zHUct7xTlWhuQs+eV0g+QL0RQdFjT54zVmbhLCP8vIVpy1wGn/gCg=="], @@ -3958,6 +3971,12 @@ "http-response-object/@types/node": ["@types/node@10.17.60", "", {}, "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw=="], + "imapflow/iconv-lite": ["iconv-lite@0.7.1", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-2Tth85cXwGFHfvRgZWszZSvdo+0Xsqmw8k8ZwxScfcBneNUraK+dxRxRm24nszx80Y0TVio8kKLt5sLE7ZCLlw=="], + + "imapflow/nodemailer": ["nodemailer@7.0.12", "", {}, "sha512-H+rnK5bX2Pi/6ms3sN4/jRQvYSMltV6vqup/0SFOrxYYY/qoNvhXPlYq3e+Pm9RFJRwrMGbMIwi81M4dxpomhA=="], + + "imapflow/pino": ["pino@10.1.0", "", { "dependencies": { "@pinojs/redact": "^0.4.0", "atomic-sleep": "^1.0.0", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^2.0.0", "pino-std-serializers": "^7.0.0", "process-warning": "^5.0.0", "quick-format-unescaped": "^4.0.3", "real-require": "^0.2.0", "safe-stable-stringify": "^2.3.1", "sonic-boom": "^4.0.1", "thread-stream": "^3.0.0" }, "bin": { "pino": "bin.js" } }, "sha512-0zZC2ygfdqvqK8zJIr1e+wT1T/L+LF6qvqvbzEQ6tiMAoTqEVK9a1K3YRu8HEUvGEvNqZyPJTtb2sNIoTkB83w=="], + "inquirer/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], "inquirer/ora": ["ora@5.4.1", "", { "dependencies": { "bl": "^4.1.0", "chalk": "^4.1.0", "cli-cursor": "^3.1.0", "cli-spinners": "^2.5.0", "is-interactive": "^1.0.0", "is-unicode-supported": "^0.1.0", "log-symbols": "^4.1.0", "strip-ansi": "^6.0.0", "wcwidth": "^1.0.1" } }, "sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ=="], @@ -4392,6 +4411,8 @@ "groq-sdk/node-fetch/whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="], + "imapflow/pino/thread-stream": ["thread-stream@3.1.0", "", { "dependencies": { "real-require": "^0.2.0" } }, "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A=="], + "inquirer/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], "inquirer/ora/is-interactive": ["is-interactive@1.0.0", "", {}, "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w=="], diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index b690c64d5f..a439330f29 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -687,6 +687,15 @@ cronjobs: successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 1 + imapWebhookPoll: + enabled: true + name: imap-webhook-poll + schedule: "*/1 * * * *" + path: "/api/webhooks/poll/imap" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 1 + renewSubscriptions: enabled: true name: renew-subscriptions From d5bcc14af74988be021e78077285845fe431d3e8 Mon Sep 17 00:00:00 2001 From: waleed Date: Fri, 2 Jan 2026 12:41:15 -0800 Subject: [PATCH 2/4] feat(imap): added parity, tested --- apps/docs/components/icons.tsx | 1 - .../sim/app/api/tools/imap/mailboxes/route.ts | 4 - .../components/dropdown/dropdown.tsx | 8 +- apps/sim/blocks/blocks/imap.ts | 1 - apps/sim/components/icons.tsx | 1 - apps/sim/lib/webhooks/imap-polling-service.ts | 222 +++++++++++------- apps/sim/lib/webhooks/utils.server.ts | 34 ++- apps/sim/triggers/gmail/poller.ts | 24 ++ apps/sim/triggers/imap/poller.ts | 52 +++- 9 files changed, 239 insertions(+), 108 deletions(-) diff --git a/apps/docs/components/icons.tsx b/apps/docs/components/icons.tsx index 751494d6e2..c7ceece206 100644 --- a/apps/docs/components/icons.tsx +++ b/apps/docs/components/icons.tsx @@ -305,7 +305,6 @@ export function MailServerIcon(props: SVGProps) { fill='none' xmlns='http://www.w3.org/2000/svg' > - {/* Server/inbox icon with mail symbol */} ({ path: mailbox.path, @@ -52,7 +51,6 @@ export async function POST(request: NextRequest) { await client.logout() - // Sort mailboxes: INBOX first, then alphabetically mailboxes.sort((a, b) => { if (a.path === 'INBOX') return -1 if (b.path === 'INBOX') return 1 @@ -64,7 +62,6 @@ export async function POST(request: NextRequest) { mailboxes, }) } catch (error) { - // Make sure to close connection on error try { await client.logout() } catch { @@ -76,7 +73,6 @@ export async function POST(request: NextRequest) { const errorMessage = error instanceof Error ? error.message : 'Unknown error' logger.error('Error fetching IMAP mailboxes:', errorMessage) - // Provide user-friendly error messages let userMessage = 'Failed to connect to IMAP server' if ( errorMessage.includes('AUTHENTICATIONFAILED') || diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/dropdown/dropdown.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/dropdown/dropdown.tsx index 0e5f6bc58a..15779d23e4 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/dropdown/dropdown.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/dropdown/dropdown.tsx @@ -113,7 +113,13 @@ export function Dropdown({ const value = isPreview ? previewValue : propValue !== undefined ? propValue : storeValue const singleValue = multiSelect ? null : (value as string | null | undefined) - const multiValues = multiSelect ? (value as string[] | null | undefined) || [] : null + const multiValues = multiSelect + ? Array.isArray(value) + ? value + : value + ? [value as string] + : [] + : null const fetchOptionsIfNeeded = useCallback(async () => { if (!fetchOptions || isPreview || disabled) return diff --git a/apps/sim/blocks/blocks/imap.ts b/apps/sim/blocks/blocks/imap.ts index 398bdadf78..6732c30d79 100644 --- a/apps/sim/blocks/blocks/imap.ts +++ b/apps/sim/blocks/blocks/imap.ts @@ -35,7 +35,6 @@ export const ImapBlock: BlockConfig = { }, outputs: { // Trigger outputs (from email) - uid: { type: 'string', description: 'IMAP message UID' }, messageId: { type: 'string', description: 'RFC Message-ID header' }, subject: { type: 'string', description: 'Email subject line' }, from: { type: 'string', description: 'Sender email address' }, diff --git a/apps/sim/components/icons.tsx b/apps/sim/components/icons.tsx index 751494d6e2..c7ceece206 100644 --- a/apps/sim/components/icons.tsx +++ b/apps/sim/components/icons.tsx @@ -305,7 +305,6 @@ export function MailServerIcon(props: SVGProps) { fill='none' xmlns='http://www.w3.org/2000/svg' > - {/* Server/inbox icon with mail symbol */} // Track UID per mailbox for multi-mailbox + lastCheckedTimestamp?: string // ISO timestamp of last successful poll maxEmailsPerPoll?: number } @@ -121,7 +123,6 @@ export async function pollImapWebhooks() { logger.info(`Found ${activeWebhooks.length} active IMAP webhooks`) - // Limit concurrency to avoid overwhelming IMAP servers const CONCURRENCY = 5 const running: Promise[] = [] @@ -143,12 +144,11 @@ export async function pollImapWebhooks() { } const fetchResult = await fetchNewEmails(config, requestId) - const { emails, latestUid } = fetchResult + const { emails, latestUidByMailbox } = fetchResult + const pollTimestamp = new Date().toISOString() if (!emails || !emails.length) { - if (latestUid && latestUid !== config.lastProcessedUid) { - await updateWebhookLastProcessedUid(webhookId, latestUid) - } + await updateWebhookLastProcessedUids(webhookId, latestUidByMailbox, pollTimestamp) await markWebhookSuccess(webhookId) logger.info(`[${requestId}] No new emails found for webhook ${webhookId}`) successCount++ @@ -164,9 +164,7 @@ export async function pollImapWebhooks() { requestId ) - if (latestUid) { - await updateWebhookLastProcessedUid(webhookId, latestUid) - } + await updateWebhookLastProcessedUids(webhookId, latestUidByMailbox, pollTimestamp) if (emailFailedCount > 0 && processedCount === 0) { await markWebhookFailed(webhookId) @@ -244,79 +242,109 @@ async function fetchNewEmails(config: ImapWebhookConfig, requestId: string) { const emails: Array<{ uid: number + mailboxPath: string // Track which mailbox this email came from envelope: FetchMessageObject['envelope'] bodyStructure: FetchMessageObject['bodyStructure'] source?: Buffer }> = [] - let latestUid = config.lastProcessedUid + + const mailboxes = getMailboxesToCheck(config) + const latestUidByMailbox: Record = { ...(config.lastProcessedUidByMailbox || {}) } try { await client.connect() logger.debug(`[${requestId}] Connected to IMAP server ${config.host}`) - const mailbox = await client.mailboxOpen(config.mailbox || 'INBOX') - logger.debug(`[${requestId}] Opened mailbox: ${mailbox.path}, exists: ${mailbox.exists}`) + const maxEmails = config.maxEmailsPerPoll || 25 + let totalEmailsCollected = 0 - // Build search criteria - let searchCriteria: any = config.searchCriteria || 'UNSEEN' + for (const mailboxPath of mailboxes) { + if (totalEmailsCollected >= maxEmails) break - // If we have a last processed UID, add UID filter - if (config.lastProcessedUid) { - // Search for messages with UID greater than last processed - const uidCriteria = { uid: `${config.lastProcessedUid + 1}:*` } - if (typeof searchCriteria === 'string') { - searchCriteria = { [searchCriteria]: true, ...uidCriteria } - } else { - searchCriteria = { ...searchCriteria, ...uidCriteria } - } - } + try { + const mailbox = await client.mailboxOpen(mailboxPath) + logger.debug(`[${requestId}] Opened mailbox: ${mailbox.path}, exists: ${mailbox.exists}`) - // Search for matching messages - const messageUids: number[] = [] - try { - for await (const msg of client.fetch(searchCriteria, { uid: true })) { - messageUids.push(msg.uid) - } - } catch (fetchError) { - // If search fails (e.g., no messages match), return empty - logger.debug(`[${requestId}] Fetch returned no messages or failed: ${fetchError}`) - await client.logout() - return { emails: [], latestUid } - } + const rawCriteria = config.searchCriteria || 'UNSEEN' + let searchCriteria: any = + typeof rawCriteria === 'string' ? { [rawCriteria.toLowerCase()]: true } : rawCriteria - if (messageUids.length === 0) { - logger.debug(`[${requestId}] No messages matching criteria`) - await client.logout() - return { emails: [], latestUid } - } + const lastUidForMailbox = latestUidByMailbox[mailboxPath] || config.lastProcessedUid - // Sort UIDs and take the most recent ones - messageUids.sort((a, b) => b - a) - const maxEmails = config.maxEmailsPerPoll || 25 - const uidsToProcess = messageUids.slice(0, maxEmails) - latestUid = Math.max(...uidsToProcess, config.lastProcessedUid || 0) - - logger.info(`[${requestId}] Processing ${uidsToProcess.length} emails from ${config.mailbox}`) - - // Fetch full message details - for await (const msg of client.fetch(uidsToProcess, { - uid: true, - envelope: true, - bodyStructure: true, - source: true, - })) { - emails.push({ - uid: msg.uid, - envelope: msg.envelope, - bodyStructure: msg.bodyStructure, - source: msg.source, - }) + if (lastUidForMailbox) { + searchCriteria = { ...searchCriteria, uid: `${lastUidForMailbox + 1}:*` } + } + + // Add time-based filtering similar to Gmail + // If lastCheckedTimestamp exists, use it with 1 minute buffer + // If first poll (no timestamp), default to last 24 hours to avoid processing ALL unseen emails + if (config.lastCheckedTimestamp) { + const lastChecked = new Date(config.lastCheckedTimestamp) + const bufferTime = new Date(lastChecked.getTime() - 60000) + searchCriteria = { ...searchCriteria, since: bufferTime } + } else { + // First poll: only get emails from last 24 hours to avoid overwhelming first run + const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000) + searchCriteria = { ...searchCriteria, since: oneDayAgo } + } + + let messageUids: number[] = [] + try { + const searchResult = await client.search(searchCriteria, { uid: true }) + messageUids = searchResult === false ? [] : searchResult + } catch (searchError) { + logger.debug( + `[${requestId}] Search returned no messages for ${mailboxPath}: ${searchError}` + ) + continue + } + + if (messageUids.length === 0) { + logger.debug(`[${requestId}] No messages matching criteria in ${mailboxPath}`) + continue + } + + messageUids.sort((a, b) => b - a) + const remainingSlots = maxEmails - totalEmailsCollected + const uidsToProcess = messageUids.slice(0, remainingSlots) + + if (uidsToProcess.length > 0) { + latestUidByMailbox[mailboxPath] = Math.max( + ...uidsToProcess, + latestUidByMailbox[mailboxPath] || 0 + ) + } + + logger.info(`[${requestId}] Processing ${uidsToProcess.length} emails from ${mailboxPath}`) + + for await (const msg of client.fetch( + uidsToProcess, + { + uid: true, + envelope: true, + bodyStructure: true, + source: true, + }, + { uid: true } + )) { + emails.push({ + uid: msg.uid, + mailboxPath, + envelope: msg.envelope, + bodyStructure: msg.bodyStructure, + source: msg.source, + }) + totalEmailsCollected++ + } + } catch (mailboxError) { + logger.warn(`[${requestId}] Error processing mailbox ${mailboxPath}:`, mailboxError) + } } await client.logout() logger.debug(`[${requestId}] Disconnected from IMAP server`) - return { emails, latestUid } + return { emails, latestUidByMailbox } } catch (error) { try { await client.logout() @@ -327,6 +355,19 @@ async function fetchNewEmails(config: ImapWebhookConfig, requestId: string) { } } +/** + * Get the list of mailboxes to check based on config + */ +function getMailboxesToCheck(config: ImapWebhookConfig): string[] { + if (!config.mailbox || (Array.isArray(config.mailbox) && config.mailbox.length === 0)) { + return ['INBOX'] + } + if (Array.isArray(config.mailbox)) { + return config.mailbox + } + return [config.mailbox] +} + function parseEmailAddress( addr: { name?: string; address?: string } | { name?: string; address?: string }[] | undefined ): string { @@ -345,24 +386,20 @@ function extractTextFromSource(source: Buffer): { text: string; html: string } { let text = '' let html = '' - // Simple extraction - look for Content-Type boundaries const parts = content.split(/--[^\r\n]+/) for (const part of parts) { const lowerPart = part.toLowerCase() if (lowerPart.includes('content-type: text/plain')) { - // Extract text content after headers (double newline) const match = part.match(/\r?\n\r?\n([\s\S]*?)(?=\r?\n--|\r?\n\.\r?\n|$)/i) if (match) { text = match[1].trim() - // Handle quoted-printable decoding if (lowerPart.includes('quoted-printable')) { text = text .replace(/=\r?\n/g, '') .replace(/=([0-9A-F]{2})/gi, (_, hex) => String.fromCharCode(Number.parseInt(hex, 16))) } - // Handle base64 decoding if (lowerPart.includes('base64')) { try { text = Buffer.from(text.replace(/\s/g, ''), 'base64').toString('utf-8') @@ -391,7 +428,6 @@ function extractTextFromSource(source: Buffer): { text: string; html: string } { } } - // If no multipart, try to get the body directly if (!text && !html) { const bodyMatch = content.match(/\r?\n\r?\n([\s\S]+)$/) if (bodyMatch) { @@ -416,7 +452,6 @@ function extractAttachmentsFromSource( for (const part of parts) { const lowerPart = part.toLowerCase() - // Look for attachment dispositions or non-text content types const dispositionMatch = part.match( /content-disposition:\s*attachment[^;]*;\s*filename="?([^"\r\n]+)"?/i ) @@ -430,12 +465,10 @@ function extractAttachmentsFromSource( const filename = dispositionMatch?.[1] || filenameMatch?.[1] || 'attachment' const mimeType = contentTypeMatch?.[1]?.trim() || 'application/octet-stream' - // Extract the attachment data const dataMatch = part.match(/\r?\n\r?\n([\s\S]*?)$/i) if (dataMatch) { const data = dataMatch[1].trim() - // Most attachments are base64 encoded if (lowerPart.includes('base64')) { try { const buffer = Buffer.from(data.replace(/\s/g, ''), 'base64') @@ -459,6 +492,7 @@ function extractAttachmentsFromSource( async function processEmails( emails: Array<{ uid: number + mailboxPath: string envelope: FetchMessageObject['envelope'] bodyStructure: FetchMessageObject['bodyStructure'] source?: Buffer @@ -470,7 +504,6 @@ async function processEmails( let processedCount = 0 let failedCount = 0 - // Create a new client for marking messages const client = new ImapFlow({ host: config.host, port: config.port || 993, @@ -485,26 +518,26 @@ async function processEmails( logger: false, }) + let currentOpenMailbox: string | null = null + const lockState: { lock: MailboxLockObject | null } = { lock: null } + try { if (config.markAsRead) { await client.connect() - await client.mailboxOpen(config.mailbox || 'INBOX') } for (const email of emails) { try { await pollingIdempotency.executeWithIdempotency( 'imap', - `${webhookData.id}:${email.uid}`, + `${webhookData.id}:${email.mailboxPath}:${email.uid}`, async () => { const envelope = email.envelope - // Extract body content const { text: bodyText, html: bodyHtml } = email.source ? extractTextFromSource(email.source) : { text: '', html: '' } - // Extract attachments if enabled let attachments: ImapAttachment[] = [] const hasAttachments = email.bodyStructure ? JSON.stringify(email.bodyStructure).toLowerCase().includes('attachment') @@ -524,7 +557,7 @@ async function processEmails( date: envelope?.date ? new Date(envelope.date).toISOString() : null, bodyText, bodyHtml, - mailbox: config.mailbox || 'INBOX', + mailbox: email.mailboxPath, hasAttachments, attachments, } @@ -556,10 +589,17 @@ async function processEmails( throw new Error(`Webhook request failed: ${response.status} - ${errorText}`) } - // Mark as read if configured if (config.markAsRead) { try { - await client.messageFlagsAdd({ uid: email.uid }, ['\\Seen']) + if (currentOpenMailbox !== email.mailboxPath) { + if (lockState.lock) { + lockState.lock.release() + lockState.lock = null + } + lockState.lock = await client.getMailboxLock(email.mailboxPath) + currentOpenMailbox = email.mailboxPath + } + await client.messageFlagsAdd({ uid: email.uid }, ['\\Seen'], { uid: true }) } catch (flagError) { logger.warn( `[${requestId}] Failed to mark message ${email.uid} as read:`, @@ -577,7 +617,7 @@ async function processEmails( ) logger.info( - `[${requestId}] Successfully processed email ${email.uid} for webhook ${webhookData.id}` + `[${requestId}] Successfully processed email ${email.uid} from ${email.mailboxPath} for webhook ${webhookData.id}` ) processedCount++ } catch (error) { @@ -589,6 +629,9 @@ async function processEmails( } finally { if (config.markAsRead) { try { + if (lockState.lock) { + lockState.lock.release() + } await client.logout() } catch { // Ignore logout errors @@ -599,15 +642,28 @@ async function processEmails( return { processedCount, failedCount } } -async function updateWebhookLastProcessedUid(webhookId: string, uid: number) { +async function updateWebhookLastProcessedUids( + webhookId: string, + uidByMailbox: Record, + timestamp: string +) { const result = await db.select().from(webhook).where(eq(webhook.id, webhookId)) const existingConfig = (result[0]?.providerConfig as Record) || {} + + const existingUidByMailbox = existingConfig.lastProcessedUidByMailbox || {} + const mergedUidByMailbox = { ...existingUidByMailbox } + + for (const [mailbox, uid] of Object.entries(uidByMailbox)) { + mergedUidByMailbox[mailbox] = Math.max(uid, mergedUidByMailbox[mailbox] || 0) + } + await db .update(webhook) .set({ providerConfig: { ...existingConfig, - lastProcessedUid: uid, + lastProcessedUidByMailbox: mergedUidByMailbox, + lastCheckedTimestamp: timestamp, } as any, updatedAt: new Date(), }) diff --git a/apps/sim/lib/webhooks/utils.server.ts b/apps/sim/lib/webhooks/utils.server.ts index 8f538c2690..64f2de2ece 100644 --- a/apps/sim/lib/webhooks/utils.server.ts +++ b/apps/sim/lib/webhooks/utils.server.ts @@ -870,9 +870,34 @@ export async function formatWebhookInput( } if (foundWebhook.provider === 'imap') { - // IMAP email data is already formatted by the polling service if (body && typeof body === 'object' && 'email' in body) { - return body + const email = body.email as Record + return { + messageId: email?.messageId, + subject: email?.subject, + from: email?.from, + to: email?.to, + cc: email?.cc, + date: email?.date, + bodyText: email?.bodyText, + bodyHtml: email?.bodyHtml, + mailbox: email?.mailbox, + hasAttachments: email?.hasAttachments, + attachments: email?.attachments, + email, + timestamp: body.timestamp, + webhook: { + data: { + provider: 'imap', + path: foundWebhook.path, + providerConfig: foundWebhook.providerConfig, + payload: body, + headers: Object.fromEntries(request.headers.entries()), + method: request.method, + }, + }, + workflowId: foundWorkflow.id, + } } return body } @@ -2578,7 +2603,6 @@ export async function configureImapPolling(webhookData: any, requestId: string): const providerConfig = (webhookData.providerConfig as Record) || {} const now = new Date() - // Validate required IMAP connection settings if (!providerConfig.host || !providerConfig.username || !providerConfig.password) { logger.error( `[${requestId}] Missing required IMAP connection settings for webhook ${webhookData.id}` @@ -2591,17 +2615,13 @@ export async function configureImapPolling(webhookData: any, requestId: string): .set({ providerConfig: { ...providerConfig, - // Connection defaults port: providerConfig.port || '993', secure: providerConfig.secure !== false, rejectUnauthorized: providerConfig.rejectUnauthorized !== false, - // Mailbox defaults mailbox: providerConfig.mailbox || 'INBOX', searchCriteria: providerConfig.searchCriteria || 'UNSEEN', - // Processing options markAsRead: providerConfig.markAsRead || false, includeAttachments: providerConfig.includeAttachments !== false, - // Polling state lastCheckedTimestamp: now.toISOString(), setupCompleted: true, }, diff --git a/apps/sim/triggers/gmail/poller.ts b/apps/sim/triggers/gmail/poller.ts index 04b2f4cf8b..cfda07a2d9 100644 --- a/apps/sim/triggers/gmail/poller.ts +++ b/apps/sim/triggers/gmail/poller.ts @@ -85,6 +85,30 @@ export const gmailPollingTrigger: TriggerConfig = { 'Optional Gmail search query to filter emails. Use the same format as Gmail search box (e.g., "subject:invoice", "from:boss@company.com", "has:attachment"). Leave empty to search all emails.', required: false, mode: 'trigger', + wandConfig: { + enabled: true, + maintainHistory: true, + prompt: `You are an expert in Gmail search syntax. Generate Gmail search queries based on user descriptions. + +Gmail search operators include: +- from: / to: / cc: / bcc: - Filter by sender/recipient +- subject: - Search in subject line +- has:attachment - Emails with attachments +- filename: - Search attachment filenames +- is:unread / is:read / is:starred +- after: / before: / older: / newer: - Date filters (YYYY/MM/DD) +- label: - Filter by label +- in:inbox / in:spam / in:trash +- larger: / smaller: - Size filters (e.g., 10M, 1K) +- OR / AND / - (NOT) - Boolean operators +- "exact phrase" - Exact match +- ( ) - Grouping + +Current query: {context} + +Return ONLY the Gmail search query, no explanations or markdown.`, + placeholder: 'Describe what emails you want to filter...', + }, }, { id: 'markAsRead', diff --git a/apps/sim/triggers/imap/poller.ts b/apps/sim/triggers/imap/poller.ts index 4adf5383da..4b01a5e2b9 100644 --- a/apps/sim/triggers/imap/poller.ts +++ b/apps/sim/triggers/imap/poller.ts @@ -75,12 +75,14 @@ export const imapPollingTrigger: TriggerConfig = { // Mailbox selection { id: 'mailbox', - title: 'Mailbox', + title: 'Mailboxes to Monitor', type: 'dropdown', - placeholder: 'Select mailbox to monitor', - description: 'Choose which mailbox/folder to monitor for new emails', - required: true, - options: [{ id: 'INBOX', label: 'INBOX' }], + multiSelect: true, + placeholder: 'Select mailboxes to monitor', + description: + 'Choose which mailbox/folder(s) to monitor for new emails. Leave empty to monitor INBOX.', + required: false, + options: [], fetchOptions: async (blockId: string, _subBlockId: string) => { const store = useSubBlockStore.getState() const host = store.getValue(blockId, 'host') as string | null @@ -120,7 +122,7 @@ export const imapPollingTrigger: TriggerConfig = { label: mailbox.name, })) } - return [{ id: 'INBOX', label: 'INBOX' }] + return [] } catch (error) { logger.error('Error fetching IMAP mailboxes:', error) throw error @@ -140,6 +142,40 @@ export const imapPollingTrigger: TriggerConfig = { defaultValue: 'UNSEEN', required: false, mode: 'trigger', + wandConfig: { + enabled: true, + maintainHistory: true, + prompt: `You are an expert in IMAP search syntax (RFC 3501). Generate IMAP search criteria based on user descriptions. + +IMAP search keys include: +- ALL - All messages +- UNSEEN / SEEN - Unread/read messages +- FLAGGED / UNFLAGGED - Starred/unstarred +- FROM "string" - Sender contains string +- TO "string" - Recipient contains string +- SUBJECT "string" - Subject contains string +- BODY "string" - Body contains string +- TEXT "string" - Headers or body contains string +- BEFORE date / SINCE date / ON date - Date filters (DD-Mon-YYYY, e.g., 01-Jan-2024) +- LARGER n / SMALLER n - Size in bytes +- HEADER field-name "string" - Custom header search +- NOT criteria - Negate +- OR criteria1 criteria2 - Either matches +- (criteria) - Grouping + +Multiple criteria are AND'd together by default. + +Examples: +- UNSEEN FROM "boss@company.com" +- OR FROM "alice" FROM "bob" +- SINCE 01-Jan-2024 SUBJECT "report" +- NOT SEEN FLAGGED + +Current criteria: {context} + +Return ONLY the IMAP search criteria, no explanations or markdown.`, + placeholder: 'Describe what emails you want to filter...', + }, }, // Processing options { @@ -193,10 +229,6 @@ export const imapPollingTrigger: TriggerConfig = { outputs: { email: { - uid: { - type: 'string', - description: 'IMAP message UID', - }, messageId: { type: 'string', description: 'RFC Message-ID header', From 26caa7f5a940d2b9d7b1680622621f2848cf18b1 Mon Sep 17 00:00:00 2001 From: waleed Date: Fri, 2 Jan 2026 14:47:02 -0800 Subject: [PATCH 3/4] ack PR comments --- apps/sim/blocks/blocks/imap.ts | 2 -- apps/sim/blocks/registry.ts | 2 +- apps/sim/lib/webhooks/imap-polling-service.ts | 32 ++++++++++++++++--- apps/sim/lib/workflows/comparison/compare.ts | 8 +++-- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/apps/sim/blocks/blocks/imap.ts b/apps/sim/blocks/blocks/imap.ts index 6732c30d79..683928f197 100644 --- a/apps/sim/blocks/blocks/imap.ts +++ b/apps/sim/blocks/blocks/imap.ts @@ -21,7 +21,6 @@ export const ImapBlock: BlockConfig = { }, }, inputs: { - // Connection settings (stored in providerConfig) host: { type: 'string', description: 'IMAP server hostname' }, port: { type: 'string', description: 'IMAP server port' }, secure: { type: 'boolean', description: 'Use SSL/TLS encryption' }, @@ -34,7 +33,6 @@ export const ImapBlock: BlockConfig = { includeAttachments: { type: 'boolean', description: 'Include email attachments' }, }, outputs: { - // Trigger outputs (from email) messageId: { type: 'string', description: 'RFC Message-ID header' }, subject: { type: 'string', description: 'Email subject line' }, from: { type: 'string', description: 'Sender email address' }, diff --git a/apps/sim/blocks/registry.ts b/apps/sim/blocks/registry.ts index 2bf7392669..43cb633924 100644 --- a/apps/sim/blocks/registry.ts +++ b/apps/sim/blocks/registry.ts @@ -196,8 +196,8 @@ export const registry: Record = { huggingface: HuggingFaceBlock, human_in_the_loop: HumanInTheLoopBlock, hunter: HunterBlock, - imap: ImapBlock, image_generator: ImageGeneratorBlock, + imap: ImapBlock, incidentio: IncidentioBlock, input_trigger: InputTriggerBlock, intercom: IntercomBlock, diff --git a/apps/sim/lib/webhooks/imap-polling-service.ts b/apps/sim/lib/webhooks/imap-polling-service.ts index 44c4192b27..764950e57b 100644 --- a/apps/sim/lib/webhooks/imap-polling-service.ts +++ b/apps/sim/lib/webhooks/imap-polling-service.ts @@ -1,6 +1,7 @@ import { db } from '@sim/db' import { webhook, workflow } from '@sim/db/schema' import { createLogger } from '@sim/logger' +import type { InferSelectModel } from 'drizzle-orm' import { and, eq, sql } from 'drizzle-orm' import type { FetchMessageObject, MailboxLockObject } from 'imapflow' import { ImapFlow } from 'imapflow' @@ -11,6 +12,8 @@ import { MAX_CONSECUTIVE_FAILURES } from '@/triggers/constants' const logger = createLogger('ImapPollingService') +type WebhookRecord = InferSelectModel + interface ImapWebhookConfig { host: string port: number @@ -489,6 +492,27 @@ function extractAttachmentsFromSource( return attachments } +/** + * Checks if a body structure contains attachments by examining disposition + */ +function hasAttachmentsInBodyStructure(structure: FetchMessageObject['bodyStructure']): boolean { + if (!structure) return false + + if (structure.disposition === 'attachment') { + return true + } + + if (structure.disposition === 'inline' && structure.dispositionParameters?.filename) { + return true + } + + if (structure.childNodes && Array.isArray(structure.childNodes)) { + return structure.childNodes.some((child) => hasAttachmentsInBodyStructure(child)) + } + + return false +} + async function processEmails( emails: Array<{ uid: number @@ -497,7 +521,7 @@ async function processEmails( bodyStructure: FetchMessageObject['bodyStructure'] source?: Buffer }>, - webhookData: any, + webhookData: WebhookRecord, config: ImapWebhookConfig, requestId: string ) { @@ -539,9 +563,7 @@ async function processEmails( : { text: '', html: '' } let attachments: ImapAttachment[] = [] - const hasAttachments = email.bodyStructure - ? JSON.stringify(email.bodyStructure).toLowerCase().includes('attachment') - : false + const hasAttachments = hasAttachmentsInBodyStructure(email.bodyStructure) if (config.includeAttachments && hasAttachments && email.source) { attachments = extractAttachmentsFromSource(email.source, email.bodyStructure) @@ -573,7 +595,7 @@ async function processEmails( method: 'POST', headers: { 'Content-Type': 'application/json', - 'X-Webhook-Secret': webhookData.secret || '', + 'X-Webhook-Secret': '', 'User-Agent': 'Sim/1.0', }, body: JSON.stringify(payload), diff --git a/apps/sim/lib/workflows/comparison/compare.ts b/apps/sim/lib/workflows/comparison/compare.ts index 187894d0cc..a34521e23b 100644 --- a/apps/sim/lib/workflows/comparison/compare.ts +++ b/apps/sim/lib/workflows/comparison/compare.ts @@ -1,5 +1,5 @@ import type { WorkflowState } from '@/stores/workflows/workflow/types' -import { TRIGGER_RUNTIME_SUBBLOCK_IDS } from '@/triggers/constants' +import { SYSTEM_SUBBLOCK_IDS, TRIGGER_RUNTIME_SUBBLOCK_IDS } from '@/triggers/constants' import { normalizedStringify, normalizeEdge, @@ -103,11 +103,13 @@ export function hasWorkflowChanged( subBlocks: undefined, } - // Get all subBlock IDs from both states, excluding runtime metadata + // Get all subBlock IDs from both states, excluding runtime metadata and UI-only elements const allSubBlockIds = [ ...new Set([...Object.keys(currentSubBlocks), ...Object.keys(deployedSubBlocks)]), ] - .filter((id) => !TRIGGER_RUNTIME_SUBBLOCK_IDS.includes(id)) + .filter( + (id) => !TRIGGER_RUNTIME_SUBBLOCK_IDS.includes(id) && !SYSTEM_SUBBLOCK_IDS.includes(id) + ) .sort() // Normalize and compare each subBlock From dc81e30aba810c9bf9fe2ac3282ac58167476b83 Mon Sep 17 00:00:00 2001 From: waleed Date: Fri, 2 Jan 2026 15:16:09 -0800 Subject: [PATCH 4/4] final cleanup --- .../sim/app/api/tools/imap/mailboxes/route.ts | 6 ++ .../sim/lib/webhooks/gmail-polling-service.ts | 10 ++-- apps/sim/lib/webhooks/imap-polling-service.ts | 29 ++++++--- .../lib/webhooks/outlook-polling-service.ts | 10 ++-- apps/sim/triggers/imap/poller.ts | 59 ++++++++++--------- 5 files changed, 71 insertions(+), 43 deletions(-) diff --git a/apps/sim/app/api/tools/imap/mailboxes/route.ts b/apps/sim/app/api/tools/imap/mailboxes/route.ts index 07e36038be..49543a662c 100644 --- a/apps/sim/app/api/tools/imap/mailboxes/route.ts +++ b/apps/sim/app/api/tools/imap/mailboxes/route.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { ImapFlow } from 'imapflow' import { type NextRequest, NextResponse } from 'next/server' +import { getSession } from '@/lib/auth' const logger = createLogger('ImapMailboxesAPI') @@ -14,6 +15,11 @@ interface ImapMailboxRequest { } export async function POST(request: NextRequest) { + const session = await getSession() + if (!session?.user?.id) { + return NextResponse.json({ success: false, message: 'Unauthorized' }, { status: 401 }) + } + try { const body = (await request.json()) as ImapMailboxRequest const { host, port, secure, rejectUnauthorized, username, password } = body diff --git a/apps/sim/lib/webhooks/gmail-polling-service.ts b/apps/sim/lib/webhooks/gmail-polling-service.ts index ee30f82223..cc62c30afc 100644 --- a/apps/sim/lib/webhooks/gmail-polling-service.ts +++ b/apps/sim/lib/webhooks/gmail-polling-service.ts @@ -238,18 +238,20 @@ export async function pollGmailWebhooks() { } for (const webhookData of activeWebhooks) { - const promise = enqueue(webhookData) - .then(() => {}) + const promise: Promise = enqueue(webhookData) .catch((err) => { logger.error('Unexpected error in webhook processing:', err) failureCount++ }) + .finally(() => { + const idx = running.indexOf(promise) + if (idx !== -1) running.splice(idx, 1) + }) running.push(promise) if (running.length >= CONCURRENCY) { - const completedIdx = await Promise.race(running.map((p, i) => p.then(() => i))) - running.splice(completedIdx, 1) + await Promise.race(running) } } diff --git a/apps/sim/lib/webhooks/imap-polling-service.ts b/apps/sim/lib/webhooks/imap-polling-service.ts index 764950e57b..48f568aef4 100644 --- a/apps/sim/lib/webhooks/imap-polling-service.ts +++ b/apps/sim/lib/webhooks/imap-polling-service.ts @@ -190,18 +190,21 @@ export async function pollImapWebhooks() { } for (const webhookData of activeWebhooks) { - const promise = enqueue(webhookData) - .then(() => {}) + const promise: Promise = enqueue(webhookData) .catch((err) => { logger.error('Unexpected error in webhook processing:', err) failureCount++ }) + .finally(() => { + // Self-remove from running array when completed + const idx = running.indexOf(promise) + if (idx !== -1) running.splice(idx, 1) + }) running.push(promise) if (running.length >= CONCURRENCY) { - const completedIdx = await Promise.race(running.map((p, i) => p.then(() => i))) - running.splice(completedIdx, 1) + await Promise.race(running) } } @@ -268,9 +271,19 @@ async function fetchNewEmails(config: ImapWebhookConfig, requestId: string) { const mailbox = await client.mailboxOpen(mailboxPath) logger.debug(`[${requestId}] Opened mailbox: ${mailbox.path}, exists: ${mailbox.exists}`) - const rawCriteria = config.searchCriteria || 'UNSEEN' - let searchCriteria: any = - typeof rawCriteria === 'string' ? { [rawCriteria.toLowerCase()]: true } : rawCriteria + // Parse search criteria - expects JSON object from UI + let searchCriteria: any = { unseen: true } + if (config.searchCriteria) { + if (typeof config.searchCriteria === 'object') { + searchCriteria = config.searchCriteria + } else if (typeof config.searchCriteria === 'string') { + try { + searchCriteria = JSON.parse(config.searchCriteria) + } catch { + logger.warn(`[${requestId}] Invalid search criteria JSON, using default`) + } + } + } const lastUidForMailbox = latestUidByMailbox[mailboxPath] || config.lastProcessedUid @@ -307,7 +320,7 @@ async function fetchNewEmails(config: ImapWebhookConfig, requestId: string) { continue } - messageUids.sort((a, b) => b - a) + messageUids.sort((a, b) => a - b) // Sort ascending to process oldest first const remainingSlots = maxEmails - totalEmailsCollected const uidsToProcess = messageUids.slice(0, remainingSlots) diff --git a/apps/sim/lib/webhooks/outlook-polling-service.ts b/apps/sim/lib/webhooks/outlook-polling-service.ts index 279d9416c7..68f93385ac 100644 --- a/apps/sim/lib/webhooks/outlook-polling-service.ts +++ b/apps/sim/lib/webhooks/outlook-polling-service.ts @@ -276,18 +276,20 @@ export async function pollOutlookWebhooks() { } for (const webhookData of activeWebhooks) { - const promise = enqueue(webhookData) - .then(() => {}) + const promise: Promise = enqueue(webhookData) .catch((err) => { logger.error('Unexpected error in webhook processing:', err) failureCount++ }) + .finally(() => { + const idx = running.indexOf(promise) + if (idx !== -1) running.splice(idx, 1) + }) running.push(promise) if (running.length >= CONCURRENCY) { - const completedIdx = await Promise.race(running.map((p, i) => p.then(() => i))) - running.splice(completedIdx, 1) + await Promise.race(running) } } diff --git a/apps/sim/triggers/imap/poller.ts b/apps/sim/triggers/imap/poller.ts index 4b01a5e2b9..fe0ae078fe 100644 --- a/apps/sim/triggers/imap/poller.ts +++ b/apps/sim/triggers/imap/poller.ts @@ -135,45 +135,50 @@ export const imapPollingTrigger: TriggerConfig = { { id: 'searchCriteria', title: 'Search Criteria', - type: 'short-input', - placeholder: 'UNSEEN', - description: - 'IMAP search criteria (e.g., UNSEEN, FROM "sender@example.com", SUBJECT "report"). Default: UNSEEN', - defaultValue: 'UNSEEN', + type: 'code', + placeholder: '{ "unseen": true }', + description: 'ImapFlow search criteria as JSON object. Default: unseen messages only.', + defaultValue: '{ "unseen": true }', required: false, mode: 'trigger', wandConfig: { enabled: true, maintainHistory: true, - prompt: `You are an expert in IMAP search syntax (RFC 3501). Generate IMAP search criteria based on user descriptions. + generationType: 'json-object', + prompt: `Generate ImapFlow search criteria as a JSON object based on the user's description. -IMAP search keys include: -- ALL - All messages -- UNSEEN / SEEN - Unread/read messages -- FLAGGED / UNFLAGGED - Starred/unstarred -- FROM "string" - Sender contains string -- TO "string" - Recipient contains string -- SUBJECT "string" - Subject contains string -- BODY "string" - Body contains string -- TEXT "string" - Headers or body contains string -- BEFORE date / SINCE date / ON date - Date filters (DD-Mon-YYYY, e.g., 01-Jan-2024) -- LARGER n / SMALLER n - Size in bytes -- HEADER field-name "string" - Custom header search -- NOT criteria - Negate -- OR criteria1 criteria2 - Either matches -- (criteria) - Grouping +Available properties (all are optional, combine as needed): +- "unseen": true - Unread messages +- "seen": true - Read messages +- "flagged": true - Starred/flagged messages +- "answered": true - Replied messages +- "deleted": true - Deleted messages +- "draft": true - Draft messages +- "from": "sender@example.com" - From address contains +- "to": "recipient@example.com" - To address contains +- "cc": "cc@example.com" - CC address contains +- "subject": "keyword" - Subject contains +- "body": "text" - Body contains +- "text": "search" - Headers or body contains +- "since": "2024-01-01" - Emails since date (ISO format) +- "before": "2024-12-31" - Emails before date +- "larger": 10240 - Larger than N bytes +- "smaller": 1048576 - Smaller than N bytes +- "header": { "X-Priority": "1" } - Custom header search +- "or": [{ "from": "a@x.com" }, { "from": "b@x.com" }] - OR conditions +- "not": { "from": "spam@x.com" } - Negate condition -Multiple criteria are AND'd together by default. +Multiple properties are combined with AND. Examples: -- UNSEEN FROM "boss@company.com" -- OR FROM "alice" FROM "bob" -- SINCE 01-Jan-2024 SUBJECT "report" -- NOT SEEN FLAGGED +- Unread from boss: { "unseen": true, "from": "boss@company.com" } +- From Alice or Bob: { "or": [{ "from": "alice@x.com" }, { "from": "bob@x.com" }] } +- Recent with keyword: { "since": "2024-01-01", "subject": "report" } +- Exclude spam: { "unseen": true, "not": { "from": "newsletter@x.com" } } Current criteria: {context} -Return ONLY the IMAP search criteria, no explanations or markdown.`, +Return ONLY valid JSON, no explanations or markdown.`, placeholder: 'Describe what emails you want to filter...', }, },