fix(webflow): paginate collection item selectors

Author: Bill Leoutsakos

apps/docs/content/docs/integrations/webflow.mdx

@@ -36,7 +36,7 @@ Integrates Webflow CMS into the workflow. Can create, get, list, update, or dele
 
 ### Webflow List Items
 
-List all items from a Webflow CMS collection
+List items from a Webflow CMS collection
 
 #### Input
 
@@ -64,6 +64,7 @@ List all items from a Webflow CMS collection
 |   ↳ `itemCount` | number | Number of items returned |
 |   ↳ `offset` | number | Pagination offset |
 |   ↳ `limit` | number | Maximum items per page |
+|   ↳ `total` | number | Total number of matching items |
 
 ### Webflow Get Item
 

apps/sim/lib/selectors/manifest.ts

@@ -304,7 +304,9 @@ export const selectorManifest = {
   }),
   'webflow.items': providerSelector(['collectionId'], {
     readiness: { all: ['oauthCredential', 'collectionId'] },
+    listMode: 'paginated',
     search: true,
+    detail: true,
     staleTime: SEARCH_SELECTOR_STALE_TIME,
   }),
   'cloudwatch.logGroups': rawProviderSelector(

apps/sim/lib/selectors/server/providers/webflow.test.ts

@@ -0,0 +1,106 @@
+/**
+ * @vitest-environment node
+ */
+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({
+  mockFetch: vi.fn(),
+  mockResolveSelectorOAuthAccessToken: vi.fn(),
+}))
+
+vi.mock('@/lib/selectors/server/credentials', () => ({
+  resolveSelectorOAuthAccessToken: mockResolveSelectorOAuthAccessToken,
+}))
+
+import { createSelectorProtectedValues } from '@/lib/selectors/server/protected-values'
+import { webflowSelectorAttachments } from '@/lib/selectors/server/providers/webflow'
+import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types'
+
+const collectionId = '680000000000000000000001'
+
+function args(request: ExecuteServerSelectorArgs['request']): ExecuteServerSelectorArgs {
+  return {
+    selectorKey: 'webflow.items',
+    context: { oauthCredential: 'credential-1', collectionId },
+    request,
+    scope: { kind: 'workspace', workspaceId: 'workspace-1' },
+    workspaceId: 'workspace-1',
+    principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' },
+    requesterUserId: 'user-1',
+    credential: { suppliedId: 'credential-1' },
+    references: new Map(),
+    protectedValues: createSelectorProtectedValues(),
+  }
+}
+
+describe('Webflow server selector adapter', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    vi.stubGlobal('fetch', mockFetch)
+    mockResolveSelectorOAuthAccessToken.mockResolvedValue('server-only-token')
+  })
+
+  afterAll(() => vi.unstubAllGlobals())
+
+  it('fetches one searched item page beyond the old 50-page boundary', async () => {
+    const itemId = '680000000000000000001389'
+    mockFetch.mockResolvedValueOnce(
+      new Response(
+        JSON.stringify({
+          items: [{ id: itemId, fieldData: { name: 'Needle beyond fifty' } }],
+          pagination: { limit: 100, offset: 5000, total: 5002 },
+        }),
+        { status: 200 }
+      )
+    )
+
+    await expect(
+      webflowSelectorAttachments['webflow.items'].execute(
+        args({ kind: 'list', search: '  Needle  ', cursor: '5000' })
+      )
+    ).resolves.toEqual({
+      kind: 'list',
+      items: [{ id: itemId, label: 'Needle beyond fifty' }],
+      nextCursor: '5001',
+    })
+
+    const url = new URL(String(mockFetch.mock.calls[0]?.[0]))
+    expect(url.pathname).toBe(`/v2/collections/${collectionId}/items`)
+    expect(url.searchParams.get('limit')).toBe('100')
+    expect(url.searchParams.get('offset')).toBe('5000')
+    expect(url.searchParams.get('filter[name][contains]')).toBe('Needle')
+    expect(mockFetch).toHaveBeenCalledTimes(1)
+  })
+
+  it('hydrates saved items directly and treats a missing item as absent', async () => {
+    const itemId = '680000000000000000001389'
+    const missingItemId = '680000000000000000001390'
+    mockFetch
+      .mockResolvedValueOnce(
+        new Response(JSON.stringify({ id: itemId, fieldData: { title: 'Saved item title' } }), {
+          status: 200,
+        })
+      )
+      .mockResolvedValueOnce(new Response(null, { status: 404 }))
+
+    await expect(
+      webflowSelectorAttachments['webflow.items'].execute(args({ kind: 'detail', id: itemId }))
+    ).resolves.toEqual({
+      kind: 'detail',
+      item: { id: itemId, label: 'Saved item title' },
+    })
+    await expect(
+      webflowSelectorAttachments['webflow.items'].execute(
+        args({ kind: 'detail', id: missingItemId })
+      )
+    ).resolves.toEqual({ kind: 'detail', item: null })
+
+    expect(String(mockFetch.mock.calls[0]?.[0])).toBe(
+      `https://api.webflow.com/v2/collections/${collectionId}/items/${itemId}`
+    )
+    expect(String(mockFetch.mock.calls[1]?.[0])).toBe(
+      `https://api.webflow.com/v2/collections/${collectionId}/items/${missingItemId}`
+    )
+    expect(mockFetch).toHaveBeenCalledTimes(2)
+  })
+})

apps/sim/lib/selectors/server/providers/webflow.ts

@@ -7,19 +7,41 @@ import {
   SelectorOptionsUnavailableError,
 } from '@/lib/selectors/server/errors'
 import { flatSelectorResult } from '@/lib/selectors/server/providers/flat-results'
-import { fetchProviderJson } from '@/lib/selectors/server/providers/provider-http'
+import {
+  fetchProviderJson,
+  fetchProviderJsonWithStatus,
+} from '@/lib/selectors/server/providers/provider-http'
 import type {
   ExecuteServerSelectorArgs,
   ServerSelectorAttachmentMap,
 } from '@/lib/selectors/server/types'
+import { detailSelectorResult, listSelectorResult } from '@/lib/selectors/server/types'
 import type { SafeSelectorOption } from '@/lib/selectors/types'
 
 type WebflowSelectorKey = Extract<
   ServerSelectorKey,
   'webflow.sites' | 'webflow.collections' | 'webflow.items'
 >
 
-const WEBFLOW_MAX_ITEM_PAGES = 50
+const WEBFLOW_ITEM_PAGE_SIZE = 100
+
+interface WebflowItem {
+  id?: unknown
+  fieldData?: {
+    name?: unknown
+    title?: unknown
+    slug?: unknown
+  }
+}
+
+interface WebflowItemPage {
+  items?: unknown
+  pagination?: {
+    limit?: unknown
+    offset?: unknown
+    total?: unknown
+  }
+}
 
 const credential = {
   kind: 'stored',
@@ -43,6 +65,34 @@ function requireWebflowId(value: string | undefined, name: string): string {
   return validation.sanitized ?? value
 }
 
+function parseItemOffset(cursor: string | undefined): number {
+  if (cursor === undefined) return 0
+  if (!/^(0|[1-9]\d*)$/.test(cursor)) throw new SelectorContextUnavailableError()
+  const offset = Number(cursor)
+  if (!Number.isSafeInteger(offset)) throw new SelectorContextUnavailableError()
+  return offset
+}
+
+function projectItem(item: WebflowItem): SafeSelectorOption | null {
+  if (typeof item.id !== 'string' || !item.id) return null
+  const { name, title, slug } = item.fieldData ?? {}
+  const label = [name, title, slug].find(
+    (candidate): candidate is string => typeof candidate === 'string' && candidate.length > 0
+  )
+  return { id: item.id, label: label ?? item.id }
+}
+
+function requirePaginationNumber(value: unknown, options: { positive?: boolean } = {}): number {
+  if (
+    !Number.isSafeInteger(value) ||
+    (value as number) < 0 ||
+    (options.positive && (value as number) === 0)
+  ) {
+    throw new SelectorOptionsUnavailableError()
+  }
+  return value as number
+}
+
 async function listSites(args: ExecuteServerSelectorArgs): Promise<SafeSelectorOption[]> {
   const token = await tokenFor(args)
   const data = await fetchProviderJson<{
@@ -74,69 +124,94 @@ async function listCollections(args: ExecuteServerSelectorArgs): Promise<SafeSel
   }))
 }
 
-async function listItems(
-  args: ExecuteServerSelectorArgs
-): Promise<{ items: SafeSelectorOption[]; truncated: boolean }> {
+async function listItems(args: ExecuteServerSelectorArgs) {
+  if (args.request.kind !== 'list') throw new SelectorContextUnavailableError()
+  const offset = parseItemOffset(args.request.cursor)
   const collectionId = requireWebflowId(args.context.collectionId, 'collectionId')
   const token = await tokenFor(args)
-  const items: Array<{
-    id: string
-    fieldData?: { name?: string; title?: string; slug?: string }
-  }> = []
-  let offset = 0
-  let truncated = false
-  for (let page = 0; page < WEBFLOW_MAX_ITEM_PAGES; page++) {
-    const url = new URL(
-      `https://api.webflow.com/v2/collections/${encodeURIComponent(collectionId)}/items`
-    )
-    url.searchParams.set('limit', '100')
-    url.searchParams.set('offset', String(offset))
-    const data = await fetchProviderJson<{
-      items?: typeof items
-      pagination?: { total?: number }
-    }>(url, {
-      headers: { Authorization: `Bearer ${token}`, accept: 'application/json' },
-      signal: args.signal,
-      redirect: 'error',
-    })
-    const pageItems = data.items ?? []
-    const reportedTotal = data.pagination?.total
-    if (reportedTotal !== undefined && (!Number.isInteger(reportedTotal) || reportedTotal < 0)) {
-      throw new SelectorOptionsUnavailableError()
-    }
-    items.push(...pageItems)
-    offset += pageItems.length
-    if (reportedTotal !== undefined && items.length >= reportedTotal) {
-      break
-    }
-    if (pageItems.length === 0) {
-      if (reportedTotal !== undefined && items.length < reportedTotal) truncated = true
-      break
-    }
-    if (page === WEBFLOW_MAX_ITEM_PAGES - 1) truncated = true
+  const url = new URL(
+    `https://api.webflow.com/v2/collections/${encodeURIComponent(collectionId)}/items`
+  )
+  url.searchParams.set('limit', String(WEBFLOW_ITEM_PAGE_SIZE))
+  url.searchParams.set('offset', String(offset))
+  const search = args.request.search?.trim()
+  if (search) url.searchParams.set('filter[name][contains]', search)
+
+  const data = await fetchProviderJson<WebflowItemPage>(url, {
+    headers: { Authorization: `Bearer ${token}`, accept: 'application/json' },
+    signal: args.signal,
+    redirect: 'error',
+  })
+  if (
+    !data ||
+    typeof data !== 'object' ||
+    !Array.isArray(data.items) ||
+    !data.pagination ||
+    typeof data.pagination !== 'object' ||
+    Array.isArray(data.pagination)
+  ) {
+    throw new SelectorOptionsUnavailableError()
   }
 
-  const search = args.request.kind === 'list' ? args.request.search?.toLowerCase() : undefined
-  return {
-    items: items.flatMap((item) => {
-      if (!item.id) return []
-      const label = item.fieldData?.name || item.fieldData?.title || item.fieldData?.slug || item.id
-      return search && !label.toLowerCase().includes(search) ? [] : [{ id: item.id, label }]
-    }),
-    truncated,
+  const reportedLimit = requirePaginationNumber(data.pagination.limit, { positive: true })
+  const reportedOffset = requirePaginationNumber(data.pagination.offset)
+  const reportedTotal = requirePaginationNumber(data.pagination.total)
+  if (
+    reportedLimit > WEBFLOW_ITEM_PAGE_SIZE ||
+    reportedOffset !== offset ||
+    data.items.length > reportedLimit
+  ) {
+    throw new SelectorOptionsUnavailableError()
+  }
+  if (data.items.length === 0 && reportedOffset < reportedTotal) {
+    throw new SelectorOptionsUnavailableError()
   }
-}
 
-async function executeItems(args: ExecuteServerSelectorArgs) {
-  const { items, truncated } = await listItems(args)
-  return flatSelectorResult(
-    args.request,
+  const nextOffset = reportedOffset + data.items.length
+  if (
+    !Number.isSafeInteger(nextOffset) ||
+    (nextOffset <= reportedOffset && data.items.length > 0)
+  ) {
+    throw new SelectorOptionsUnavailableError()
+  }
+  const items = data.items.flatMap((item) => {
+    if (!item || typeof item !== 'object') return []
+    const option = projectItem(item as WebflowItem)
+    return option ? [option] : []
+  })
+  return listSelectorResult(
     items,
-    false,
-    truncated ? { truncated: { reason: 'provider-cap', pages: WEBFLOW_MAX_ITEM_PAGES } } : undefined
+    data.items.length > 0 && nextOffset < reportedTotal ? String(nextOffset) : undefined
   )
 }
 
+async function getItem(args: ExecuteServerSelectorArgs) {
+  if (args.request.kind !== 'detail') throw new SelectorContextUnavailableError()
+  const collectionId = requireWebflowId(args.context.collectionId, 'collectionId')
+  const itemId = requireWebflowId(args.request.id, 'itemId')
+  const token = await tokenFor(args)
+  const result = await fetchProviderJsonWithStatus<WebflowItem>(
+    `https://api.webflow.com/v2/collections/${encodeURIComponent(collectionId)}/items/${encodeURIComponent(itemId)}`,
+    {
+      headers: { Authorization: `Bearer ${token}`, accept: 'application/json' },
+      signal: args.signal,
+      redirect: 'error',
+    },
+    { passthroughStatuses: [404] }
+  )
+  if (!result.ok) return detailSelectorResult(null)
+  if (!result.data || typeof result.data !== 'object') {
+    throw new SelectorOptionsUnavailableError()
+  }
+  const item = projectItem(result.data)
+  if (!item || item.id !== itemId) throw new SelectorOptionsUnavailableError()
+  return detailSelectorResult(item)
+}
+
+function executeItems(args: ExecuteServerSelectorArgs) {
+  return args.request.kind === 'detail' ? getItem(args) : listItems(args)
+}
+
 export const webflowSelectorAttachments = {
   'webflow.sites': {
     credential,

apps/sim/tools/generated/tool-metadata.ts

@@ -8,7 +8,7 @@ import {
 export const webflowListItemsTool: ToolConfig<WebflowListItemsParams, WebflowListItemsResponse> = {
   id: 'webflow_list_items',
   name: 'Webflow List Items',
-  description: 'List all items from a Webflow CMS collection',
+  description: 'List items from a Webflow CMS collection',
   version: '1.0.0',
 
   oauth: {
@@ -78,8 +78,9 @@ export const webflowListItemsTool: ToolConfig<WebflowListItemsParams, WebflowLis
         items: data.items || [],
         metadata: {
           itemCount: (data.items || []).length,
-          offset: data.offset,
-          limit: data.limit,
+          offset: data.pagination?.offset,
+          limit: data.pagination?.limit,
+          total: data.pagination?.total,
         },
       },
     }

apps/sim/tools/generated/tool-outputs.ts

@@ -32,6 +32,7 @@ export const WEBFLOW_LIST_METADATA_OUTPUT_PROPERTIES = {
   itemCount: { type: 'number', description: 'Number of items returned' },
   offset: { type: 'number', description: 'Pagination offset', optional: true },
   limit: { type: 'number', description: 'Maximum items per page', optional: true },
+  total: { type: 'number', description: 'Total number of matching items', optional: true },
 } as const satisfies Record<string, OutputProperty>
 
 interface WebflowBaseParams {
@@ -51,6 +52,7 @@ interface WebflowListItemsOutput {
     itemCount: number
     offset?: number
     limit?: number
+    total?: number
   }
 }