Skip to content

feat: recordHookStream #711

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
carlbrugger wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

feat: recordHookStream #711

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3d9dfcc
feat: recordHookStream
carlbrugger Nov 26, 2024
887e29a
update messaging
carlbrugger Nov 26, 2024
7933d87
fix: replace unbounded Set with simple counter to prevent memory leak
carlbrugger Aug 13, 2025
4d1bdc9
docs: update README with accurate streaming documentation
carlbrugger Aug 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions flatfilers/sandbox/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"description": "",
"license": "ISC",
"scripts": {
"dev": "node --enable-source-maps --inspect ../../node_modules/.bin/flatfile develop",
"dev": "node --enable-source-maps --trace-warnings --inspect ../../node_modules/.bin/flatfile develop",
"dev:local": "dotenvx run --env-file=.env.local -- npm run dev",
"dev:staging": "dotenvx run --env-file=.env.staging -- npm run dev",
"dev:prod": "dotenvx run --env-file=.env.prod -- npm run dev",
Expand All @@ -30,6 +30,6 @@
},
"devDependencies": {
"@dotenvx/dotenvx": "^0.39.0",
"flatfile": "^3.6.1"
"flatfile": "^3.8.0"
}
}
146 changes: 42 additions & 104 deletions flatfilers/sandbox/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,115 +1,53 @@
import type { FlatfileListener } from '@flatfile/listener'
import { exportDelimitedZip } from '@flatfile/plugin-export-delimited-zip'
import { JSONExtractor } from '@flatfile/plugin-json-extractor'
import type { FlatfileEvent, FlatfileListener } from '@flatfile/listener'
import {
bulkRecordHook,
type FlatfileRecord,
} from '@flatfile/plugin-record-hook'
import { recordHookStream } from '@flatfile/plugin-record-hook-stream'
import { configureSpace } from '@flatfile/plugin-space-configure'
import { contactsSheet, oneHundredSheet } from '../../playground/src/blueprints'

export default async function (listener: FlatfileListener) {
listener.use(JSONExtractor())
// listener.use(
// bulkRecordHook(
// '**',
// (records: FlatfileRecord[], event: FlatfileEvent) => {
// for (const record of records) {
// // for (const field of oneHundredSheet.fields) {
// // record.addError(field.key, 'Testing streaming')
// // }
// record.addError('one', 'Testing streaming')
// }
// return records
// },
// { debug: true }
// )
// )
listener.use(
exportDelimitedZip({
job: 'export-delimited-zip',
delimiter: '\t',
fileExtension: 'tsv',
debug: true,
})
recordHookStream(
'**',
(records, event: FlatfileEvent) => {
for (const record of records) {
// for (const field of oneHundredSheet.fields) {
// record.err(field.key, 'Testing something')
// }
record.err('one', 'Testing something')
}
return records
},
{ includeMessages: true }
)
)
listener.use(
configureSpace({
workbooks: [
{
name: 'Sandbox',
sheets: [
{
name: 'Sales',
slug: 'sales',
fields: [
{
key: 'date',
type: 'string',
label: 'Date',
},
{
key: 'product',
type: 'string',
label: 'Product',
},
{
key: 'category',
type: 'string',
label: 'Category',
},
{
key: 'region',
type: 'string',
label: 'Region',
},
{
key: 'salesAmount',
type: 'number',
label: 'Sales Amount',
},
],
actions: [
{
operation: 'generateExampleRecords',
label: 'Generate Example Records',
description:
'This custom action code generates example records using Anthropic.',
primary: false,
mode: 'foreground',
},
],
},
{
name: 'Sales 2',
slug: 'sales-2',
fields: [
{
key: 'date',
type: 'string',
label: 'Date',
},
{
key: 'product',
type: 'string',
label: 'Product',
},
{
key: 'category',
type: 'string',
label: 'Category',
},
{
key: 'region',
type: 'string',
label: 'Region',
},
{
key: 'salesAmount',
type: 'number',
label: 'Sales Amount',
},
],
actions: [
{
operation: 'export-external-api',
label: 'Export to External API',
description:
'This custom action code exports the records in the Sales sheet to an external API.',
primary: false,
mode: 'foreground',
},
],
},
],
actions: [
{
operation: 'export-delimited-zip',
label: 'Export to Delimited ZIP',
mode: 'foreground',
},
],
},
// {
// name: 'Sandbox',
// sheets: [
// contactsSheet,
// ],
// },
oneHundredSheet,
],
})
)
Expand Down
Loading