From 8fc56556db1a808c7e10f4b5503f5215cd9b28c4 Mon Sep 17 00:00:00 2001 From: onlyjackfrost Date: Thu, 18 Jan 2024 16:50:51 +0800 Subject: [PATCH 1/2] fix: fix incorrect condition to use restDataStream --- .../extension-driver-duckdb/src/lib/duckdbDataSource.ts | 3 +-- packages/extension-driver-duckdb/src/lib/sqlBuilder.ts | 6 +++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/extension-driver-duckdb/src/lib/duckdbDataSource.ts b/packages/extension-driver-duckdb/src/lib/duckdbDataSource.ts index 28f2c82f..32ce3fe1 100644 --- a/packages/extension-driver-duckdb/src/lib/duckdbDataSource.ts +++ b/packages/extension-driver-duckdb/src/lib/duckdbDataSource.ts @@ -100,7 +100,6 @@ export class DuckDBDataSource extends DataSource { const { db, configurationParameters, ...options } = this.dbMapping.get(profileName)!; const [firstDataSQL, restDataSQL] = buildSQL(sql, operations); - // create new connection for each query const parameters = Array.from(bindParams.values()); this.logRequest(firstDataSQL, parameters, options); @@ -144,7 +143,7 @@ export class DuckDBDataSource extends DataSource { this.push(null); }, }); - if (firstData.length >= chunkSize) { + if (firstData.length >= chunkSize && restDataStream) { readable._read = async function () { if (restDataStream) { for await (const row of restDataStream) { diff --git a/packages/extension-driver-duckdb/src/lib/sqlBuilder.ts b/packages/extension-driver-duckdb/src/lib/sqlBuilder.ts index 03ffcb0b..ee76e557 100644 --- a/packages/extension-driver-duckdb/src/lib/sqlBuilder.ts +++ b/packages/extension-driver-duckdb/src/lib/sqlBuilder.ts @@ -35,7 +35,11 @@ export const buildSQL = ( sql: string, operations: Partial> ): string[] => { - if (isNoOP(operations) && !/^select/.test(sql.toLowerCase())) + if ( + isNoOP(operations) && + !/^select/.test(sql.toLowerCase()) && + !/^with recursive/.test(sql.toLowerCase()) + ) return [sql, '']; let builtSQL = ''; builtSQL += `SELECT * FROM (${removeEndingSemiColon(sql)})`; From 8482e16a18692526393b18af2e14b1d5a52b0156 Mon Sep 17 00:00:00 2001 From: onlyjackfrost Date: Thu, 18 Jan 2024 17:24:08 +0800 Subject: [PATCH 2/2] fix test case --- .../test/restApiCaller.spec.ts | 197 +++++++++--------- 1 file changed, 102 insertions(+), 95 deletions(-) diff --git a/packages/extension-api-caller/test/restApiCaller.spec.ts b/packages/extension-api-caller/test/restApiCaller.spec.ts index bff23980..ae739ef2 100644 --- a/packages/extension-api-caller/test/restApiCaller.spec.ts +++ b/packages/extension-api-caller/test/restApiCaller.spec.ts @@ -1,7 +1,6 @@ import { getTestCompiler } from '@vulcan-sql/test-utility'; import * as path from 'path'; - describe('Test "rest_api" filter', () => { it( 'Should throw error when not pass the "url" argument', @@ -16,9 +15,7 @@ describe('Test "rest_api" filter', () => { await compileAndLoad(sql); // Assert - await expect(execute({})).rejects.toThrow( - 'url is required' - ); + await expect(execute({})).rejects.toThrow('url is required'); }, 50 * 1000 ); @@ -36,19 +33,20 @@ describe('Test "rest_api" filter', () => { stock: 94, brand: 'Apple', category: 'smartphones', - thumbnail: 'https://i.dummyjson.com/data/products/1/thumbnail.jpg', + thumbnail: 'https://cdn.dummyjson.com/product-images/1/thumbnail.jpg', images: [ - 'https://i.dummyjson.com/data/products/1/1.jpg', - 'https://i.dummyjson.com/data/products/1/2.jpg', - 'https://i.dummyjson.com/data/products/1/3.jpg', - 'https://i.dummyjson.com/data/products/1/4.jpg', - 'https://i.dummyjson.com/data/products/1/thumbnail.jpg' - ] + 'https://cdn.dummyjson.com/product-images/1/1.jpg', + 'https://cdn.dummyjson.com/product-images/1/2.jpg', + 'https://cdn.dummyjson.com/product-images/1/3.jpg', + 'https://cdn.dummyjson.com/product-images/1/4.jpg', + 'https://cdn.dummyjson.com/product-images/1/thumbnail.jpg', + ], }); - const { compileAndLoad, execute, getExecutedQueries, getCreatedBinding } = await getTestCompiler({ - extensions: { rest_api: path.join(__dirname, '..', 'src') }, - }); + const { compileAndLoad, execute, getExecutedQueries, getCreatedBinding } = + await getTestCompiler({ + extensions: { rest_api: path.join(__dirname, '..', 'src') }, + }); const sql = `{% set value = { "path": { "id": 1 } } %}SELECT {{ value | rest_api(url='https://dummyjson.com/products/:id') }}`; @@ -64,96 +62,104 @@ describe('Test "rest_api" filter', () => { expect(bindings[0].get('$1')).toEqual(expected); }, 50 * 1000 - ) + ); it( 'Should work with template engine and issue a GET request using the query parameter in value', async () => { const expected = JSON.stringify({ - "products": [ + products: [ { - "id": 1, - "title": "iPhone 9", - "description": "An apple mobile which is nothing like apple", - "price": 549, - "discountPercentage": 12.96, - "rating": 4.69, - "stock": 94, - "brand": "Apple", - "category": "smartphones", - "thumbnail": "https://i.dummyjson.com/data/products/1/thumbnail.jpg", - "images": [ - "https://i.dummyjson.com/data/products/1/1.jpg", - "https://i.dummyjson.com/data/products/1/2.jpg", - "https://i.dummyjson.com/data/products/1/3.jpg", - "https://i.dummyjson.com/data/products/1/4.jpg", - "https://i.dummyjson.com/data/products/1/thumbnail.jpg" - ] + id: 1, + title: 'iPhone 9', + description: 'An apple mobile which is nothing like apple', + price: 549, + discountPercentage: 12.96, + rating: 4.69, + stock: 94, + brand: 'Apple', + category: 'smartphones', + thumbnail: + 'https://cdn.dummyjson.com/product-images/1/thumbnail.jpg', + images: [ + 'https://cdn.dummyjson.com/product-images/1/1.jpg', + 'https://cdn.dummyjson.com/product-images/1/2.jpg', + 'https://cdn.dummyjson.com/product-images/1/3.jpg', + 'https://cdn.dummyjson.com/product-images/1/4.jpg', + 'https://cdn.dummyjson.com/product-images/1/thumbnail.jpg', + ], }, { - "id": 2, - "title": "iPhone X", - "description": "SIM-Free, Model A19211 6.5-inch Super Retina HD display with OLED technology A12 Bionic chip with ...", - "price": 899, - "discountPercentage": 17.94, - "rating": 4.44, - "stock": 34, - "brand": "Apple", - "category": "smartphones", - "thumbnail": "https://i.dummyjson.com/data/products/2/thumbnail.jpg", - "images": [ - "https://i.dummyjson.com/data/products/2/1.jpg", - "https://i.dummyjson.com/data/products/2/2.jpg", - "https://i.dummyjson.com/data/products/2/3.jpg", - "https://i.dummyjson.com/data/products/2/thumbnail.jpg" - ] + id: 2, + title: 'iPhone X', + description: + 'SIM-Free, Model A19211 6.5-inch Super Retina HD display with OLED technology A12 Bionic chip with ...', + price: 899, + discountPercentage: 17.94, + rating: 4.44, + stock: 34, + brand: 'Apple', + category: 'smartphones', + thumbnail: + 'https://cdn.dummyjson.com/product-images/2/thumbnail.jpg', + images: [ + 'https://cdn.dummyjson.com/product-images/2/1.jpg', + 'https://cdn.dummyjson.com/product-images/2/2.jpg', + 'https://cdn.dummyjson.com/product-images/2/3.jpg', + 'https://cdn.dummyjson.com/product-images/2/thumbnail.jpg', + ], }, { - "id": 71, - "title": "Women Shoulder Bags", - "description": "LouisWill Women Shoulder Bags Long Clutches Cross Body Bags Phone Bags PU Leather Hand Bags Large Capacity Card Holders Zipper Coin Purses Fashion Crossbody Bags for Girls Ladies", - "price": 46, - "discountPercentage": 14.65, - "rating": 4.71, - "stock": 17, - "brand": "LouisWill", - "category": "womens-bags", - "thumbnail": "https://i.dummyjson.com/data/products/71/thumbnail.jpg", - "images": [ - "https://i.dummyjson.com/data/products/71/1.jpg", - "https://i.dummyjson.com/data/products/71/2.jpg", - "https://i.dummyjson.com/data/products/71/3.webp", - "https://i.dummyjson.com/data/products/71/thumbnail.jpg" - ] + id: 71, + title: 'Women Shoulder Bags', + description: + 'LouisWill Women Shoulder Bags Long Clutches Cross Body Bags Phone Bags PU Leather Hand Bags Large Capacity Card Holders Zipper Coin Purses Fashion Crossbody Bags for Girls Ladies', + price: 46, + discountPercentage: 14.65, + rating: 4.71, + stock: 17, + brand: 'LouisWill', + category: 'womens-bags', + thumbnail: + 'https://cdn.dummyjson.com/product-images/71/thumbnail.jpg', + images: [ + 'https://cdn.dummyjson.com/product-images/71/1.jpg', + 'https://cdn.dummyjson.com/product-images/71/2.jpg', + 'https://cdn.dummyjson.com/product-images/71/3.webp', + 'https://cdn.dummyjson.com/product-images/71/thumbnail.jpg', + ], }, { - "id": 86, - "title": "Bluetooth Aux", - "description": "Bluetooth Aux Bluetooth Car Aux Car Bluetooth Transmitter Aux Audio Receiver Handfree Car Bluetooth Music Receiver Universal 3.5mm Streaming A2DP Wireless Auto AUX Audio Adapter With Mic For Phone MP3", - "price": 25, - "discountPercentage": 10.56, - "rating": 4.57, - "stock": 22, - "brand": "Car Aux", - "category": "automotive", - "thumbnail": "https://i.dummyjson.com/data/products/86/thumbnail.jpg", - "images": [ - "https://i.dummyjson.com/data/products/86/1.jpg", - "https://i.dummyjson.com/data/products/86/2.webp", - "https://i.dummyjson.com/data/products/86/3.jpg", - "https://i.dummyjson.com/data/products/86/4.jpg", - "https://i.dummyjson.com/data/products/86/thumbnail.jpg" - ] - } + id: 86, + title: 'Bluetooth Aux', + description: + 'Bluetooth Aux Bluetooth Car Aux Car Bluetooth Transmitter Aux Audio Receiver Handfree Car Bluetooth Music Receiver Universal 3.5mm Streaming A2DP Wireless Auto AUX Audio Adapter With Mic For Phone MP3', + price: 25, + discountPercentage: 10.56, + rating: 4.57, + stock: 22, + brand: 'Car Aux', + category: 'automotive', + thumbnail: + 'https://cdn.dummyjson.com/product-images/86/thumbnail.jpg', + images: [ + 'https://cdn.dummyjson.com/product-images/86/1.jpg', + 'https://cdn.dummyjson.com/product-images/86/2.webp', + 'https://cdn.dummyjson.com/product-images/86/3.jpg', + 'https://cdn.dummyjson.com/product-images/86/4.jpg', + 'https://cdn.dummyjson.com/product-images/86/thumbnail.jpg', + ], + }, ], - "total": 4, - "skip": 0, - "limit": 4 + total: 4, + skip: 0, + limit: 4, }); - const { compileAndLoad, execute, getExecutedQueries, getCreatedBinding } = await getTestCompiler({ - extensions: { rest_api: path.join(__dirname, '..', 'src') }, - }); + const { compileAndLoad, execute, getExecutedQueries, getCreatedBinding } = + await getTestCompiler({ + extensions: { rest_api: path.join(__dirname, '..', 'src') }, + }); const sql = `{% set value = { "query": { "q": "phone" } } %}SELECT {{ value | rest_api(url='https://dummyjson.com/products/search') }}`; @@ -169,19 +175,20 @@ describe('Test "rest_api" filter', () => { expect(bindings[0].get('$1')).toEqual(expected); }, 50 * 1000 - ) + ); it( 'Should work with template engine and issue a POST request with body and header in value', async () => { const expected = JSON.stringify({ id: 101, - title: 'BMW Pencil' + title: 'BMW Pencil', }); - const { compileAndLoad, execute, getExecutedQueries, getCreatedBinding } = await getTestCompiler({ - extensions: { rest_api: path.join(__dirname, '..', 'src') }, - }); + const { compileAndLoad, execute, getExecutedQueries, getCreatedBinding } = + await getTestCompiler({ + extensions: { rest_api: path.join(__dirname, '..', 'src') }, + }); const sql = `{% set value = { "body": { "title": "BMW Pencil" }, "headers": { "Content-Type": "application/json" } } %}SELECT {{ value | rest_api(url='https://dummyjson.com/products/add', method='POST') }}`; @@ -197,5 +204,5 @@ describe('Test "rest_api" filter', () => { expect(bindings[0].get('$1')).toEqual(expected); }, 50 * 1000 - ) + ); });