Skip to content

Fix: fix incorrect condition to use restDataStream #332

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
onlyjackfrost merged 2 commits into from
Jan 18, 2024
Merged

Fix: fix incorrect condition to use restDataStream #332

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8fc5655
fix: fix incorrect condition to use restDataStream
onlyjackfrost Jan 18, 2024
8482e16
fix test case
onlyjackfrost Jan 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
197 changes: 102 additions & 95 deletions packages/extension-api-caller/test/restApiCaller.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import { getTestCompiler } from '@vulcan-sql/test-utility';
import * as path from 'path';


describe('Test "rest_api" filter', () => {
it(
'Should throw error when not pass the "url" argument',
Expand All @@ -16,9 +15,7 @@ describe('Test "rest_api" filter', () => {
await compileAndLoad(sql);

// Assert
await expect(execute({})).rejects.toThrow(
'url is required'
);
await expect(execute({})).rejects.toThrow('url is required');
},
50 * 1000
);
Expand All @@ -36,19 +33,20 @@ describe('Test "rest_api" filter', () => {
stock: 94,
brand: 'Apple',
category: 'smartphones',
thumbnail: 'https://i.dummyjson.com/data/products/1/thumbnail.jpg',
thumbnail: 'https://cdn.dummyjson.com/product-images/1/thumbnail.jpg',
images: [
'https://i.dummyjson.com/data/products/1/1.jpg',
'https://i.dummyjson.com/data/products/1/2.jpg',
'https://i.dummyjson.com/data/products/1/3.jpg',
'https://i.dummyjson.com/data/products/1/4.jpg',
'https://i.dummyjson.com/data/products/1/thumbnail.jpg'
]
'https://cdn.dummyjson.com/product-images/1/1.jpg',
'https://cdn.dummyjson.com/product-images/1/2.jpg',
'https://cdn.dummyjson.com/product-images/1/3.jpg',
'https://cdn.dummyjson.com/product-images/1/4.jpg',
'https://cdn.dummyjson.com/product-images/1/thumbnail.jpg',
],
});

const { compileAndLoad, execute, getExecutedQueries, getCreatedBinding } = await getTestCompiler({
extensions: { rest_api: path.join(__dirname, '..', 'src') },
});
const { compileAndLoad, execute, getExecutedQueries, getCreatedBinding } =
await getTestCompiler({
extensions: { rest_api: path.join(__dirname, '..', 'src') },
});

const sql = `{% set value = { "path": { "id": 1 } } %}SELECT {{ value | rest_api(url='https://dummyjson.com/products/:id') }}`;

Expand All @@ -64,96 +62,104 @@ describe('Test "rest_api" filter', () => {
expect(bindings[0].get('$1')).toEqual(expected);
},
50 * 1000
)
);

it(
'Should work with template engine and issue a GET request using the query parameter in value',
async () => {
const expected = JSON.stringify({
"products": [
products: [
{
"id": 1,
"title": "iPhone 9",
"description": "An apple mobile which is nothing like apple",
"price": 549,
"discountPercentage": 12.96,
"rating": 4.69,
"stock": 94,
"brand": "Apple",
"category": "smartphones",
"thumbnail": "https://i.dummyjson.com/data/products/1/thumbnail.jpg",
"images": [
"https://i.dummyjson.com/data/products/1/1.jpg",
"https://i.dummyjson.com/data/products/1/2.jpg",
"https://i.dummyjson.com/data/products/1/3.jpg",
"https://i.dummyjson.com/data/products/1/4.jpg",
"https://i.dummyjson.com/data/products/1/thumbnail.jpg"
]
id: 1,
title: 'iPhone 9',
description: 'An apple mobile which is nothing like apple',
price: 549,
discountPercentage: 12.96,
rating: 4.69,
stock: 94,
brand: 'Apple',
category: 'smartphones',
thumbnail:
'https://cdn.dummyjson.com/product-images/1/thumbnail.jpg',
images: [
'https://cdn.dummyjson.com/product-images/1/1.jpg',
'https://cdn.dummyjson.com/product-images/1/2.jpg',
'https://cdn.dummyjson.com/product-images/1/3.jpg',
'https://cdn.dummyjson.com/product-images/1/4.jpg',
'https://cdn.dummyjson.com/product-images/1/thumbnail.jpg',
],
},
{
"id": 2,
"title": "iPhone X",
"description": "SIM-Free, Model A19211 6.5-inch Super Retina HD display with OLED technology A12 Bionic chip with ...",
"price": 899,
"discountPercentage": 17.94,
"rating": 4.44,
"stock": 34,
"brand": "Apple",
"category": "smartphones",
"thumbnail": "https://i.dummyjson.com/data/products/2/thumbnail.jpg",
"images": [
"https://i.dummyjson.com/data/products/2/1.jpg",
"https://i.dummyjson.com/data/products/2/2.jpg",
"https://i.dummyjson.com/data/products/2/3.jpg",
"https://i.dummyjson.com/data/products/2/thumbnail.jpg"
]
id: 2,
title: 'iPhone X',
description:
'SIM-Free, Model A19211 6.5-inch Super Retina HD display with OLED technology A12 Bionic chip with ...',
price: 899,
discountPercentage: 17.94,
rating: 4.44,
stock: 34,
brand: 'Apple',
category: 'smartphones',
thumbnail:
'https://cdn.dummyjson.com/product-images/2/thumbnail.jpg',
images: [
'https://cdn.dummyjson.com/product-images/2/1.jpg',
'https://cdn.dummyjson.com/product-images/2/2.jpg',
'https://cdn.dummyjson.com/product-images/2/3.jpg',
'https://cdn.dummyjson.com/product-images/2/thumbnail.jpg',
],
},
{
"id": 71,
"title": "Women Shoulder Bags",
"description": "LouisWill Women Shoulder Bags Long Clutches Cross Body Bags Phone Bags PU Leather Hand Bags Large Capacity Card Holders Zipper Coin Purses Fashion Crossbody Bags for Girls Ladies",
"price": 46,
"discountPercentage": 14.65,
"rating": 4.71,
"stock": 17,
"brand": "LouisWill",
"category": "womens-bags",
"thumbnail": "https://i.dummyjson.com/data/products/71/thumbnail.jpg",
"images": [
"https://i.dummyjson.com/data/products/71/1.jpg",
"https://i.dummyjson.com/data/products/71/2.jpg",
"https://i.dummyjson.com/data/products/71/3.webp",
"https://i.dummyjson.com/data/products/71/thumbnail.jpg"
]
id: 71,
title: 'Women Shoulder Bags',
description:
'LouisWill Women Shoulder Bags Long Clutches Cross Body Bags Phone Bags PU Leather Hand Bags Large Capacity Card Holders Zipper Coin Purses Fashion Crossbody Bags for Girls Ladies',
price: 46,
discountPercentage: 14.65,
rating: 4.71,
stock: 17,
brand: 'LouisWill',
category: 'womens-bags',
thumbnail:
'https://cdn.dummyjson.com/product-images/71/thumbnail.jpg',
images: [
'https://cdn.dummyjson.com/product-images/71/1.jpg',
'https://cdn.dummyjson.com/product-images/71/2.jpg',
'https://cdn.dummyjson.com/product-images/71/3.webp',
'https://cdn.dummyjson.com/product-images/71/thumbnail.jpg',
],
},
{
"id": 86,
"title": "Bluetooth Aux",
"description": "Bluetooth Aux Bluetooth Car Aux Car Bluetooth Transmitter Aux Audio Receiver Handfree Car Bluetooth Music Receiver Universal 3.5mm Streaming A2DP Wireless Auto AUX Audio Adapter With Mic For Phone MP3",
"price": 25,
"discountPercentage": 10.56,
"rating": 4.57,
"stock": 22,
"brand": "Car Aux",
"category": "automotive",
"thumbnail": "https://i.dummyjson.com/data/products/86/thumbnail.jpg",
"images": [
"https://i.dummyjson.com/data/products/86/1.jpg",
"https://i.dummyjson.com/data/products/86/2.webp",
"https://i.dummyjson.com/data/products/86/3.jpg",
"https://i.dummyjson.com/data/products/86/4.jpg",
"https://i.dummyjson.com/data/products/86/thumbnail.jpg"
]
}
id: 86,
title: 'Bluetooth Aux',
description:
'Bluetooth Aux Bluetooth Car Aux Car Bluetooth Transmitter Aux Audio Receiver Handfree Car Bluetooth Music Receiver Universal 3.5mm Streaming A2DP Wireless Auto AUX Audio Adapter With Mic For Phone MP3',
price: 25,
discountPercentage: 10.56,
rating: 4.57,
stock: 22,
brand: 'Car Aux',
category: 'automotive',
thumbnail:
'https://cdn.dummyjson.com/product-images/86/thumbnail.jpg',
images: [
'https://cdn.dummyjson.com/product-images/86/1.jpg',
'https://cdn.dummyjson.com/product-images/86/2.webp',
'https://cdn.dummyjson.com/product-images/86/3.jpg',
'https://cdn.dummyjson.com/product-images/86/4.jpg',
'https://cdn.dummyjson.com/product-images/86/thumbnail.jpg',
],
},
],
"total": 4,
"skip": 0,
"limit": 4
total: 4,
skip: 0,
limit: 4,
});

const { compileAndLoad, execute, getExecutedQueries, getCreatedBinding } = await getTestCompiler({
extensions: { rest_api: path.join(__dirname, '..', 'src') },
});
const { compileAndLoad, execute, getExecutedQueries, getCreatedBinding } =
await getTestCompiler({
extensions: { rest_api: path.join(__dirname, '..', 'src') },
});

const sql = `{% set value = { "query": { "q": "phone" } } %}SELECT {{ value | rest_api(url='https://dummyjson.com/products/search') }}`;

Expand All @@ -169,19 +175,20 @@ describe('Test "rest_api" filter', () => {
expect(bindings[0].get('$1')).toEqual(expected);
},
50 * 1000
)
);

it(
'Should work with template engine and issue a POST request with body and header in value',
async () => {
const expected = JSON.stringify({
id: 101,
title: 'BMW Pencil'
title: 'BMW Pencil',
});

const { compileAndLoad, execute, getExecutedQueries, getCreatedBinding } = await getTestCompiler({
extensions: { rest_api: path.join(__dirname, '..', 'src') },
});
const { compileAndLoad, execute, getExecutedQueries, getCreatedBinding } =
await getTestCompiler({
extensions: { rest_api: path.join(__dirname, '..', 'src') },
});

const sql = `{% set value = { "body": { "title": "BMW Pencil" }, "headers": { "Content-Type": "application/json" } } %}SELECT {{ value | rest_api(url='https://dummyjson.com/products/add', method='POST') }}`;

Expand All @@ -197,5 +204,5 @@ describe('Test "rest_api" filter', () => {
expect(bindings[0].get('$1')).toEqual(expected);
},
50 * 1000
)
);
});
3 changes: 1 addition & 2 deletions packages/extension-driver-duckdb/src/lib/duckdbDataSource.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,6 @@ export class DuckDBDataSource extends DataSource<any, DuckDBOptions> {
const { db, configurationParameters, ...options } =
this.dbMapping.get(profileName)!;
const [firstDataSQL, restDataSQL] = buildSQL(sql, operations);

// create new connection for each query
const parameters = Array.from(bindParams.values());
this.logRequest(firstDataSQL, parameters, options);
Expand Down Expand Up @@ -144,7 +143,7 @@ export class DuckDBDataSource extends DataSource<any, DuckDBOptions> {
this.push(null);
},
});
if (firstData.length >= chunkSize) {
if (firstData.length >= chunkSize && restDataStream) {
readable._read = async function () {
if (restDataStream) {
for await (const row of restDataStream) {
Expand Down
6 changes: 5 additions & 1 deletion packages/extension-driver-duckdb/src/lib/sqlBuilder.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,11 @@ export const buildSQL = (
sql: string,
operations: Partial<Parameterized<SQLClauseOperation>>
): string[] => {
if (isNoOP(operations) && !/^select/.test(sql.toLowerCase()))
if (
isNoOP(operations) &&
!/^select/.test(sql.toLowerCase()) &&
!/^with recursive/.test(sql.toLowerCase())
)
return [sql, ''];
let builtSQL = '';
builtSQL += `SELECT * FROM (${removeEndingSemiColon(sql)})`;
Expand Down