Skip to content

Commit cfbaeb3

Browse files
jmdobryjmdobry
committed
Simplify BigQuery samples according to our standard.
1 parent 8554ebd commit cfbaeb3

File tree

9 files changed

+464
-703
lines changed

9 files changed

+464
-703
lines changed

‎bigquery/datasets.js‎

Lines changed: 30 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -13,79 +13,50 @@
1313

1414
'use strict';
1515

16-
// [START all]
1716
// [START setup]
1817
// By default, the client will authenticate using the service account file
1918
// specified by the GOOGLE_APPLICATION_CREDENTIALS environment variable and use
2019
// the project specified by the GCLOUD_PROJECT environment variable. See
21-
// https://googlecloudplatform.github.io/gcloud-node/#/docs/google-cloud/latest/guides/authentication
20+
// https://googlecloudplatform.github.io/google-cloud-node/#/docs/google-cloud/latest/guides/authentication
2221
var BigQuery = require('@google-cloud/bigquery');
23-
24-
// Instantiate the bigquery client
25-
var bigquery = BigQuery();
2622
// [END setup]
2723

28-
// Control-flow helper library
29-
var async = require('async');
24+
function createDataset (datasetId, callback) {
25+
var bigquery = BigQuery();
26+
var dataset = bigquery.dataset(datasetId);
3027

31-
// [START create_dataset]
32-
/**
33-
* List datasets in the authenticated project.
34-
*
35-
* @param {string} name The name for the new dataset.
36-
* @param {function} callback The callback function.
37-
*/
38-
function createDataset (name, callback) {
39-
var dataset = bigquery.dataset(name);
40-
41-
// See https://googlecloudplatform.github.io/gcloud-node/#/docs/bigquery/latest/bigquery
42-
dataset.create(function (err, dataset) {
28+
// See https://googlecloudplatform.github.io/google-cloud-node/#/docs/bigquery/latest/bigquery/dataset?method=create
29+
dataset.create(function (err, dataset, apiResponse) {
4330
if (err) {
4431
return callback(err);
4532
}
4633

47-
console.log('Created dataset: %s', name);
48-
return callback(null, dataset);
34+
console.log('Created dataset: %s', datasetId);
35+
return callback(null, dataset, apiResponse);
4936
});
5037
}
51-
// [END create_dataset]
52-
53-
// [START delete_dataset]
54-
/**
55-
* List datasets in the authenticated project.
56-
*
57-
* @param {string} name The name for the new dataset.
58-
* @param {function} callback The callback function.
59-
*/
60-
function deleteDataset (name, callback) {
61-
var dataset = bigquery.dataset(name);
62-
63-
// See https://googlecloudplatform.github.io/gcloud-node/#/docs/bigquery/latest/bigquery
38+
39+
function deleteDataset (datasetId, callback) {
40+
var bigquery = BigQuery();
41+
var dataset = bigquery.dataset(datasetId);
42+
43+
// See https://googlecloudplatform.github.io/google-cloud-node/#/docs/bigquery/latest/bigquery/dataset?method=delete
6444
dataset.delete(function (err) {
6545
if (err) {
6646
return callback(err);
6747
}
6848

69-
console.log('Deleted dataset: %s', name);
49+
console.log('Deleted dataset: %s', datasetId);
7050
return callback(null);
7151
});
7252
}
73-
// [END delete_dataset]
74-
75-
// [START list_datasets]
76-
/**
77-
* List datasets in the authenticated project.
78-
*
79-
* @param {string} projectId The project ID to use.
80-
* @param {function} callback The callback function.
81-
*/
53+
8254
function listDatasets (projectId, callback) {
83-
// Instantiate a bigquery client
8455
var bigquery = BigQuery({
8556
projectId: projectId
8657
});
8758

88-
// See https://googlecloudplatform.github.io/gcloud-node/#/docs/bigquery/latest/bigquery
59+
// See https://googlecloudplatform.github.io/google-cloud-node/#/docs/bigquery/latest/bigquery?method=getDatasets
8960
bigquery.getDatasets(function (err, datasets) {
9061
if (err) {
9162
return callback(err);
@@ -95,31 +66,27 @@ function listDatasets (projectId, callback) {
9566
return callback(null, datasets);
9667
});
9768
}
98-
// [END list_datasets]
9969

10070
// [START get_dataset_size]
101-
/**
102-
* Calculate the size of the specified dataset.
103-
*
104-
* @param {string} datasetId The ID of the dataset.
105-
* @param {string} projectId The project ID.
106-
* @param {function} callback The callback function.
107-
*/
71+
// Control-flow helper library
72+
var async = require('async');
73+
10874
function getDatasetSize (datasetId, projectId, callback) {
10975
// Instantiate a bigquery client
11076
var bigquery = BigQuery({
11177
projectId: projectId
11278
});
11379
var dataset = bigquery.dataset(datasetId);
11480

115-
// See https://googlecloudplatform.github.io/gcloud-node/#/docs/bigquery/latest/bigquery/dataset
81+
// See https://googlecloudplatform.github.io/google-cloud-node/#/docs/bigquery/latest/bigquery/dataset?method=getTables
11682
dataset.getTables(function (err, tables) {
11783
if (err) {
11884
return callback(err);
11985
}
12086

12187
return async.map(tables, function (table, cb) {
12288
// Fetch more detailed info for each table
89+
// See https://googlecloudplatform.github.io/google-cloud-node/#/docs/bigquery/latest/bigquery/table?method=get
12390
table.get(function (err, tableInfo) {
12491
if (err) {
12592
return cb(err);
@@ -142,7 +109,6 @@ function getDatasetSize (datasetId, projectId, callback) {
142109
});
143110
}
144111
// [END get_dataset_size]
145-
// [END all]
146112

147113
// The command-line program
148114
var cli = require('yargs');
@@ -161,13 +127,13 @@ var program = module.exports = {
161127

162128
cli
163129
.demand(1)
164-
.command('create <name>', 'Create a new dataset.', {}, function (options) {
165-
program.createDataset(options.name, makeHandler());
130+
.command('create <datasetId>', 'Create a new dataset with the specified ID.', {}, function (options) {
131+
program.createDataset(options.datasetId, makeHandler());
166132
})
167-
.command('delete <datasetId>', 'Delete the specified dataset.', {}, function (options) {
133+
.command('delete <datasetId>', 'Delete the dataset with the specified ID.', {}, function (options) {
168134
program.deleteDataset(options.datasetId, makeHandler());
169135
})
170-
.command('list', 'List datasets in the authenticated project.', {}, function (options) {
136+
.command('list', 'List datasets in the specified project.', {}, function (options) {
171137
program.listDatasets(options.projectId, makeHandler(true, 'id'));
172138
})
173139
.command('size <datasetId>', 'Calculate the size of the specified dataset.', {}, function (options) {
@@ -181,13 +147,13 @@ cli
181147
description: 'Optionally specify the project ID to use.',
182148
global: true
183149
})
184-
.example('node $0 create my_dataset', 'Create a new dataset named "my_dataset".')
185-
.example('node $0 delete my_dataset', 'Delete "my_dataset".')
150+
.example('node $0 create my_dataset', 'Create a new dataset with the ID "my_dataset".')
151+
.example('node $0 delete my_dataset', 'Delete a dataset identified as "my_dataset".')
186152
.example('node $0 list', 'List datasets.')
187-
.example('node $0 list -p bigquery-public-data', 'List datasets in a project other than the authenticated project.')
153+
.example('node $0 list -p bigquery-public-data', 'List datasets in the "bigquery-public-data" project.')
188154
.example('node $0 size my_dataset', 'Calculate the size of "my_dataset".')
189155
.example('node $0 size hacker_news -p bigquery-public-data', 'Calculate the size of "bigquery-public-data:hacker_news".')
190-
.wrap(100)
156+
.wrap(120)
191157
.recommendCommands()
192158
.epilogue('For more information, see https://cloud.google.com/bigquery/docs');
193159

‎bigquery/queries.js‎

Lines changed: 60 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -24,120 +24,97 @@
2424

2525
'use strict';
2626

27-
// [START auth]
27+
// [START setup]
2828
// By default, gcloud will authenticate using the service account file specified
2929
// by the GOOGLE_APPLICATION_CREDENTIALS environment variable and use the
3030
// project specified by the GCLOUD_PROJECT environment variable. See
3131
// https://googlecloudplatform.github.io/gcloud-node/#/docs/guides/authentication
3232
var BigQuery = require('@google-cloud/bigquery');
33+
// [END setup]
3334

34-
// Instantiate the bigquery client
35-
var bigquery = BigQuery();
36-
// [END auth]
37-
38-
// [START sync_query]
39-
/**
40-
* Run a synchronous query.
41-
* @param {string} query The BigQuery query to run, as a string.
42-
* @param {function} callback Callback function to receive query results.
43-
*/
44-
function syncQuery (query, callback) {
45-
if (!query) {
46-
return callback(new Error('"query" is required!'));
47-
}
35+
function syncQuery (sqlQuery, callback) {
36+
var bigquery = BigQuery();
4837

4938
// Construct query object.
5039
// Query options list: https://cloud.google.com/bigquery/docs/reference/v2/jobs/query
51-
var queryObj = {
52-
query: query,
53-
timeoutMs: 10000 // Time out after 10 seconds.
54-
};
55-
56-
// Run query
57-
bigquery.query(queryObj, function (err, rows) {
58-
if (err) {
59-
return callback(err);
60-
}
40+
var options = {
41+
query: sqlQuery,
6142

62-
console.log('SyncQuery: found %d rows!', rows.length);
63-
return callback(null, rows);
64-
});
65-
}
66-
// [END sync_query]
43+
// Time out after 10 seconds.
44+
timeoutMs: 10000,
6745

68-
// [START async_query]
69-
/**
70-
* Run an asynchronous query.
71-
* @param {string} query The BigQuery query to run, as a string.
72-
* @param {function} callback Callback function to receive job data.
73-
*/
74-
function asyncQuery (query, callback) {
75-
if (!query) {
76-
return callback(new Error('"query" is required!'));
77-
}
78-
79-
// Construct query object
80-
// Query options list: https://cloud.google.com/bigquery/docs/reference/v2/jobs/query
81-
var queryObj = {
82-
query: query
46+
// Use standard SQL syntax for queries.
47+
// See: https://cloud.google.com/bigquery/sql-reference/
48+
useLegacySql: false
8349
};
8450

85-
// Submit query asynchronously
86-
bigquery.startQuery(queryObj, function (err, job) {
51+
// Run the query
52+
// See https://googlecloudplatform.github.io/google-cloud-node/#/docs/bigquery/latest/bigquery?method=query
53+
bigquery.query(options, function (err, rows) {
8754
if (err) {
8855
return callback(err);
8956
}
9057

91-
console.log('AsyncQuery: submitted job %s!', job.id);
92-
return callback(null, job);
58+
console.log('Received %d row(s)!', rows.length);
59+
return callback(null, rows);
9360
});
9461
}
9562

96-
/**
97-
* Poll an asynchronous query job for results.
98-
* @param {object} jobId The ID of the BigQuery job to poll.
99-
* @param {function} callback Callback function to receive query results.
100-
*/
101-
function asyncPoll (jobId, callback) {
102-
if (!jobId) {
103-
return callback(new Error('"jobId" is required!'));
104-
}
63+
function waitForJob (jobId, callback) {
64+
var bigquery = BigQuery();
10565

106-
// Check for job status
66+
// See https://googlecloudplatform.github.io/google-cloud-node/#/docs/bigquery/latest/bigquery/job
10767
var job = bigquery.job(jobId);
108-
job.getMetadata(function (err, metadata) {
109-
if (err) {
110-
return callback(err);
111-
}
112-
console.log('Job status: %s', metadata.status.state);
11368

114-
// If job is done, get query results; if not, return an error.
115-
if (metadata.status.state === 'DONE') {
69+
job
70+
.on('error', callback)
71+
.on('complete', function (metadata) {
72+
// The job is done, get query results
73+
// See https://googlecloudplatform.github.io/google-cloud-node/#/docs/bigquery/latest/bigquery/job?method=getQueryResults
11674
job.getQueryResults(function (err, rows) {
11775
if (err) {
11876
return callback(err);
11977
}
12078

121-
console.log('AsyncQuery: polled job %s; got %d rows!', jobId, rows.length);
79+
console.log('Job complete, received %d row(s)!', rows.length);
12280
return callback(null, rows);
12381
});
124-
} else {
125-
return callback(new Error('Job %s is not done', jobId));
82+
});
83+
}
84+
85+
function asyncQuery (sqlQuery, callback) {
86+
var bigquery = BigQuery();
87+
88+
// Construct query object
89+
// Query options list: https://cloud.google.com/bigquery/docs/reference/v2/jobs/query
90+
var options = {
91+
query: sqlQuery,
92+
93+
// Use standard SQL syntax for queries.
94+
// See: https://cloud.google.com/bigquery/sql-reference/
95+
useLegacySql: false
96+
};
97+
98+
// Run the query asynchronously
99+
// See https://googlecloudplatform.github.io/google-cloud-node/#/docs/bigquery/latest/bigquery?method=startQuery
100+
bigquery.startQuery(options, function (err, job) {
101+
if (err) {
102+
return callback(err);
126103
}
104+
105+
console.log('Started job: %s', job.id);
106+
return waitForJob(job.id, callback);
127107
});
128108
}
129-
// [END async_query]
130-
// [END all]
131109

132110
// The command-line program
133111
var cli = require('yargs');
134112
var makeHandler = require('../utils').makeHandler;
135113

136114
var program = module.exports = {
137115
asyncQuery: asyncQuery,
138-
asyncPoll: asyncPoll,
116+
waitForJob: waitForJob,
139117
syncQuery: syncQuery,
140-
bigquery: bigquery,
141118
main: function (args) {
142119
// Run the command-line program
143120
cli.help().strict().parse(args).argv;
@@ -146,19 +123,19 @@ var program = module.exports = {
146123

147124
cli
148125
.demand(1)
149-
.command('sync <query>', 'Run a synchronous query.', {}, function (options) {
150-
program.syncQuery(options.query, makeHandler());
126+
.command('sync <sqlQuery>', 'Run the specified synchronous query.', {}, function (options) {
127+
program.syncQuery(options.sqlQuery, makeHandler());
151128
})
152-
.command('async <query>', 'Start an asynchronous query.', {}, function (options) {
153-
program.asyncQuery(options.query, makeHandler());
129+
.command('async <sqlQuery>', 'Start the specified asynchronous query.', {}, function (options) {
130+
program.asyncQuery(options.sqlQuery, makeHandler());
154131
})
155-
.command('poll <jobId>', 'Get the status of a job.', {}, function (options) {
156-
program.asyncPoll(options.jobId, makeHandler());
132+
.command('wait <jobId>', 'Wait for the specified job to complete and retrieve its results.', {}, function (options) {
133+
program.waitForJob(options.jobId, makeHandler());
157134
})
158-
.example('node $0 sync "SELECT * FROM publicdata:samples.natality LIMIT 5;"')
159-
.example('node $0 async "SELECT * FROM publicdata:samples.natality LIMIT 5;"')
160-
.example('node $0 poll 12345')
161-
.wrap(80)
135+
.example('node $0 sync "SELECT * FROM `publicdata.samples.natality` LIMIT 5;"')
136+
.example('node $0 async "SELECT * FROM `publicdata.samples.natality` LIMIT 5;"')
137+
.example('node $0 wait job_VwckYXnR8yz54GBDMykIGnrc2')
138+
.wrap(120)
162139
.recommendCommands()
163140
.epilogue('For more information, see https://cloud.google.com/bigquery/docs');
164141

0 commit comments

Comments
 (0)