From a6b9c284c1fdc76da0a793cf91e5e36d898ae85d Mon Sep 17 00:00:00 2001 From: Steren Giannini Date: Fri, 25 May 2018 16:53:12 +0200 Subject: [PATCH 01/14] puppeteer sample (Headless Chrome) --- appengine/headless-chrome/README.md | 12 +++++++ appengine/headless-chrome/app.yaml | 17 +++++++++ appengine/headless-chrome/package.json | 40 +++++++++++++++++++++ appengine/headless-chrome/server.js | 49 ++++++++++++++++++++++++++ 4 files changed, 118 insertions(+) create mode 100644 appengine/headless-chrome/README.md create mode 100644 appengine/headless-chrome/app.yaml create mode 100644 appengine/headless-chrome/package.json create mode 100644 appengine/headless-chrome/server.js diff --git a/appengine/headless-chrome/README.md b/appengine/headless-chrome/README.md new file mode 100644 index 0000000000..3fcb3ac0cc --- /dev/null +++ b/appengine/headless-chrome/README.md @@ -0,0 +1,12 @@ +# Headless Chrome on Google App Engine + +This sample application demonstrates how to use Headless Chrome via the [Puppeteer](https://developers.google.com/web/tools/puppeteer/) module to take screenshots of webpages on [Google App Engine](https://cloud.google.com/appengine) Node.js [standard environment](https://cloud.google.com/appengine/docs/standard/nodejs). + +## Running locally + +* Install dependencies with `npm install` (Headless Chrome will take some time to download) +* Start the application locally using `npm start`. + +## Deploy to App Engine + +* Run `gcloud app deploy` \ No newline at end of file diff --git a/appengine/headless-chrome/app.yaml b/appengine/headless-chrome/app.yaml new file mode 100644 index 0000000000..cd81ee53b4 --- /dev/null +++ b/appengine/headless-chrome/app.yaml @@ -0,0 +1,17 @@ +# Copyright 2018, Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +# [START app_yaml] +runtime: nodejs8 +instance_class: F4_1G +# [END app_yaml] \ No newline at end of file diff --git a/appengine/headless-chrome/package.json b/appengine/headless-chrome/package.json new file mode 100644 index 0000000000..c52c2b8830 --- /dev/null +++ b/appengine/headless-chrome/package.json @@ -0,0 +1,40 @@ +{ + "name": "puppeteer-sample", + "engines": { + "node": "8.x" + }, + "version": "1.0.0", + "description": "An example of taking screenshot with Puppeteer (Headless Chrome) in Node.js on Google App Engine.", + "scripts": { + "start": "node server.js", + "system-test": "repo-tools test app", + "lint": "repo-tools lint", + "pretest": "npm run lint", + "test": "npm run system-test" + }, + "repository": { + "type": "git", + "url": "https://github.com/GoogleCloudPlatform/nodejs-docs-samples.git" + }, + "author": { + "name": "Google LLC" + }, + "license": "Apache-2.0", + "dependencies": { + "express": "^4.16.3", + "puppeteer": "^1.2.0" + }, + "devDependencies": { + "@google-cloud/nodejs-repo-tools": "2.2.1", + "semistandard": "^12.0.1" + }, + "cloud-repo-tools": { + "test": { + "app": { + "msg": "Please provide URL" + } + }, + "requiresKeyFile": true, + "requiresProjectId": true + } +} diff --git a/appengine/headless-chrome/server.js b/appengine/headless-chrome/server.js new file mode 100644 index 0000000000..083d3bd399 --- /dev/null +++ b/appengine/headless-chrome/server.js @@ -0,0 +1,49 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +'use strict'; + +// [START full_sample] +const express = require('express'); +const puppeteer = require('puppeteer'); +const app = express(); + +app.use(async (req, res) => { + const url = req.query.url; + + if (!url) { + return res.send('Please provide URL as GET parameter, for example: ?url=https://example.com'); + } + + // [START browser] + const browser = await puppeteer.launch({ + args: ['--no-sandbox', '--disable-setuid-sandbox'] + }); + const page = await browser.newPage(); + await page.goto(url); + const imageBuffer = await page.screenshot(); + await browser.close(); + // [END browser] + + res.set('Content-Type', 'image/png'); + res.send(imageBuffer); +}); + +const server = app.listen(process.env.PORT || 8080, err => { + if (err) return console.error(err); + const port = server.address().port; + console.info(`App listening on port ${port}`); +}); +// [END sample] From f40b4426aedcaca5d72cfecb1b7c0ffb22b9f208 Mon Sep 17 00:00:00 2001 From: Steren Giannini Date: Fri, 25 May 2018 16:57:44 +0200 Subject: [PATCH 02/14] Fix snippet name --- appengine/headless-chrome/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appengine/headless-chrome/server.js b/appengine/headless-chrome/server.js index 083d3bd399..b1a48cc6bd 100644 --- a/appengine/headless-chrome/server.js +++ b/appengine/headless-chrome/server.js @@ -46,4 +46,4 @@ const server = app.listen(process.env.PORT || 8080, err => { const port = server.address().port; console.info(`App listening on port ${port}`); }); -// [END sample] +// [END full_sample] From 9708edcd6e9416389a9bb26fa132141e570e8184 Mon Sep 17 00:00:00 2001 From: Steren Giannini Date: Fri, 25 May 2018 16:58:06 +0200 Subject: [PATCH 03/14] newline for app.yaml --- appengine/headless-chrome/app.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appengine/headless-chrome/app.yaml b/appengine/headless-chrome/app.yaml index cd81ee53b4..7b9ee1e02b 100644 --- a/appengine/headless-chrome/app.yaml +++ b/appengine/headless-chrome/app.yaml @@ -14,4 +14,4 @@ # [START app_yaml] runtime: nodejs8 instance_class: F4_1G -# [END app_yaml] \ No newline at end of file +# [END app_yaml] From ecf1f2008563d7db33816fd2238692c8c564bcd2 Mon Sep 17 00:00:00 2001 From: Steren Giannini Date: Tue, 29 May 2018 18:02:19 +0200 Subject: [PATCH 04/14] update test to use `server` instead of `app` --- appengine/headless-chrome/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appengine/headless-chrome/package.json b/appengine/headless-chrome/package.json index c52c2b8830..b0e95c88d4 100644 --- a/appengine/headless-chrome/package.json +++ b/appengine/headless-chrome/package.json @@ -7,7 +7,7 @@ "description": "An example of taking screenshot with Puppeteer (Headless Chrome) in Node.js on Google App Engine.", "scripts": { "start": "node server.js", - "system-test": "repo-tools test app", + "system-test": "repo-tools test server", "lint": "repo-tools lint", "pretest": "npm run lint", "test": "npm run system-test" @@ -30,7 +30,7 @@ }, "cloud-repo-tools": { "test": { - "app": { + "server": { "msg": "Please provide URL" } }, From 6c05d34c635c0769e1d9a9f8324f3c138c2e48e0 Mon Sep 17 00:00:00 2001 From: Steren Giannini Date: Tue, 29 May 2018 18:09:20 +0200 Subject: [PATCH 05/14] Use app.js to fix cloud-repo-tools error --- appengine/headless-chrome/{server.js => app.js} | 0 appengine/headless-chrome/package.json | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename appengine/headless-chrome/{server.js => app.js} (100%) diff --git a/appengine/headless-chrome/server.js b/appengine/headless-chrome/app.js similarity index 100% rename from appengine/headless-chrome/server.js rename to appengine/headless-chrome/app.js diff --git a/appengine/headless-chrome/package.json b/appengine/headless-chrome/package.json index b0e95c88d4..d41f86b8b8 100644 --- a/appengine/headless-chrome/package.json +++ b/appengine/headless-chrome/package.json @@ -6,8 +6,8 @@ "version": "1.0.0", "description": "An example of taking screenshot with Puppeteer (Headless Chrome) in Node.js on Google App Engine.", "scripts": { - "start": "node server.js", - "system-test": "repo-tools test server", + "start": "node app.js", + "system-test": "repo-tools test app", "lint": "repo-tools lint", "pretest": "npm run lint", "test": "npm run system-test" @@ -30,7 +30,7 @@ }, "cloud-repo-tools": { "test": { - "server": { + "app": { "msg": "Please provide URL" } }, From 9cae70e597a587c47db2e5675973749bbf4943a9 Mon Sep 17 00:00:00 2001 From: Steren Giannini Date: Fri, 1 Jun 2018 17:14:06 -0700 Subject: [PATCH 06/14] Add test for screenshot --- appengine/headless-chrome/app.js | 2 ++ appengine/headless-chrome/package.json | 6 ++-- appengine/headless-chrome/test/app.test.js | 39 ++++++++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 appengine/headless-chrome/test/app.test.js diff --git a/appengine/headless-chrome/app.js b/appengine/headless-chrome/app.js index b1a48cc6bd..0dd0d40f6d 100644 --- a/appengine/headless-chrome/app.js +++ b/appengine/headless-chrome/app.js @@ -47,3 +47,5 @@ const server = app.listen(process.env.PORT || 8080, err => { console.info(`App listening on port ${port}`); }); // [END full_sample] + +module.exports = app; diff --git a/appengine/headless-chrome/package.json b/appengine/headless-chrome/package.json index d41f86b8b8..919c4e760f 100644 --- a/appengine/headless-chrome/package.json +++ b/appengine/headless-chrome/package.json @@ -1,5 +1,5 @@ { - "name": "puppeteer-sample", + "name": "headless-chrome-sample", "engines": { "node": "8.x" }, @@ -8,9 +8,10 @@ "scripts": { "start": "node app.js", "system-test": "repo-tools test app", + "unit-test": "ava --verbose test/*.test.js", "lint": "repo-tools lint", "pretest": "npm run lint", - "test": "npm run system-test" + "test": "npm run unit-test && npm run system-test" }, "repository": { "type": "git", @@ -26,6 +27,7 @@ }, "devDependencies": { "@google-cloud/nodejs-repo-tools": "2.2.1", + "ava": "^0.25.0", "semistandard": "^12.0.1" }, "cloud-repo-tools": { diff --git a/appengine/headless-chrome/test/app.test.js b/appengine/headless-chrome/test/app.test.js new file mode 100644 index 0000000000..cd4762ed5b --- /dev/null +++ b/appengine/headless-chrome/test/app.test.js @@ -0,0 +1,39 @@ +// Copyright 2017, Google, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// NOTE: +// This app can only be fully tested when deployed, because +// Pub/Sub requires a live endpoint URL to hit. Nevertheless, +// these tests mock it and partially test it locally. + +'use strict'; + +const test = require(`ava`); +const path = require(`path`); +const utils = require(`@google-cloud/nodejs-repo-tools`); + +const cwd = path.join(__dirname, `../`); +const requestObj = utils.getRequest({ cwd: cwd }); + +test.serial.cb(`should return a screenshot`, (t) => { + requestObj + .get(`/?url=https://example.com`) + .send() + .expect(200) + .expect((response) => { + t.is(response.type, `image/png`); + t.true(response.body instanceof Buffer); + t.true(response.body.length > 0); + }) + .end(t.end); +}); From 8638c40ee9c8eabeb924db029e1ac72a5cfa5973 Mon Sep 17 00:00:00 2001 From: Steren Giannini Date: Fri, 1 Jun 2018 18:34:23 -0700 Subject: [PATCH 07/14] Update browser snippet --- appengine/headless-chrome/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appengine/headless-chrome/app.js b/appengine/headless-chrome/app.js index 0dd0d40f6d..036a8bab43 100644 --- a/appengine/headless-chrome/app.js +++ b/appengine/headless-chrome/app.js @@ -31,11 +31,11 @@ app.use(async (req, res) => { const browser = await puppeteer.launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] }); + // [END browser] const page = await browser.newPage(); await page.goto(url); const imageBuffer = await page.screenshot(); await browser.close(); - // [END browser] res.set('Content-Type', 'image/png'); res.send(imageBuffer); From d478a66b6aa417e25dcc8fdc36bad7647cc66053 Mon Sep 17 00:00:00 2001 From: Steren Giannini Date: Mon, 4 Jun 2018 18:08:57 -0700 Subject: [PATCH 08/14] Remove unecessary `()` --- appengine/headless-chrome/test/app.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appengine/headless-chrome/test/app.test.js b/appengine/headless-chrome/test/app.test.js index cd4762ed5b..60280b0832 100644 --- a/appengine/headless-chrome/test/app.test.js +++ b/appengine/headless-chrome/test/app.test.js @@ -25,7 +25,7 @@ const utils = require(`@google-cloud/nodejs-repo-tools`); const cwd = path.join(__dirname, `../`); const requestObj = utils.getRequest({ cwd: cwd }); -test.serial.cb(`should return a screenshot`, (t) => { +test.serial.cb(`should return a screenshot`, t => { requestObj .get(`/?url=https://example.com`) .send() From 1d00f381dd9b45948dc8f9311b1ed0928ed2518b Mon Sep 17 00:00:00 2001 From: Steren Giannini Date: Mon, 4 Jun 2018 18:19:44 -0700 Subject: [PATCH 09/14] Update README --- appengine/headless-chrome/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appengine/headless-chrome/README.md b/appengine/headless-chrome/README.md index 3fcb3ac0cc..4007025678 100644 --- a/appengine/headless-chrome/README.md +++ b/appengine/headless-chrome/README.md @@ -4,9 +4,10 @@ This sample application demonstrates how to use Headless Chrome via the [Puppete ## Running locally -* Install dependencies with `npm install` (Headless Chrome will take some time to download) +* Install dependencies with `npm install` (Headless Chrome will take some time to download), * Start the application locally using `npm start`. ## Deploy to App Engine +* Install the [Google Cloud SDK](https://cloud.google.com/sdk/) admn create a Google Cloud project * Run `gcloud app deploy` \ No newline at end of file From a610c3fefdfb0b424f4d6ff0cf9cb3486f17ba1e Mon Sep 17 00:00:00 2001 From: Steren Giannini Date: Mon, 4 Jun 2018 18:20:41 -0700 Subject: [PATCH 10/14] Fix typo --- appengine/headless-chrome/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appengine/headless-chrome/README.md b/appengine/headless-chrome/README.md index 4007025678..168e73a8d4 100644 --- a/appengine/headless-chrome/README.md +++ b/appengine/headless-chrome/README.md @@ -9,5 +9,5 @@ This sample application demonstrates how to use Headless Chrome via the [Puppete ## Deploy to App Engine -* Install the [Google Cloud SDK](https://cloud.google.com/sdk/) admn create a Google Cloud project +* Install the [Google Cloud SDK](https://cloud.google.com/sdk/) and create a Google Cloud project * Run `gcloud app deploy` \ No newline at end of file From cf566d60093f61ac28aa67197b81ce1b5e75b539 Mon Sep 17 00:00:00 2001 From: Steren Giannini Date: Tue, 5 Jun 2018 08:57:54 -0700 Subject: [PATCH 11/14] Remove unecessary `()` --- appengine/headless-chrome/test/app.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appengine/headless-chrome/test/app.test.js b/appengine/headless-chrome/test/app.test.js index 60280b0832..02dac67deb 100644 --- a/appengine/headless-chrome/test/app.test.js +++ b/appengine/headless-chrome/test/app.test.js @@ -30,7 +30,7 @@ test.serial.cb(`should return a screenshot`, t => { .get(`/?url=https://example.com`) .send() .expect(200) - .expect((response) => { + .expect(response => { t.is(response.type, `image/png`); t.true(response.body instanceof Buffer); t.true(response.body.length > 0); From 2a584edd4a5fdc42f54577d1e6d7b72519d1f383 Mon Sep 17 00:00:00 2001 From: Steren Giannini Date: Tue, 5 Jun 2018 13:01:36 -0700 Subject: [PATCH 12/14] Use app.standard.yaml --- appengine/headless-chrome/README.md | 2 +- appengine/headless-chrome/{app.yaml => app.standard.yaml} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename appengine/headless-chrome/{app.yaml => app.standard.yaml} (100%) diff --git a/appengine/headless-chrome/README.md b/appengine/headless-chrome/README.md index 168e73a8d4..bfccae60c0 100644 --- a/appengine/headless-chrome/README.md +++ b/appengine/headless-chrome/README.md @@ -10,4 +10,4 @@ This sample application demonstrates how to use Headless Chrome via the [Puppete ## Deploy to App Engine * Install the [Google Cloud SDK](https://cloud.google.com/sdk/) and create a Google Cloud project -* Run `gcloud app deploy` \ No newline at end of file +* Run `gcloud app deploy app.standard.yaml` \ No newline at end of file diff --git a/appengine/headless-chrome/app.yaml b/appengine/headless-chrome/app.standard.yaml similarity index 100% rename from appengine/headless-chrome/app.yaml rename to appengine/headless-chrome/app.standard.yaml From e7616bb7ba5a0faf2471af8aa8f6bd3523f78775 Mon Sep 17 00:00:00 2001 From: Steren Giannini Date: Tue, 5 Jun 2018 13:09:03 -0700 Subject: [PATCH 13/14] Update @google-cloud/nodejs-repo-tools dep --- appengine/headless-chrome/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appengine/headless-chrome/package.json b/appengine/headless-chrome/package.json index 919c4e760f..0188f0a5e5 100644 --- a/appengine/headless-chrome/package.json +++ b/appengine/headless-chrome/package.json @@ -26,7 +26,7 @@ "puppeteer": "^1.2.0" }, "devDependencies": { - "@google-cloud/nodejs-repo-tools": "2.2.1", + "@google-cloud/nodejs-repo-tools": "2.3.0", "ava": "^0.25.0", "semistandard": "^12.0.1" }, From dcc642e0b962d81aca4de4d97df612e233d8da07 Mon Sep 17 00:00:00 2001 From: Steren Giannini Date: Tue, 5 Jun 2018 13:15:03 -0700 Subject: [PATCH 14/14] point to GAE instructions for local run and deployment --- appengine/headless-chrome/README.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/appengine/headless-chrome/README.md b/appengine/headless-chrome/README.md index bfccae60c0..36ebb7fc99 100644 --- a/appengine/headless-chrome/README.md +++ b/appengine/headless-chrome/README.md @@ -2,12 +2,6 @@ This sample application demonstrates how to use Headless Chrome via the [Puppeteer](https://developers.google.com/web/tools/puppeteer/) module to take screenshots of webpages on [Google App Engine](https://cloud.google.com/appengine) Node.js [standard environment](https://cloud.google.com/appengine/docs/standard/nodejs). -## Running locally +## Running and deploying -* Install dependencies with `npm install` (Headless Chrome will take some time to download), -* Start the application locally using `npm start`. - -## Deploy to App Engine - -* Install the [Google Cloud SDK](https://cloud.google.com/sdk/) and create a Google Cloud project -* Run `gcloud app deploy app.standard.yaml` \ No newline at end of file +Refer to the [appengine/README.md](../README.md) file for instructions on running and deploying.