feat: add dedupe option to transport (#1640)

* feat: add dedupe option to transport

* docs: add transport dedupe info

Author: nekosaur

docs/api.md

@@ -1200,6 +1200,7 @@ For more on transports, how they work, and how to create them see the [`Transpor
 * `worker`: [Worker thread](https://nodejs.org/api/worker_threads.html#worker_threads_new_worker_filename_options) configuration options. Additionally, the `worker` option supports `worker.autoEnd`. If this is set to `false` logs will not be flushed on process exit. It is then up to the developer to call `transport.end()` to flush logs.
 * `targets`: May be specified instead of `target`. Must be an array of transport configurations. Transport configurations include the aforementioned `options` and `target` options plus a `level` option which will send only logs above a specified level to a transport.
 * `pipeline`: May be specified instead of `target`. Must be an array of transport configurations. Transport configurations include the aforementioned `options` and `target` options. All intermediate steps in the pipeline _must_ be `Transform` streams and not `Writable`.
+* `dedupe`: See [pino.multistream options](#pino-multistream)
 
 <a id="pino-multistream"></a>
 

docs/transports.md

@@ -111,6 +111,19 @@ const transport = pino.transport({
 pino(transport)
 ```
 
+It is also possible to use the `dedupe` option to send logs only to the stream with the higher level.
+```js
+const pino = require('pino')
+const transport = pino.transport({
+  targets: [
+    { target: '/absolute/path/to/my-transport.mjs', level: 'error' },
+    { target: 'some-file-transport', options: { destination: '/dev/null' }
+  ],
+  dedupe: true
+})
+pino(transport)
+```
+
 For more details on `pino.transport` see the [API docs for `pino.transport`][pino-transport].
 
 [pino-transport]: /docs/api.md#pino-transport

lib/transport.js

@@ -71,7 +71,7 @@ function flush (stream) {
 }
 
 function transport (fullOptions) {
-  const { pipeline, targets, levels, options = {}, worker = {}, caller = getCallers() } = fullOptions
+  const { pipeline, targets, levels, dedupe, options = {}, worker = {}, caller = getCallers() } = fullOptions
 
   // Backwards compatibility
   const callers = typeof caller === 'string' ? [caller] : caller
@@ -107,6 +107,10 @@ function transport (fullOptions) {
     options.levels = levels
   }
 
+  if (dedupe) {
+    options.dedupe = dedupe
+  }
+
   return buildStream(fixTarget(target), options, worker)
 
   function fixTarget (origin) {

lib/worker.js

@@ -9,7 +9,7 @@ const loadTransportStreamBuilder = require('./transport-stream')
 
 /* istanbul ignore file */
 
-module.exports = async function ({ targets, levels }) {
+module.exports = async function ({ targets, levels, dedupe }) {
   targets = await Promise.all(targets.map(async (t) => {
     const fn = await loadTransportStreamBuilder(t.target)
     const stream = await fn(t.options)
@@ -38,7 +38,7 @@ module.exports = async function ({ targets, levels }) {
   })
 
   function process (stream) {
-    const multi = pino.multistream(targets, { levels })
+    const multi = pino.multistream(targets, { levels, dedupe })
     // TODO manage backpressure
     stream.on('data', function (chunk) {
       const { lastTime, lastMsg, lastObj, lastLevel } = this

pino.d.ts

@@ -262,6 +262,7 @@ declare namespace pino {
     interface TransportMultiOptions<TransportOptions = Record<string, any>> extends TransportBaseOptions<TransportOptions>{
         targets: readonly TransportTargetOptions<TransportOptions>[],
         levels?: Record<string, number>
+        dedupe?: boolean
     }
 
     interface MultiStreamOptions {

test/transport/core.test.js

@@ -201,6 +201,44 @@ test('pino.transport with two files and custom levels', async ({ same, teardown
   })
 })
 
+test('pino.transport with two files and dedupe', async ({ same, teardown }) => {
+  const dest1 = file()
+  const dest2 = file()
+  const transport = pino.transport({
+    dedupe: true,
+    targets: [{
+      level: 'info',
+      target: join(__dirname, '..', 'fixtures', 'to-file-transport.js'),
+      options: { destination: dest1 }
+    }, {
+      level: 'error',
+      target: join(__dirname, '..', 'fixtures', 'to-file-transport.js'),
+      options: { destination: dest2 }
+    }]
+  })
+  teardown(transport.end.bind(transport))
+  const instance = pino(transport)
+  instance.info('hello')
+  instance.error('world')
+  await Promise.all([watchFileCreated(dest1), watchFileCreated(dest2)])
+  const result1 = JSON.parse(await readFile(dest1))
+  delete result1.time
+  same(result1, {
+    pid,
+    hostname,
+    level: 30,
+    msg: 'hello'
+  })
+  const result2 = JSON.parse(await readFile(dest2))
+  delete result2.time
+  same(result2, {
+    pid,
+    hostname,
+    level: 50,
+    msg: 'world'
+  })
+})
+
 test('pino.transport with an array including a pino-pretty destination', async ({ same, match, teardown }) => {
   const dest1 = file()
   const dest2 = file()

test/types/pino-transport.test-d.ts

@@ -120,3 +120,9 @@ pino.transport({
     },
     options: { id: 'abc' }
 })
+
+// Dedupe
+pino.transport({
+    targets: [],
+    dedupe: true,
+})