[Filestream] New Input package

.github/CODEOWNERS

@@ -55,6 +55,7 @@
 /packages/elasticsearch @elastic/infra-monitoring-ui
 /packages/etcd @elastic/integrations
 /packages/f5 @elastic/security-external-integrations
+/packages/filestream @elastic/obs-service-integrations
 /packages/fim @elastic/security-external-integrations
 /packages/fireeye @elastic/security-external-integrations
 /packages/fleet_server @elastic/elastic-agent-control-plane

packages/filestream/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: [email protected]

packages/filestream/_dev/build/docs/README.md

@@ -0,0 +1,29 @@
+# Custom Filestream Log integration
+
+The `filestream` custom input is used to read lines from active log files. It is the
+new, improved alternative to the `log` input. It comes with various improvements
+to the existing input:
+
+1. Checking of `close_*` options happens out of band. Thus, if an output is blocked,
+Elastic Agent can close the reader and avoid keeping too many files open.
+
+2. Detailed metrics are available for all files that match the `paths` configuration
+regardless of the `harvester_limit`. This way, you can keep track of all files,
+even ones that are not actively read.
+
+3. The order of `parsers` is configurable. So it is possible to parse JSON lines and then
+aggregate the contents into a multiline event.
+
+4. Some position updates and metadata changes no longer depend on the publishing pipeline.
+If the pipeline is blocked some changes are still applied to the registry.
+
+5. Only the most recent updates are serialized to the registry. In contrast, the `log` input
+has to serialize the complete registry on each ACK from the outputs. This makes the registry updates
+much quicker with this input.
+
+6. The input ensures that only offsets updates are written to the registry append only log.
+The `log` writes the complete file state.
+
+7. Stale entries can be removed from the registry, even if there is no active input.
+
+More information can be found on the [Filestream documentation page](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-filestream.html)

packages/filestream/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,8 @@
+version: '2.3'
+services:
+  filestream-logfile:
+    image: alpine
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+      - ${SERVICE_LOGS_DIR}:/var/log
+    command: /bin/sh -c "cp /sample_logs/* /var/log/"

packages/filestream/_dev/deploy/docker/sample_logs/test-filestream.log

@@ -0,0 +1 @@
+<188>date=2020-04-23 time=12:17:48 devname="testswitch1" devid="somerouterid" logid="0316013056" type="utm" subtype="webfilter" eventtype="ftgd_blk" level="warning" vd="root" eventtime=1587230269052907555 tz="-0500" policyid=100602 sessionid=1234 user="elasticuser" group="elasticgroup" authserver="elasticauth" srcip=192.168.2.1 srcport=61930 srcintf="port1" srcintfrole="lan" dstip=67.43.156.13 dstport=443 dstintf="wan1" dstintfrole="wan" proto=6 service="HTTPS" hostname="elastic.co" profile="elasticruleset" action="blocked" reqtype="direct" url="/config/" sentbyte=1152 rcvdbyte=1130 direction="outgoing" msg="URL belongs to a denied category in policy" method="domain" cat=76 catdesc="Internet Telephony"

packages/filestream/changelog.yml

@@ -0,0 +1,5 @@
+- version: "1.0.0"
+  changes:
+    - description: Initial Release
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2724

packages/filestream/data_stream/generic/_dev/test/system/test-filestream-config.yml

@@ -0,0 +1,6 @@
+service: filestream-logfile
+input: filestream
+data_stream:
+  vars:
+    paths:
+      - "{{SERVICE_LOGS_DIR}}/test-filestream.log"

packages/filestream/data_stream/generic/agent/stream/filestream.yml.hbs

@@ -0,0 +1,116 @@
+data_stream:
+  dataset: {{data_stream.dataset}}
+paths:
+{{#each paths as |path i|}}
+  - {{path}}
+{{/each}}
+{{#if pipeline}}
+pipeline: {{pipeline}}
+{{/if}}
+{{#if recursive_glob}}
+prospector.scanner.recursive_glob: {{recursive_glob}}
+{{/if}}
+{{#if exclude_files}}
+prospector.scanner.exclude_files:
+{{#each exclude_files as |exclude_file i|}}
+  - {{exclude_file}}
+{{/each}}
+{{/if}}
+{{#if include_files}}
+prospector.scanner.include_files:
+{{#each include_files as |include_file i|}}
+  - {{include_file}}
+{{/each}}
+{{/if}}
+{{#if symlinks}}
+prospector.scanner.symlinks: {{symlinks}}
+{{/if}}
+{{#if resend_on_touch}}
+prospector.scanner.resend_on_touch: {{resend_on_touch}}
+{{/if}}
+{{#if check_interval}}
+prospector.scanner.check_interval: {{check_interval}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
+{{#if ignore_inactive}}
+ignore_inactive: {{ignore_inactive}}
+{{/if}}
+{{#if close_on_state_changed_inactive}}
+close.on_state_change.inactive: {{close_on_state_changed_inactive}}
+{{/if}}
+{{#if close_on_state_changed_renamed}}
+close.on_state_change.renamed: {{close_on_state_changed_renamed}}
+{{/if}}
+{{#if close_on_state_changed_removed}}
+close.on_state_change.removed: {{close_on_state_changed_removed}}
+{{/if}}
+{{#if close_reader_eof}}
+close.reader.on_eof: {{close_reader_eof}}
+{{/if}}
+{{#if close_reader_after_interval}}
+close.reader.after_interval: {{close_reader_after_interval}}
+{{/if}}
+{{#if clean_inactive}}
+clean_inactive: {{clean_inactive}}
+{{/if}}
+{{#if clean_removed}}
+clean_removed: {{clean_removed}}
+{{/if}}
+{{#if backoff_init}}
+backoff.init: {{backoff_init}}
+{{/if}}
+{{#if backoff_max}}
+backoff.max: {{backoff_max}}
+{{/if}}
+{{#if file_identity_native}}
+file_identity.native: {{file_identity_native}}
+{{/if}}
+{{#if file_identity_path}}
+file_identity.path: {{file_identity_path}}
+{{/if}}
+{{#if file_identity_inode_marker_path}}
+file_identity.inode_marker.path: {{file_identity_inode_marker_path}}
+{{/if}}
+{{#if rotation_external_strategy_copytruncate}}
+rotation.external.strategy.copytruncate: {{rotation_external_strategy_copytruncate}}
+{{/if}}
+{{#if encoding}}
+encoding: {{encoding}}
+{{/if}}
+{{#if exclude_lines}}
+exclude_lines:
+{{#each exclude_lines as |exclude_line i|}}
+  - {{exclude_line}}
+{{/each}}
+{{/if}}
+{{#if include_lines}}
+include_lines:
+{{#each include_lines as |include_line i|}}
+  - {{include_line}}
+{{/each}}
+{{/if}}
+{{#if buffer_size}}
+buffer_size: {{buffer_size}}
+{{/if}}
+{{#if message_max_bytes}}
+message_max_bytes: {{message_max_bytes}}
+{{/if}}
+{{#if parsers}}
+parsers: 
+{{parsers}}
+{{/if}}
+{{#if tags}}
+tags:
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{/if}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/filestream/data_stream/generic/fields/base-fields.yml

@@ -0,0 +1,20 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: event.module
+  type: constant_keyword
+  description: Event module
+  value: filestream
+- name: event.dataset
+  type: constant_keyword
+  description: Event dataset
+  value: filestream.generic
+- name: "@timestamp"
+  type: date
+  description: Event timestamp.

packages/filestream/data_stream/generic/fields/beats.yml

@@ -0,0 +1,6 @@
+- name: input.type
+  description: Type of Filebeat input.
+  type: keyword
+- name: tags
+  type: keyword
+  description: User defined tags

packages/filestream/data_stream/generic/fields/ecs.yml

@@ -0,0 +1,12 @@
+- name: ecs.version
+  external: ecs
+- name: log.file.path
+  external: ecs
+- name: log.offset
+  description: Current log offset
+- name: log.level
+  external: ecs
+- name: message
+  external: ecs
+- name: event.original
+  external: ecs