diff --git a/Dockerfile b/Dockerfile index 280cd7d6..ae40e6c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,9 +15,14 @@ RUN CGO_ENABLED=0 GOOS=linux \ -o stress-tester cmd/stress-tester/main.go FROM rust:1.51.0-alpine3.13 as rust-builder -COPY ./src/rust/receiver-mock /build -WORKDIR /build RUN apk update && apk upgrade && apk add g++ + +WORKDIR /receiver-mock +COPY ./src/rust/receiver-mock . +RUN cargo build --release + +WORKDIR /logs-generator +COPY ./src/rust/logs-generator . RUN cargo build --release FROM alpine:3.13.5 @@ -45,7 +50,8 @@ RUN set -ex \ && curl -LJ https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /usr/bin/kubectl \ && chmod +x /usr/bin/kubectl \ && curl -LJ "${UPGRADE_2_0_SCRIPT_URL}" -o /usr/local/bin/upgrade-2.0.0.sh \ - && chmod +x /usr/local/bin/upgrade-2.0.0.sh + && chmod +x /usr/local/bin/upgrade-2.0.0.sh \ + && curl -LJ https://raw.githubusercontent.com/dwyl/english-words/master/words.txt -o /usr/local/wordlist.txt COPY \ ./src/ssh/motd \ @@ -72,7 +78,8 @@ COPY --from=go-builder \ /usr/bin/ COPY --from=rust-builder \ - /build/target/release/receiver-mock \ + /receiver-mock/target/release/receiver-mock \ + /logs-generator/target/release/logs-generator \ /usr/bin/ CMD ["/usr/bin/tools-usage"] diff --git a/README.md b/README.md index e594ecd4..e56d9ffd 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,21 @@ You can add additional parameters (like `--version=1.0.0`) at the end of the com List of supported arguments is compatible with [`helm show values`](https://helm.sh/docs/helm/helm_show_values/). +### Logs generator + +Logs generator is a tool for generating logs (text lines) using patterns, +which can specify changing parts (words, digits). + +```bash +kubectl run template-dependency \ + -it --quiet --rm \ + --restart=Never -n sumologic \ + --image sumologic/kubernetes-tools \ + -- logs-generator --help +``` + +[More information](src/rust/logs-generator/README.md) + ### Interactive mode The pod can be also run in interactive mode: diff --git a/scripts/test-image.sh b/scripts/test-image.sh index e48c6c02..688f7c48 100755 --- a/scripts/test-image.sh +++ b/scripts/test-image.sh @@ -13,6 +13,7 @@ function test_image() { template template-dependency template-prometheus-mixin + logs-generator " readonly apps local tag="${1}" diff --git a/src/rust/receiver-mock/.rustfmt.toml b/src/rust/.rustfmt.toml similarity index 100% rename from src/rust/receiver-mock/.rustfmt.toml rename to src/rust/.rustfmt.toml diff --git a/src/rust/logs-generator/.gitignore b/src/rust/logs-generator/.gitignore new file mode 100644 index 00000000..a905994a --- /dev/null +++ b/src/rust/logs-generator/.gitignore @@ -0,0 +1,3 @@ +/target +**/*.rs.bk +tmp diff --git a/src/rust/logs-generator/Cargo.lock b/src/rust/logs-generator/Cargo.lock new file mode 100644 index 00000000..a072471a --- /dev/null +++ b/src/rust/logs-generator/Cargo.lock @@ -0,0 +1,417 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "atty" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" +dependencies = [ + "libc", + "termion", + "winapi", +] + +[[package]] +name = "autocfg" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e5f34df7a019573fb8bdc7e24a2bfebe51a2a1d6bfdbaeccedb3c41fc574727" + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "clap-v3" +version = "3.0.0-beta.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfac055d61c39ace5061621530f7f55651a261a4fba296ce1bad06d41a8de65e" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "clap_derive-v3", + "indexmap", + "lazy_static", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "clap_derive-v3" +version = "3.0.0-beta.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6dd675567eb3e35787bd2583d129e85fabc7503b0a093d08c51198a307e2091" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "heck" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "indexmap" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" +dependencies = [ + "autocfg 1.0.1", + "hashbrown", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a4cd09b2ff695f9734c1934145a315594b7986398496841c7031a5a1bbdbd" + +[[package]] +name = "logs-generator" +version = "0.1.0" +dependencies = [ + "clap-v3", + "rand", +] + +[[package]] +name = "proc-macro-error" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a5b4b77fdb63c1eca72173d68d24501c54ab1269409f6b672c85deb18af69de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "syn-mid", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3906503e80ac6cbcacb2c2973fa8e473f24d7e2747c8c92bb230c2441cad96b5" +dependencies = [ + "autocfg 0.1.1", + "libc", + "rand_chacha", + "rand_core", + "rand_hc", + "rand_isaac", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.1", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0905b6b7079ec73b314d4c748701f6931eb79fd97c668caa3f1899b22b32c6db" + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_os" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46fbd5550acf75b0c2730f5dd1873751daf9beb8f11b44027778fae50d7feca" +dependencies = [ + "cloudabi", + "fuchsia-zircon", + "libc", + "rand_core", + "rdrand", + "winapi", +] + +[[package]] +name = "rand_pcg" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "086bd09a33c7044e56bb44d5bdde5a60e7f119a9e95b0775f545de759a32fe05" +dependencies = [ + "rand_core", + "rustc_version", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ee9a534dc1301776eff45b4fa92d2c39b1d8c3d3357e6eb593e0d795506fc2" + +[[package]] +name = "redox_termios" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" +dependencies = [ + "redox_syscall", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "strsim" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" + +[[package]] +name = "syn" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48fe99c6bd8b1cc636890bcc071842de909d902c81ac7dab53ba33c421ab8ffb" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "syn-mid" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa8e7560a164edb1621a55d18a0c59abf49d360f47aa7b821061dd7eea7fac9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "termion" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" +dependencies = [ + "libc", + "redox_syscall", + "redox_termios", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + +[[package]] +name = "unicode-width" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "vec_map" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" + +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + +[[package]] +name = "winapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92c1eb33641e276cfa214a0522acad57be5c56b10cb348b3c5117db75f3ac4b0" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/src/rust/logs-generator/Cargo.toml b/src/rust/logs-generator/Cargo.toml new file mode 100644 index 00000000..73194890 --- /dev/null +++ b/src/rust/logs-generator/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "logs-generator" +version = "0.1.0" +authors = ["Sumo Logic "] +edition = "2018" + +[dependencies] +clap-v3 = "3.0.0-beta.1" +rand = "0.6" diff --git a/src/rust/logs-generator/Makefile b/src/rust/logs-generator/Makefile new file mode 100644 index 00000000..1e75ba4c --- /dev/null +++ b/src/rust/logs-generator/Makefile @@ -0,0 +1,10 @@ +RUSTFMT_FLAGS = --config-path ../.rustfmt.toml --edition 2018 +RUST_SOURCE_FILES = $(shell find . -name "*.rs" -not -path "./target/*") + +.PHONY: rustfmt +rustfmt: + rustfmt $(RUSTFMT_FLAGS) $(RUST_SOURCE_FILES) + +.PHONY: check-rustfmt +check-rustfmt: + rustfmt --check $(RUSTFMT_FLAGS) $(RUST_SOURCE_FILES) diff --git a/src/rust/logs-generator/README.md b/src/rust/logs-generator/README.md new file mode 100644 index 00000000..dca691cc --- /dev/null +++ b/src/rust/logs-generator/README.md @@ -0,0 +1,55 @@ +# Logs generator + +Logs generator is tool for generating logs (text lines) using patterns, +which can specify changing parts (words, digits) + +## General options + +Logs generation can be control by multiple options: + +- `duration` - defines how much time in seconds logs should be genereated. +- `total-logs` - defines how many logs in total should be generated. +- `throughput` - maximum throughput (bytes per second). +- `logs-throughput` - maximum number of logs generated per second. + +## Patterns + +Generator uses patterns to generate logs. +Pattern is basically a log line with placeholders used to differentiate logs. + +Currently supported placeholders: + +- `{w}` is going to be replaced with random word from wordlist +- `{d}` is going to be replaced with random digit +- `{c}` is going to be replaced with logs counter + +Example patterns: + +```text +log number is {c}: {w} is random word and {d} is random digit +todays digits are: {d} {d} {d} {d} {d} +this log is always the same +``` + +### Patterns Generation + +Patterns can be randomly generated using given options: + +- `random-patterns` - number of patterns to be generated +- `min` - minimum length of pattern (in words) +- `max` - maximum length of pattern (in words) +- `known-words` - ratio of known words +- `random-words` - ratio of random words (`{w}` placeholder) +- `random-digits` - ratio of random digits (`{d}` placeholder) + +Ratio is calculated from all of those values using following formula: +`/( + + )` + +### Providing patterns + +There are three ways of providing patterns into generator: + +- `pattern-file` - including patterns from file (one per line) +- `pattern` - including patterns from string (separated with `$`) eg, `--pattern='{d}${w}'`, + mind `'` to avoid `$` evaluation +- `random-patterns` - generate patterns randomly. See [patterns generation](#patterns-generation) diff --git a/src/rust/logs-generator/src/main.rs b/src/rust/logs-generator/src/main.rs new file mode 100644 index 00000000..bfa7e9cd --- /dev/null +++ b/src/rust/logs-generator/src/main.rs @@ -0,0 +1,436 @@ +#[macro_use] +extern crate clap_v3; + +use clap_v3::{App, Arg}; +use rand::Rng; +use std::fs::File; +use std::fs::OpenOptions; +use std::io::{BufRead, BufReader, Write}; +use std::process; +use std::time::{Duration, Instant}; +use std::vec::Vec; + +fn main() { + let matches = App::new("fluent-test") + .version("v0.1") + + .help_heading("General options") + .arg(Arg::with_name("path") + .short('p') + .long("path") + .value_name("path") + .help("Path to the ouput file (/dev/stdout by default)") + .required(false) + .takes_value(true)) + .arg(Arg::with_name("logs-throughput") + .short('n') + .long("logs-throughput") + .value_name("logs-throughput") + .help("Maximum number of logs generated per second") + .required(false) + .takes_value(true)) + .arg(Arg::with_name("throughput") + .short('b') + .long("throughput") + .value_name("throughput") + .help("Maximum throughput (bytes per second)") + .required(false) + .takes_value(true)) + .arg(Arg::with_name("total-logs") + .short('T') + .long("total-logs") + .value_name("total-logs") + .help("Total number of generated logs") + .required(false) + .takes_value(true)) + .arg(Arg::with_name("wordlist") + .short('w') + .long("wordlist") + .value_name("wordlist") + .help("Wordlist to use") + .required(false) + .takes_value(true)) + .arg(Arg::with_name("verbose") + .short('S') + .long("verbose") + .value_name("verbose") + .help("Print additional logs") + .required(false) + .takes_value(true)) + .arg(Arg::with_name("duration") + .short('i') + .long("duration") + .value_name("duration") + .help("Logs generation duration") + .required(false) + .takes_value(true)) + .arg(Arg::with_name("time-resolution") + .short('I') + .long("time-resolution") + .value_name("time-resolution") + .help("Time resolution defines how often throughput should be verified and statistics printed") + .required(false) + .takes_value(true)) + + .help_heading("Pattern generation") + .arg(Arg::with_name("pattern") + .short('x') + .long("pattern") + .value_name("pattern") + .help("Pattern to use. You can use special words {w}, {d} and {c} to include random word, digit and log counter") + .required(false) + .takes_value(true)) + .arg(Arg::with_name("pattern-file") + .short('L') + .long("pattern-file") + .value_name("pattern-file") + .help("Pattern file to use. Every line is considered as separate pattern") + .required(false) + .takes_value(true)) + .arg(Arg::with_name("random-patterns") + .short('z') + .long("random-patterns") + .value_name("random-patterns") + .help("Amount of random patterns to generate") + .required(false) + .takes_value(true)) + .arg(Arg::with_name("min") + .short('m') + .long("min") + .value_name("min") + .help("Minimum random pattern length (words)") + .required(false) + .takes_value(true)) + .arg(Arg::with_name("max") + .short('M') + .long("max") + .value_name("max") + .help("Maximum random pattern length (words)") + .required(false) + .takes_value(true)) + .arg(Arg::with_name("random_words") + .short('W') + .long("random_words") + .value_name("random_words") + .help("Define ratio of random words to be used in the pattern") + .required(false) + .takes_value(true)) + .arg(Arg::with_name("random_digits") + .short('D') + .long("random_digits") + .value_name("random_digits") + .help("Define ratio of random digits to be used in the pattern") + .required(false) + .takes_value(true)) + .arg(Arg::with_name("known_words") + .short('K') + .long("known_words") + .value_name("known_words") + .help("Define ratio of known words to be used in the pattern") + .required(false) + .takes_value(true)) + .get_matches(); + + let time_resolution = value_t!(matches, "time-resolution", u64).unwrap_or(10); + let logs_per_s = value_t!(matches, "logs-throughput", u64).unwrap_or(0); + let bytes_per_s = value_t!(matches, "throughput", u64).unwrap_or(0); + let total_logs = value_t!(matches, "total-logs", u64).unwrap_or(0); + let path = value_t!(matches, "path", String).unwrap_or("/dev/stdout".to_string()); + let verbose = value_t!(matches, "verbose", bool).unwrap_or(true); + let duration = value_t!(matches, "duration", u64).unwrap_or(0); + let duration = Duration::from_secs(duration); + let no_duration = Duration::from_secs(0); + + // Open file to write + let f = OpenOptions::new().append(true).create(true).open(&path); + + let fd = match f { + Ok(val) => Some(val), + Err(err) => { + eprintln!("Error while opening file to write: {}", err); + process::exit(1); + } + }; + + // Read words from dictionary file + let wordlist = value_t!(matches, "wordlist", String).unwrap_or("/usr/local/wordlist.txt".to_string()); + let wordlist = read_wordlist(&wordlist); + print( + verbose, + format!("[s] {} words read from dictionary!", wordlist.len()), + ); + + // Summarise configuration + print( + verbose, + format!( + "[s] Going to generate logs into {} with average {} lps and average {} Bps with time resolution: {}", + &path, + logs_per_s, + bytes_per_s, + time_resolution + ) + ); + + // Prepare list of patterns + // if pattern-file is specified, patterns are get from it + // otherwise pattern is generated using known_words, random_words, random digits, min and max + // - known_words, random_words and random_digits represents ratio eg, 4:5:1, means that 40% of pattern should be known_words etc + // - min and max limits length of the pattern in terms of words + let pattern = value_t!(matches, "pattern", String).unwrap_or("".to_string()); + let random_patterns = value_t!(matches, "random-patterns", u32).unwrap_or(0); + let pattern_file = value_t!(matches, "pattern-file", String).unwrap_or("".to_string()); + let min = value_t!(matches, "min", u32).unwrap_or(5); + let max = value_t!(matches, "max", u32).unwrap_or(20); + let random_words = value_t!(matches, "random_words", u32).unwrap_or(2); + let random_digits = value_t!(matches, "random_digits", u32).unwrap_or(1); + let known_words = value_t!(matches, "known_words", u32).unwrap_or(7); + + let patterns: Vec = match pattern_file.as_ref() { + "" => collect_patterns( + pattern, + random_patterns, + &wordlist, + min, + max, + known_words, + random_words, + random_digits, + ), + x => read_patterns(x.to_string()), + }; + + for pattern in &patterns { + print(verbose, format!("Pattern: {}", pattern)); + } + + let start = Instant::now(); + let mut now = Instant::now(); + let mut count_logs = 0; + let mut count_bytes = 0; + let mut current_logs = 0; + let mut current_bytes = 0; + let mut counter = 0; + loop { + // Print statistics + if now.elapsed().as_secs() >= time_resolution { + print( + verbose, + format!( + "{} pps\t {} b/s", + current_logs / now.elapsed().as_secs(), + current_bytes / now.elapsed().as_secs() + ), + ); + print( + verbose, + format!("Total stats: {} logs, {} bytes", count_logs, count_bytes), + ); + now = Instant::now(); + current_logs = 0; + current_bytes = 0; + } + + // Skip iteration because limit of logs/s already reached + if logs_per_s > 0 && current_logs >= logs_per_s * time_resolution { + continue; + } + + // Skip iteration because limit of bytes/s already reached + if bytes_per_s > 0 && current_bytes >= bytes_per_s * time_resolution { + continue; + } + + // Stop generation if configured duration reached + if duration > no_duration && now - start > duration { + print( + verbose, + format!( + "Logs generation finished after {} seconds", + (now - start).as_secs() + ), + ); + break; + } + + // Stop generation if configured limit reached + if total_logs != 0 && count_logs >= total_logs { + break; + } + + // Generate logs from patterns + for pattern in &patterns { + let log = build_log(&pattern, &wordlist, &mut counter); + let mut saved_logs: u64 = 0; + + match &fd { + Some(x) => saved_logs = save_log(&x, &log), + None => {} + } + + if saved_logs == 1 { + current_logs += 1; + current_bytes += log.len() as u64; + count_logs += 1; + count_bytes += log.len() as u64; + } + + if total_logs != 0 && count_logs >= total_logs { + break; + } + } + } + print(verbose, format!("Sent {} logs in total", count_logs)); + print(verbose, format!("Sent {} bytes in total", count_bytes)); +} + +fn save_log(mut fd: &File, log: &String) -> u64 { + let write = fd.write_all(log.as_bytes()); + match write { + Ok(_result) => { + return 1; + } + Err(_err) => { + return 0; + } + } +} + +fn read_wordlist(filename: &String) -> Vec { + let mut vec = Vec::::new(); + let file = File::open(filename).unwrap(); + for line in BufReader::new(file).lines() { + vec.push(line.unwrap()); + } + return vec; +} + +fn build_log(pattern: &String, wordlist: &Vec, counter: &mut u64) -> String { + let mut rng = rand::thread_rng(); + let slices = pattern.split_whitespace(); + let mut log = "".to_owned(); + + for slice in slices { + if slice.starts_with("{") && slice.ends_with("}") { + // replace {w} with random word + if slice.contains("w") { + log += &get_random_word(wordlist); + } + // replace {d} with random digit + else if slice.contains("d") { + log += &rng.gen_range(0, 0xffffff).to_string(); + } + // replace {c} with counter value + else if slice.contains("c") { + // counter + log += &counter.to_string(); + *counter += 1; + } + } else { + log += slice; + } + log += " "; + } + log += "\n"; + return log; +} + +fn get_random_word(wordlist: &Vec) -> &String { + let mut rng = rand::thread_rng(); + return wordlist.get(rng.gen_range(0, wordlist.len())).unwrap(); +} + +fn generate_pattern( + wordlist: &Vec, + min: u32, + max: u32, + known_words: u32, + random_words: u32, + random_digits: u32, +) -> String { + let mut rng = rand::thread_rng(); + let mut pattern = "{c} : ".to_string(); + let mut possible_slices = Vec::::new(); + + for _ in 0..known_words { + possible_slices.push(get_random_word(wordlist).to_string()); + } + + for _ in 0..random_words { + possible_slices.push("{w}".to_string()); + } + + for _ in 0..random_digits { + possible_slices.push("{d}".to_string()); + } + + let pattern_slices = rng.gen_range(min, max + 1); + + for _ in 0..pattern_slices { + let position = rng.gen_range(0, possible_slices.len()); + let mut current_pattern = possible_slices.remove(position); + pattern += ¤t_pattern; + + if current_pattern != "{d}" && current_pattern != "{w}" { + current_pattern = get_random_word(wordlist).to_string(); + } + + possible_slices.push(current_pattern); + pattern += " "; + } + + return pattern; +} + +// Collect patterns from argument and merge with random patterns +fn collect_patterns( + pattern: String, + random_patterns: u32, + wordlist: &Vec, + min: u32, + max: u32, + known_words: u32, + random_words: u32, + random_digits: u32, +) -> Vec { + let mut patterns = Vec::::new(); + + if !pattern.eq("") { + let _patterns = pattern.split('$'); + for pattern in _patterns { + patterns.push(pattern.to_string()); + } + } + + for _ in 0..random_patterns { + patterns.push(generate_pattern( + wordlist, + min, + max, + known_words, + random_words, + random_digits, + )); + } + + return patterns; +} + +// Read patterns from file +fn read_patterns(filename: String) -> Vec { + let mut patterns = Vec::::new(); + + let file = File::open(filename).unwrap(); + for line in BufReader::new(file).lines() { + patterns.push(line.unwrap()); + } + + return patterns; +} + +// Prints message to stderr if verbose is true +fn print(verbose: bool, message: String) { + if verbose { + eprintln!("{}", message); + } +} diff --git a/src/rust/receiver-mock/Cargo.toml b/src/rust/receiver-mock/Cargo.toml index f4675c18..2cd45a80 100644 --- a/src/rust/receiver-mock/Cargo.toml +++ b/src/rust/receiver-mock/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "receiver-mock" version = "0.1.0" -authors = ["Dominik Rosiek "] +authors = ["Sumo Logic "] edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/src/rust/receiver-mock/Makefile b/src/rust/receiver-mock/Makefile index a58107bb..c0aeed22 100644 --- a/src/rust/receiver-mock/Makefile +++ b/src/rust/receiver-mock/Makefile @@ -1,4 +1,4 @@ -RUSTFMT_FLAGS = --config-path .rustfmt.toml --edition 2018 +RUSTFMT_FLAGS = --config-path ../.rustfmt.toml --edition 2018 RUST_SOURCE_FILES = $(shell find . -name "*.rs" -not -path "./target/*") .PHONY: rustfmt diff --git a/src/rust/receiver-mock/src/main.rs b/src/rust/receiver-mock/src/main.rs index b3d7e6d1..64ec542f 100644 --- a/src/rust/receiver-mock/src/main.rs +++ b/src/rust/receiver-mock/src/main.rs @@ -21,7 +21,7 @@ mod time; async fn main() -> std::io::Result<()> { let matches = App::new("Receiver mock") .version("0.0") - .author("Dominik Rosiek ") + .author("Sumo Logic ") .about("Receiver mock can be used for testing performance or functionality of kubernetes collection without sending data to sumologic") .arg(Arg::with_name("port") .short("p")