diff --git a/README.md b/README.md index 26b3d4a727..6f69782e05 100644 --- a/README.md +++ b/README.md @@ -203,7 +203,6 @@ CONFIGURATIONS: -sml, -show-match-line show match lines for file templates, works with extractors only -ztls use ztls library with autofallback to standard one for tls13 [Deprecated] autofallback to ztls is enabled by default -sni string tls sni hostname to use (default: input domain name) - -dt, -dialer-timeout value timeout for network requests. -dka, -dialer-keep-alive value keep-alive duration for network requests. -lfa, -allow-local-file-access allows file (payload) access anywhere on the system -lna, -restrict-local-network-access blocks connections to the local / private network @@ -212,7 +211,6 @@ CONFIGURATIONS: -sip, -source-ip string source ip address to use for network scan -rsr, -response-size-read int max response size to read in bytes -rss, -response-size-save int max response size to read in bytes (default 1048576) - -rrt, -response-read-timeout value response read timeout in seconds (default 5s) -reset reset removes all nuclei configuration and data files (including nuclei-templates) -tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization -hae, -http-api-endpoint string experimental http api endpoint @@ -238,7 +236,7 @@ FUZZING: UNCOVER: -uc, -uncover enable uncover engine -uq, -uncover-query string[] uncover search query - -ue, -uncover-engine string[] uncover search engine (shodan,censys,fofa,shodan-idb,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow) (default shodan) + -ue, -uncover-engine string[] uncover search engine (shodan,censys,fofa,shodan-idb,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow,google) (default shodan) -uf, -uncover-field string uncover fields to return (ip,port,host) (default "ip:port") -ul, -uncover-limit int uncover results to return (default 100) -ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60) @@ -311,9 +309,10 @@ STATISTICS: -mp, -metrics-port int port to expose nuclei metrics on (default 9092) CLOUD: - -auth configure projectdiscovery cloud (pdcp) api key - -cup, -cloud-upload upload scan results to pdcp dashboard - -sid, -scan-id string upload scan results to given scan id + -auth configure projectdiscovery cloud (pdcp) api key (default true) + -cup, -cloud-upload upload scan results to pdcp dashboard + -sid, -scan-id string upload scan results to existing scan id (optional) + -sname, -scan-name string scan name to set (optional) AUTHENTICATION: -sf, -secret-file string[] path to config file containing secrets for nuclei authenticated scan diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 0db5711f5a..d6449e29e4 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -4339,19 +4339,6 @@ Code contains code to execute for the javascript request.
-timeout int - -
-
- -Timeout in seconds is optional timeout for each javascript script execution (i.e init, pre-condition, code) - -
- -
- -
- stop-at-first-match bool
diff --git a/cmd/integration-test/loader.go b/cmd/integration-test/loader.go index 2c2beb4cd9..8e5ea40ff8 100644 --- a/cmd/integration-test/loader.go +++ b/cmd/integration-test/loader.go @@ -172,7 +172,14 @@ func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error ts := httptest.NewServer(router) defer ts.Close() - _, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/404") + configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]` + err := os.WriteFile("test-config.yaml", []byte(configFileData), permissionutil.ConfigFilePermission) + if err != nil { + return err + } + defer os.Remove("test-config.yaml") + + _, err = testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/404", "-config", "test-config.yaml") if err == nil { return fmt.Errorf("expected error for nonexisting workflow url") } @@ -188,7 +195,14 @@ func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error ts := httptest.NewServer(router) defer ts.Close() - _, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-workflow-url", ts.URL+"/404") + configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]` + err := os.WriteFile("test-config.yaml", []byte(configFileData), permissionutil.ConfigFilePermission) + if err != nil { + return err + } + defer os.Remove("test-config.yaml") + + _, err = testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-workflow-url", ts.URL+"/404", "-config", "test-config.yaml") if err == nil { return fmt.Errorf("expected error for nonexisting workflow url") } diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 5aea16337c..2d2be0aa1a 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -297,7 +297,6 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.ShowMatchLine, "show-match-line", "sml", false, "show match lines for file templates, works with extractors only"), flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13 [Deprecated] autofallback to ztls is enabled by default"), //nolint:all flagSet.StringVar(&options.SNI, "sni", "", "tls sni hostname to use (default: input domain name)"), - flagSet.DurationVarP(&options.DialerTimeout, "dialer-timeout", "dt", 0, "timeout for network requests."), flagSet.DurationVarP(&options.DialerKeepAlive, "dialer-keep-alive", "dka", 0, "keep-alive duration for network requests."), flagSet.BoolVarP(&options.AllowLocalFileAccess, "allow-local-file-access", "lfa", false, "allows file (payload) access anywhere on the system"), flagSet.BoolVarP(&options.RestrictLocalNetworkAccess, "restrict-local-network-access", "lna", false, "blocks connections to the local / private network"), @@ -306,7 +305,6 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.SourceIP, "source-ip", "sip", "", "source ip address to use for network scan"), flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 0, "max response size to read in bytes"), flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", unitutils.Mega, "max response size to read in bytes"), - flagSet.DurationVarP(&options.ResponseReadTimeout, "response-read-timeout", "rrt", time.Duration(5*time.Second), "response read timeout in seconds"), flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"), flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"), flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"), diff --git a/examples/simple/simple.go b/examples/simple/simple.go index 42d3fe4387..29de464655 100644 --- a/examples/simple/simple.go +++ b/examples/simple/simple.go @@ -1,9 +1,13 @@ package main -import nuclei "github.com/projectdiscovery/nuclei/v3/lib" +import ( + "context" + + nuclei "github.com/projectdiscovery/nuclei/v3/lib" +) func main() { - ne, err := nuclei.NewNucleiEngine( + ne, err := nuclei.NewNucleiEngineCtx(context.Background(), nuclei.WithTemplateFilters(nuclei.TemplateFilters{Tags: []string{"oast"}}), nuclei.EnableStatsWithOpts(nuclei.StatsOptions{MetricServerPort: 6064}), // optionally enable metrics server for better observability ) diff --git a/go.mod b/go.mod index 6970a332a2..e9efbb8f70 100644 --- a/go.mod +++ b/go.mod @@ -20,12 +20,12 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.20 - github.com/projectdiscovery/fastdialer v0.1.4-0.20240611222741-427bffb2ea13 - github.com/projectdiscovery/hmap v0.0.46 + github.com/projectdiscovery/fastdialer v0.1.6 + github.com/projectdiscovery/hmap v0.0.49 github.com/projectdiscovery/interactsh v1.2.0 - github.com/projectdiscovery/rawhttp v0.1.51 - github.com/projectdiscovery/retryabledns v1.0.63 - github.com/projectdiscovery/retryablehttp-go v1.0.64 + github.com/projectdiscovery/rawhttp v0.1.57 + github.com/projectdiscovery/retryabledns v1.0.66 + github.com/projectdiscovery/retryablehttp-go v1.0.68 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -58,6 +58,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.13.27 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72 github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 + github.com/cespare/xxhash v1.1.0 github.com/charmbracelet/glamour v0.6.0 github.com/clbanning/mxj/v2 v2.7.0 github.com/denisenkom/go-mssqldb v0.12.3 @@ -78,11 +79,11 @@ require ( github.com/mholt/archiver v3.1.1+incompatible github.com/ory/dockertest/v3 v3.10.0 github.com/praetorian-inc/fingerprintx v1.1.9 - github.com/projectdiscovery/dsl v0.1.2 + github.com/projectdiscovery/dsl v0.1.6 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb - github.com/projectdiscovery/goflags v0.1.54 - github.com/projectdiscovery/gologger v1.1.12 + github.com/projectdiscovery/goflags v0.1.59 + github.com/projectdiscovery/gologger v1.1.14 github.com/projectdiscovery/gostruct v0.0.2 github.com/projectdiscovery/gozero v0.0.2 github.com/projectdiscovery/httpx v1.6.3 @@ -93,9 +94,9 @@ require ( github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.8 - github.com/projectdiscovery/useragent v0.0.54 - github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c - github.com/projectdiscovery/wappalyzergo v0.1.4 + github.com/projectdiscovery/useragent v0.0.60 + github.com/projectdiscovery/utils v0.2.1 + github.com/projectdiscovery/wappalyzergo v0.1.10 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 github.com/stretchr/testify v1.9.0 @@ -124,12 +125,11 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect - github.com/bits-and-blooms/bitset v1.8.0 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/bits-and-blooms/bloom/v3 v3.5.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect - github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect @@ -143,9 +143,10 @@ require ( github.com/docker/cli v24.0.5+incompatible // indirect github.com/docker/docker v24.0.9+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect - github.com/fatih/color v1.15.0 // indirect + github.com/fatih/color v1.16.0 // indirect github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/gaissmai/bart v0.9.5 // indirect github.com/geoffgarside/ber v1.1.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-gonic/gin v1.9.1 // indirect @@ -199,8 +200,8 @@ require ( github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect - github.com/projectdiscovery/asnmap v1.1.0 // indirect - github.com/projectdiscovery/cdncheck v1.0.9 // indirect + github.com/projectdiscovery/asnmap v1.1.1 // indirect + github.com/projectdiscovery/cdncheck v1.1.0 // indirect github.com/projectdiscovery/freeport v0.0.5 // indirect github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect @@ -267,7 +268,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-retryablehttp v0.7.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hdm/jarm-go v0.0.7 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect @@ -284,7 +285,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/networkpolicy v0.0.8 + github.com/projectdiscovery/networkpolicy v0.0.9 github.com/rivo/uniseg v0.4.6 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect @@ -292,7 +293,6 @@ require ( github.com/trivago/tgo v1.0.7 github.com/ulikunitz/xz v0.5.12 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/yl2chen/cidranger v1.0.2 // indirect github.com/ysmood/goob v0.4.0 // indirect github.com/ysmood/gson v0.7.3 // indirect github.com/ysmood/leakless v0.8.0 // indirect diff --git a/go.sum b/go.sum index a3702b0fdb..52dfe6a693 100644 --- a/go.sum +++ b/go.sum @@ -186,8 +186,9 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c= github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY= github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= @@ -309,8 +310,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= @@ -326,6 +327,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/gaissmai/bart v0.9.5 h1:vy+r4Px6bjZ+v2QYXAsg63vpz9IfzdW146A8Cn4GPIo= +github.com/gaissmai/bart v0.9.5/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg= github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/getkin/kin-openapi v0.123.0 h1:zIik0mRwFNLyvtXK274Q6ut+dPh6nlxBp0x7mNrPhs8= @@ -528,13 +531,13 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= -github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -823,34 +826,34 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/praetorian-inc/fingerprintx v1.1.9 h1:zWbG/Fdan0s/dvXkeaHb/CdFTz/yEEzrAF4iCzok3r8= github.com/praetorian-inc/fingerprintx v1.1.9/go.mod h1:k6EJIHe/Da4DH5e4JuoZHe+qSGq/KPUmXGaK+xW74OI= -github.com/projectdiscovery/asnmap v1.1.0 h1:ynvbLB5cNpyQ2+k9IP0Rpla+0JmCJpd3mw6KLAW13m0= -github.com/projectdiscovery/asnmap v1.1.0/go.mod h1:QNjBnGLxUBEZAgaYk/Av5cjKKWFY3i/FOfoIWCUApoY= +github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kID2iwsDqI= +github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQOocUvrssFlg= -github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= +github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/snk0cQ3kXE= +github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc= github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4= github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= -github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK7eeGw= -github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA= -github.com/projectdiscovery/fastdialer v0.1.4-0.20240611222741-427bffb2ea13 h1:mAehxpgxmvFrKcQ8taA3xQskp3gg6GayAgatTAn37bg= -github.com/projectdiscovery/fastdialer v0.1.4-0.20240611222741-427bffb2ea13/go.mod h1:PcNscfCNrCnrkmvob90yfd+rNDMh6ZOIvRIl6HFPPXE= +github.com/projectdiscovery/dsl v0.1.6 h1:6TUYMwbjcYJ5OoRoawPX1SYD7wgubX/1FreC8SwFqgc= +github.com/projectdiscovery/dsl v0.1.6/go.mod h1:1PAV9A6X+fdyMXW0SY4i0BHrO8yN2+FB2jp25+0AT04= +github.com/projectdiscovery/fastdialer v0.1.6 h1:dtdE7WjaydpndKPcUYWms5iOhBSC0nvBAHyZd6I4pdQ= +github.com/projectdiscovery/fastdialer v0.1.6/go.mod h1:a5XOjvaO4b+ddO/YLoM4nvK3CPPviW6HmoAd+AoszMw= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg= github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY= -github.com/projectdiscovery/goflags v0.1.54 h1:xbaEBNbKqXyRu4154UlhsMAFmpaDYma4jJ9zUZGvXW8= -github.com/projectdiscovery/goflags v0.1.54/go.mod h1:cfLzWWxgl/ft5cSHVJZjnvIzd4wX3A9Kz+W1pjeBZvc= -github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= -github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= +github.com/projectdiscovery/goflags v0.1.59 h1:rFScpC57L0Ln0dJmBtstyxHUH7K9H4UmHACXcSkdqqk= +github.com/projectdiscovery/goflags v0.1.59/go.mod h1:dj67QGp/D81WRLtzU0HsWR20zgoGZ0cnk3Wbt9xJcuo= +github.com/projectdiscovery/gologger v1.1.14 h1:8SS9qDCisCFffKHzWMX+GDDArxBJ9A7DhYrESEixAYo= +github.com/projectdiscovery/gologger v1.1.14/go.mod h1:CPk1nAZ3PqgspKBuSJR+xwY2i/Rm+P/ovahpP30EJy8= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E= github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8= -github.com/projectdiscovery/hmap v0.0.46 h1:1fdZ0A8Zkg30SLl0gAbEqOYb4vB3gtokbQuYI+oXUNk= -github.com/projectdiscovery/hmap v0.0.46/go.mod h1:QleSTt2HrokGNPpymUqb7PQ/2EEHEnaU5qKrrm51Va8= +github.com/projectdiscovery/hmap v0.0.49 h1:QNW98JDqd0nmaIFRMbn0sLocRIGXAud1liGip4iPdvc= +github.com/projectdiscovery/hmap v0.0.49/go.mod h1:BA4cyYTrWKMUa78815R6p3tuGiYu2MJHYnjwXmefdbU= github.com/projectdiscovery/httpx v1.6.3 h1:TSu3zEKOHGFfMsGeB0lZEFRl6Tq4nPuIgKDXi9NnYpg= github.com/projectdiscovery/httpx v1.6.3/go.mod h1:9FmsmkT71aTa4WnOyBIPzB8I9c0QAJ9PSVUqs2NkHRQ= github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8= @@ -863,18 +866,18 @@ github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQt github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8= github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc= -github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8= -github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= +github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk= +github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSXDKNNh2G1Ot9Q= github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4= -github.com/projectdiscovery/rawhttp v0.1.51 h1:T4sskU2fsX9zNR7nyae0tD/0fliIs73KwBzyoJwBs10= -github.com/projectdiscovery/rawhttp v0.1.51/go.mod h1:ejBaw3dCfJSfbtFv0ib6YfFSjQzt7ZdRVpQWJASxIUs= +github.com/projectdiscovery/rawhttp v0.1.57 h1:2vCT2i1NSZbTBH+uUBrxOJjxDPKgIl2q6BGtQjs/Hko= +github.com/projectdiscovery/rawhttp v0.1.57/go.mod h1:qtthyaU0k8eqcEdza1R/fTqwyxSK4BZ511ThxgkiQtE= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.63 h1:Ijt47ybwf+iIwul606NlPKPN+ouhOYPeA9sLHVgqLG4= -github.com/projectdiscovery/retryabledns v1.0.63/go.mod h1:lTs48OYJnMFuuBzT+3z3PrZ58K0OUBgP7Y4o3ttBwb0= -github.com/projectdiscovery/retryablehttp-go v1.0.64 h1:90VkUuWKm/JW5Tym0u+N1qseKe4gC5Rp048VJBLcxv8= -github.com/projectdiscovery/retryablehttp-go v1.0.64/go.mod h1:fXVLBcwKwBrCVEgi3FrbLPtSTIczyJEKIlp1WD6VTI0= +github.com/projectdiscovery/retryabledns v1.0.66 h1:OzKcR+TqIYdSDEED2JWJA90ZNFCEKRv+hOvpMHEZEOY= +github.com/projectdiscovery/retryabledns v1.0.66/go.mod h1:ODpvZ6kXXB2uqK7GPyfkd/tR+4wkeIm1DR5TkRQv7KE= +github.com/projectdiscovery/retryablehttp-go v1.0.68 h1:MN/kfccDnebmp1kNcCnGlGuCfvet6y5eN1Y1hIKBB1I= +github.com/projectdiscovery/retryablehttp-go v1.0.68/go.mod h1:F3QNCoJExPFT59AACrVgYVhNiDW5Fp8s5HbXc+8CaaU= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= @@ -883,12 +886,12 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo= github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiMk5xq6U4u0= github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM= -github.com/projectdiscovery/useragent v0.0.54 h1:3gM4TnhLSspSM/XkI97YcUazXA3KOkQXEU7TIBly2Fc= -github.com/projectdiscovery/useragent v0.0.54/go.mod h1:dKFkqUb44qx7MpwQDAbhL1t2IrFhcrgI5MlXUZ9E8dM= -github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c h1:0I/iRtu5nPYle1v8/R33pCLOrH5bziP5Bi0eZURxTQY= -github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c/go.mod h1:mXs6OOeG9l/dVchjB2PGvQO3+wuMiE14Y/kmHeKogoM= -github.com/projectdiscovery/wappalyzergo v0.1.4 h1:Q0j5t/c74DlPKRvGYLZFv8npTA7MBW9uwdlsMBn/Yh4= -github.com/projectdiscovery/wappalyzergo v0.1.4/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= +github.com/projectdiscovery/useragent v0.0.60 h1:qDU1rwA+XOKmSqp7yoijAN4PuvLQc2ZvuaoWH7QIkuc= +github.com/projectdiscovery/useragent v0.0.60/go.mod h1:05IDiJEy2dWl3x6dnsWtJYPwT40oWha144Us7+Fwr6w= +github.com/projectdiscovery/utils v0.2.1 h1:XcOUJtR5ReXDWqxDkJWPVpC2lSqIYTs8z8YNILcPiqk= +github.com/projectdiscovery/utils v0.2.1/go.mod h1:h3o2zmJguUm9FKvel1F9jYCLxoWhFakyytRMjqb7Dg4= +github.com/projectdiscovery/wappalyzergo v0.1.10 h1:cUTMw8xYXyKTAxUiGYltJxEh4OtJyCvaahj+RG3mHSY= +github.com/projectdiscovery/wappalyzergo v0.1.10/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= diff --git a/integration_tests/protocols/code/pre-condition.yaml b/integration_tests/protocols/code/pre-condition.yaml index 1c44e9579a..2190ffecbe 100644 --- a/integration_tests/protocols/code/pre-condition.yaml +++ b/integration_tests/protocols/code/pre-condition.yaml @@ -23,4 +23,4 @@ code: - type: dsl dsl: - true -# digest: 490a004630440220192fb8f704b078c2885047b85ac1a0491be86485c033a976d201599683a35aab0220604b1c3781e9d97079d0e5c23c18e6a2d87493c8e2b930536e692ee7d06e9247:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 4a0a0047304502200307590191cb7c766b6c21e5777d345bdddf7adf9d6da8f7d336d585d9ac4a8b022100fd30fb0c7722778eb3d861d60e721d805925b8d8df2b979ef2104c35ec57d5cb:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-env-var.yaml b/integration_tests/protocols/code/py-env-var.yaml index 9ff947c208..067c183bb7 100644 --- a/integration_tests/protocols/code/py-env-var.yaml +++ b/integration_tests/protocols/code/py-env-var.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input baz" -# digest: 4a0a00473045022033f72f1b9d5143f58a2dc79c2597000f34080251ac3702c36c3fad00917dfeeb0221009ba05c715c9e2e36dba471be6c0106a09ae3822d8a3e9e4bcf377e9f4a395a01:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 4a0a0047304502203fe1d7d52bc2a41886d576a90c82c3be42078baaa4b46e1f3d8519665d6f88b202210081feb82c41150c5b218e226fc4f299ded19f42ba01ef34ba60b0634b4ea6ee12:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-file.yaml b/integration_tests/protocols/code/py-file.yaml index ad69371d05..3521ec902d 100644 --- a/integration_tests/protocols/code/py-file.yaml +++ b/integration_tests/protocols/code/py-file.yaml @@ -18,4 +18,4 @@ code: - type: word words: - "hello from input" -# digest: 4a0a004730450220377128cb11d9f6f0fee1f4dbd841e46783de26e90a216fa55a7609ee2bc823c60221009166ee0f85e3a1811588ab19e73ea96ab3d582dc8180dbcbbad0ea9ab7e9025d:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 4b0a00483046022100afb5ebff14a40e7f9b679ffc4d93ce7849e33eb398ebb47f2e757cd24831f9dd02210089ffa21b2763e99ebce95dfc5b91e1e62da4ccdc9d2ad5c48584fa350ba335af:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-interactsh.yaml b/integration_tests/protocols/code/py-interactsh.yaml index 76d14efb2e..0ccab7a7cd 100644 --- a/integration_tests/protocols/code/py-interactsh.yaml +++ b/integration_tests/protocols/code/py-interactsh.yaml @@ -26,4 +26,4 @@ code: part: interactsh_protocol words: - "http" -# digest: 4b0a00483046022100d472d50bd83117d334f5217c7a40dcdf34138e90029eaace51697d902296bf37022100a393b49420a96f60d6d89b79b5135ee2233b2468d374851890eea114b08195d1:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 4b0a00483046022100939f83e74d43932a5bd792b1fd2c100eec2df60f2b2a8dd56b5c8ef5faa92b17022100f93031b0de373af7d78e623968ea5a2d67c4561ef70e3e6da15aef7e5c853115:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-snippet.yaml b/integration_tests/protocols/code/py-snippet.yaml index 4837fa7e0e..5e36c1a673 100644 --- a/integration_tests/protocols/code/py-snippet.yaml +++ b/integration_tests/protocols/code/py-snippet.yaml @@ -21,4 +21,4 @@ code: - type: word words: - "hello from input" -# digest: 4b0a004830460221008886054bb5dd6345e434e30f31c8fddce3c484a4f33aa6321b5185675866029d022100d188a83d0fde029f8b586061c65ab72b43755c3fb10fdd59501bb9bbadbb1ff7:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 4a0a00473045022100b8e676ce0c57b60c233a0203539dec20457bbb5f1790d351a5d45405b6668b2602204b1f2fa18e7db099f05329009597ceb2d9b7337562c1a676e8d50ea2f1c6fcbe:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/workflow/code-template-1.yaml b/integration_tests/workflow/code-template-1.yaml index d41a1a6957..81a9773ae0 100644 --- a/integration_tests/workflow/code-template-1.yaml +++ b/integration_tests/workflow/code-template-1.yaml @@ -19,4 +19,4 @@ code: regex: - 'hello from (.*)' group: 1 -# digest: 490a0046304402202c63d47bb0acdd40b3b852d95490d492ff5741b84071b2a8a40371be7797c13602202b6b977e157edf2ef70a402a2e57d4eb5a67c5ca91f0a2f9a10a966e8485ebaf:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 490a00463044022050da011362cf08c2cb81e812c7f86d7282afe0562d4bf00d390f1300d19bc910022029e9d305da69e941ac18797645aecb217abde6557f891e141301b48e89a3c0cd:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/workflow/code-template-2.yaml b/integration_tests/workflow/code-template-2.yaml index ddd7d36a4e..1fb9805501 100644 --- a/integration_tests/workflow/code-template-2.yaml +++ b/integration_tests/workflow/code-template-2.yaml @@ -18,4 +18,4 @@ code: - type: word words: - "hello from first" -# digest: 490a00463044022025661eab353b7f359c0d428a86b6287545d7f759375e8025cc8c9c77b616ca6502200bc2c019059622df3c88e7caa6dd7d1fb9b956010aa0de2ee2b9f7dd0a3c4954:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 4b0a00483046022100b3b8759c0df028455eb59b1433ac240e5d4604b011bb0c63680bd3cc159ac6f0022100f44aa11b640d11ad0e2902897f4eb51666ab3cd83c31dfd2590f6e43391e39b0:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/lib/config.go b/lib/config.go index 24bc08691f..df44566225 100644 --- a/lib/config.go +++ b/lib/config.go @@ -440,3 +440,11 @@ func WithCatalog(cat catalog.Catalog) NucleiSDKOptions { return nil } } + +// DisableUpdateCheck disables nuclei update check +func DisableUpdateCheck() NucleiSDKOptions { + return func(e *NucleiEngine) error { + DefaultConfig.DisableUpdateCheck() + return nil + } +} diff --git a/lib/sdk_private.go b/lib/sdk_private.go index 3475200bd0..b65ecf2a2f 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -231,7 +231,10 @@ func (e *NucleiEngine) init(ctx context.Context) error { // and also upgrade templates to latest version if available installer.NucleiSDKVersionCheck() - return e.processUpdateCheckResults() + if DefaultConfig.CanCheckForUpdates() { + return e.processUpdateCheckResults() + } + return nil } type syncOnce struct { diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 61bdc68ff8..73ba92381e 100644 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -941,11 +941,6 @@ "title": "code to execute in javascript", "description": "Executes inline javascript code for the request" }, - "timeout": { - "type": "integer", - "title": "timeout for javascript execution", - "description": "Timeout in seconds is optional timeout for entire javascript script execution" - }, "stop-at-first-match": { "type": "boolean", "title": "stop at first match", diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go index 0ee9f437bb..8aa6beedfe 100644 --- a/pkg/catalog/config/constants.go +++ b/pkg/catalog/config/constants.go @@ -31,7 +31,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v3.2.9` + Version = `v3.3.0` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGitHubTemplatesDirName = "github" diff --git a/pkg/catalog/disk/catalog.go b/pkg/catalog/disk/catalog.go index ce1a080fdd..38c2260758 100644 --- a/pkg/catalog/disk/catalog.go +++ b/pkg/catalog/disk/catalog.go @@ -11,17 +11,25 @@ import ( // DiskCatalog is a template catalog helper implementation based on disk type DiskCatalog struct { templatesDirectory string - templatesFS fs.FS // TODO: Refactor to use this + templatesFS fs.FS // Due to issues with how Go has implemented fs.FS, we'll have to also implement normal os operations, as well. See: https://github.com/golang/go/issues/44279 } // NewCatalog creates a new Catalog structure using provided input items // using disk based items func NewCatalog(directory string) *DiskCatalog { catalog := &DiskCatalog{templatesDirectory: directory} - if directory != "" { - catalog.templatesFS = os.DirFS(directory) - } else { - catalog.templatesFS = os.DirFS(config.DefaultConfig.GetTemplateDir()) + if directory == "" { + catalog.templatesDirectory = config.DefaultConfig.GetTemplateDir() + } + return catalog +} + +// NewFSCatalog creates a new Catalog structure using provided input items +// using the fs.FS as its filesystem. +func NewFSCatalog(fs fs.FS, directory string) *DiskCatalog { + catalog := &DiskCatalog{ + templatesDirectory: directory, + templatesFS: fs, } return catalog } @@ -29,11 +37,15 @@ func NewCatalog(directory string) *DiskCatalog { // OpenFile opens a file and returns an io.ReadCloser to the file. // It is used to read template and payload files based on catalog responses. func (d *DiskCatalog) OpenFile(filename string) (io.ReadCloser, error) { - file, err := os.Open(filename) - if err != nil { - if file, errx := os.Open(BackwardsCompatiblePaths(d.templatesDirectory, filename)); errx == nil { - return file, nil + if d.templatesFS == nil { + file, err := os.Open(filename) + if err != nil { + if file, errx := os.Open(BackwardsCompatiblePaths(d.templatesDirectory, filename)); errx == nil { + return file, nil + } } + return file, err } - return file, err + + return d.templatesFS.Open(filename) } diff --git a/pkg/catalog/disk/find.go b/pkg/catalog/disk/find.go index 088bcbfe1f..a5d46ff078 100644 --- a/pkg/catalog/disk/find.go +++ b/pkg/catalog/disk/find.go @@ -79,17 +79,21 @@ func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) { } // try to handle deprecated template paths - absPath := BackwardsCompatiblePaths(c.templatesDirectory, target) - if absPath != target && strings.TrimPrefix(absPath, c.templatesDirectory+string(filepath.Separator)) != target { - if config.DefaultConfig.LogAllEvents { - gologger.DefaultLogger.Print().Msgf("[%v] requested Template path %s is deprecated, please update to %s\n", aurora.Yellow("WRN").String(), target, absPath) + absPath := target + if c.templatesFS == nil { + absPath = BackwardsCompatiblePaths(c.templatesDirectory, target) + if absPath != target && strings.TrimPrefix(absPath, c.templatesDirectory+string(filepath.Separator)) != target { + if config.DefaultConfig.LogAllEvents { + gologger.DefaultLogger.Print().Msgf("[%v] requested Template path %s is deprecated, please update to %s\n", aurora.Yellow("WRN").String(), target, absPath) + } + deprecatedPathsCounter++ } - deprecatedPathsCounter++ - } - absPath, err := c.convertPathToAbsolute(absPath) - if err != nil { - return nil, errors.Wrapf(err, "could not find template file") + var err error + absPath, err = c.convertPathToAbsolute(absPath) + if err != nil { + return nil, errors.Wrapf(err, "could not find template file") + } } // Template input is either a file or a directory @@ -143,24 +147,60 @@ func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]s if c.templatesDirectory == "" { templateDir = "./" } - matches, _ := fs.Glob(os.DirFS(filepath.Join(templateDir, "http")), inputGlob) - if len(matches) != 0 { + + if c.templatesFS == nil { + matches, _ := fs.Glob(os.DirFS(filepath.Join(templateDir, "http")), inputGlob) + if len(matches) != 0 { + return matches + } + + // condition to support network cve related globs + matches, _ = fs.Glob(os.DirFS(filepath.Join(templateDir, "network")), inputGlob) + return matches + } else { + sub, err := fs.Sub(c.templatesFS, filepath.Join(templateDir, "http")) + if err != nil { + return nil + } + matches, _ := fs.Glob(sub, inputGlob) + if len(matches) != 0 { + return matches + } + + // condition to support network cve related globs + sub, err = fs.Sub(c.templatesFS, filepath.Join(templateDir, "network")) + if err != nil { + return nil + } + matches, _ = fs.Glob(sub, inputGlob) return matches } - // condition to support network cve related globs - matches, _ = fs.Glob(os.DirFS(filepath.Join(templateDir, "network")), inputGlob) - return matches } var matched []string - matches, err := fs.Glob(c.templatesFS, relPath) - if len(matches) != 0 { - matched = append(matched, matches...) + var matches []string + if c.templatesFS == nil { + var err error + matches, err = filepath.Glob(relPath) + if len(matches) != 0 { + matched = append(matched, matches...) + } else { + matched = append(matched, OldPathsResolver(relPath)...) + } + if err != nil && len(matched) == 0 { + return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err) + } } else { - matched = append(matched, OldPathsResolver(relPath)...) - } - if err != nil && len(matched) == 0 { - return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err) + var err error + matches, err = fs.Glob(c.templatesFS, relPath) + if len(matches) != 0 { + matched = append(matched, matches...) + } else { + matched = append(matched, OldPathsResolver(relPath)...) + } + if err != nil && len(matched) == 0 { + return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err) + } } results := make([]string, 0, len(matches)) for _, match := range matches { @@ -175,11 +215,23 @@ func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]s // findFileMatches finds if a path is an absolute file. If the path // is a file, it returns true otherwise false with no errors. func (c *DiskCatalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) { - info, err := os.Stat(absPath) + if c.templatesFS != nil { + absPath = strings.TrimPrefix(absPath, "/") + } + var info fs.File + if c.templatesFS == nil { + info, err = os.Open(absPath) + } else { + info, err = c.templatesFS.Open(absPath) + } + if err != nil { + return "", false, err + } + stat, err := info.Stat() if err != nil { return "", false, err } - if !info.Mode().IsRegular() { + if !stat.Mode().IsRegular() { return "", false, nil } if _, ok := processed[absPath]; !ok { @@ -192,22 +244,43 @@ func (c *DiskCatalog) findFileMatches(absPath string, processed map[string]struc // findDirectoryMatches finds matches for templates from a directory func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) { var results []string - err := filepath.WalkDir( - absPath, - func(path string, d fs.DirEntry, err error) error { - // continue on errors - if err != nil { + var err error + if c.templatesFS == nil { + err = filepath.WalkDir( + absPath, + func(path string, d fs.DirEntry, err error) error { + // continue on errors + if err != nil { + return nil + } + if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown { + if _, ok := processed[path]; !ok { + results = append(results, path) + processed[path] = struct{}{} + } + } return nil - } - if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown { - if _, ok := processed[path]; !ok { - results = append(results, path) - processed[path] = struct{}{} + }, + ) + } else { + err = fs.WalkDir( + c.templatesFS, + absPath, + func(path string, d fs.DirEntry, err error) error { + // continue on errors + if err != nil { + return nil } - } - return nil - }, - ) + if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown { + if _, ok := processed[path]; !ok { + results = append(results, path) + processed[path] = struct{}{} + } + } + return nil + }, + ) + } return results, err } diff --git a/pkg/core/executors.go b/pkg/core/executors.go index 0e91c3883b..8447efed13 100644 --- a/pkg/core/executors.go +++ b/pkg/core/executors.go @@ -27,7 +27,7 @@ func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*te var match bool ctx := scan.NewScanContext(ctx, contextargs.New(ctx)) if e.Callback != nil { - if results, err := template.Executer.ExecuteWithResults(ctx); err != nil { + if results, err := template.Executer.ExecuteWithResults(ctx); err == nil { for _, result := range results { e.Callback(result) } @@ -129,7 +129,7 @@ func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templ match = e.executeWorkflow(ctx, template.CompiledWorkflow) default: if e.Callback != nil { - if results, err := template.Executer.ExecuteWithResults(ctx); err != nil { + if results, err := template.Executer.ExecuteWithResults(ctx); err == nil { for _, result := range results { e.Callback(result) } @@ -194,7 +194,7 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t match = e.executeWorkflow(ctx, template.CompiledWorkflow) default: if e.Callback != nil { - if results, err := template.Executer.ExecuteWithResults(ctx); err != nil { + if results, err := template.Executer.ExecuteWithResults(ctx); err == nil { for _, result := range results { e.Callback(result) } diff --git a/pkg/input/provider/http/multiformat.go b/pkg/input/provider/http/multiformat.go index ee95079ef4..d58970fec5 100644 --- a/pkg/input/provider/http/multiformat.go +++ b/pkg/input/provider/http/multiformat.go @@ -66,9 +66,10 @@ func (i *HttpInputProvider) Count() int64 { // Iterate over all inputs in order func (i *HttpInputProvider) Iterate(callback func(value *contextargs.MetaInput) bool) { err := i.format.Parse(i.inputFile, func(request *types.RequestResponse) bool { - return callback(&contextargs.MetaInput{ - ReqResp: request, - }) + metaInput := contextargs.NewMetaInput() + metaInput.ReqResp = request + metaInput.Input = request.URL.String() + return callback(metaInput) }) if err != nil { gologger.Warning().Msgf("Could not parse input file while iterating: %s\n", err) diff --git a/pkg/input/provider/interface.go b/pkg/input/provider/interface.go index cb48fc7676..b7ad4b4480 100644 --- a/pkg/input/provider/interface.go +++ b/pkg/input/provider/interface.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" configTypes "github.com/projectdiscovery/nuclei/v3/pkg/types" errorutil "github.com/projectdiscovery/utils/errors" + stringsutil "github.com/projectdiscovery/utils/strings" ) var ( @@ -32,7 +33,7 @@ func IsErrNotImplemented(err error) bool { if err == nil { return false } - if strings.Contains(err.Error(), "provider") && strings.Contains(err.Error(), "does not implement") { + if stringsutil.ContainsAll(err.Error(), "provider", "does not implement") { return true } return false diff --git a/pkg/input/provider/list/hmap.go b/pkg/input/provider/list/hmap.go index f7e3bd14c6..edf3729193 100644 --- a/pkg/input/provider/list/hmap.go +++ b/pkg/input/provider/list/hmap.go @@ -122,7 +122,7 @@ func (i *ListInputProvider) Iterate(callback func(value *contextargs.MetaInput) }) } callbackFunc := func(k, _ []byte) error { - metaInput := &contextargs.MetaInput{} + metaInput := contextargs.NewMetaInput() if err := metaInput.Unmarshal(string(k)); err != nil { return err } @@ -153,14 +153,16 @@ func (i *ListInputProvider) Set(value string) { } return fmt.Sprintf("got empty hostname for %v skipping ip selection", URL) }) - metaInput := &contextargs.MetaInput{Input: URL} + metaInput := contextargs.NewMetaInput() + metaInput.Input = URL i.setItem(metaInput) return } // Check if input is ip or hostname if iputil.IsIP(urlx.Hostname()) { - metaInput := &contextargs.MetaInput{Input: URL} + metaInput := contextargs.NewMetaInput() + metaInput.Input = URL i.setItem(metaInput) return } @@ -181,7 +183,9 @@ func (i *ListInputProvider) Set(value string) { if ip == "" { continue } - metaInput := &contextargs.MetaInput{Input: value, CustomIP: ip} + metaInput := contextargs.NewMetaInput() + metaInput.Input = value + metaInput.CustomIP = ip i.setItem(metaInput) } return @@ -211,11 +215,13 @@ func (i *ListInputProvider) Set(value string) { } for _, ip := range ips { + metaInput := contextargs.NewMetaInput() if ip != "" { - metaInput := &contextargs.MetaInput{Input: URL, CustomIP: ip} + metaInput.Input = URL + metaInput.CustomIP = ip i.setItem(metaInput) } else { - metaInput := &contextargs.MetaInput{Input: URL} + metaInput.Input = URL i.setItem(metaInput) } } @@ -353,7 +359,8 @@ func (i *ListInputProvider) scanInputFromReader(reader io.Reader) { // isExcluded checks if a URL is in the exclusion list func (i *ListInputProvider) isExcluded(URL string) bool { - metaInput := &contextargs.MetaInput{Input: URL} + metaInput := contextargs.NewMetaInput() + metaInput.Input = URL key, err := metaInput.MarshalString() if err != nil { gologger.Warning().Msgf("%s\n", err) @@ -378,14 +385,16 @@ func (i *ListInputProvider) Del(value string) { } return fmt.Sprintf("got empty hostname for %v skipping ip selection", URL) }) - metaInput := &contextargs.MetaInput{Input: URL} + metaInput := contextargs.NewMetaInput() + metaInput.Input = URL i.delItem(metaInput) return } // Check if input is ip or hostname if iputil.IsIP(urlx.Hostname()) { - metaInput := &contextargs.MetaInput{Input: URL} + metaInput := contextargs.NewMetaInput() + metaInput.Input = URL i.delItem(metaInput) return } @@ -406,7 +415,9 @@ func (i *ListInputProvider) Del(value string) { if ip == "" { continue } - metaInput := &contextargs.MetaInput{Input: value, CustomIP: ip} + metaInput := contextargs.NewMetaInput() + metaInput.Input = value + metaInput.CustomIP = ip i.delItem(metaInput) } return @@ -436,11 +447,13 @@ func (i *ListInputProvider) Del(value string) { } for _, ip := range ips { + metaInput := contextargs.NewMetaInput() if ip != "" { - metaInput := &contextargs.MetaInput{Input: URL, CustomIP: ip} + metaInput.Input = URL + metaInput.CustomIP = ip i.delItem(metaInput) } else { - metaInput := &contextargs.MetaInput{Input: URL} + metaInput.Input = URL i.delItem(metaInput) } } @@ -514,7 +527,8 @@ func (i *ListInputProvider) addTargets(targets []string) { func (i *ListInputProvider) removeTargets(targets []string) { for _, target := range targets { - metaInput := &contextargs.MetaInput{Input: target} + metaInput := contextargs.NewMetaInput() + metaInput.Input = target i.delItem(metaInput) } } diff --git a/pkg/input/provider/list/hmap_test.go b/pkg/input/provider/list/hmap_test.go index e8a7e8eea8..cd28b247a1 100644 --- a/pkg/input/provider/list/hmap_test.go +++ b/pkg/input/provider/list/hmap_test.go @@ -40,7 +40,7 @@ func Test_expandCIDR(t *testing.T) { // scan got := []string{} input.hostMap.Scan(func(k, _ []byte) error { - var metainput contextargs.MetaInput + metainput := contextargs.NewMetaInput() if err := metainput.Unmarshal(string(k)); err != nil { return err } @@ -141,7 +141,7 @@ func Test_scanallips_normalizeStoreInputValue(t *testing.T) { // scan got := []string{} input.hostMap.Scan(func(k, v []byte) error { - var metainput contextargs.MetaInput + metainput := contextargs.NewMetaInput() if err := metainput.Unmarshal(string(k)); err != nil { return err } @@ -184,7 +184,7 @@ func Test_expandASNInputValue(t *testing.T) { // scan the hmap got := []string{} input.hostMap.Scan(func(k, v []byte) error { - var metainput contextargs.MetaInput + metainput := contextargs.NewMetaInput() if err := metainput.Unmarshal(string(k)); err != nil { return err } diff --git a/pkg/input/provider/simple.go b/pkg/input/provider/simple.go index 71b280c2f5..c85f7871b9 100644 --- a/pkg/input/provider/simple.go +++ b/pkg/input/provider/simple.go @@ -43,7 +43,9 @@ func (s *SimpleInputProvider) Iterate(callback func(value *contextargs.MetaInput // Set adds an item to the input provider func (s *SimpleInputProvider) Set(value string) { - s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: value}) + metaInput := contextargs.NewMetaInput() + metaInput.Input = value + s.Inputs = append(s.Inputs, metaInput) } // SetWithProbe adds an item to the input provider with HTTP probing @@ -52,13 +54,17 @@ func (s *SimpleInputProvider) SetWithProbe(value string, probe types.InputLivene if err != nil { return err } - s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: probedValue}) + metaInput := contextargs.NewMetaInput() + metaInput.Input = probedValue + s.Inputs = append(s.Inputs, metaInput) return nil } // SetWithExclusions adds an item to the input provider if it doesn't match any of the exclusions func (s *SimpleInputProvider) SetWithExclusions(value string) error { - s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: value}) + metaInput := contextargs.NewMetaInput() + metaInput.Input = value + s.Inputs = append(s.Inputs, metaInput) return nil } diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go index 367bd013b2..f50e44ff2a 100644 --- a/pkg/js/compiler/compiler.go +++ b/pkg/js/compiler/compiler.go @@ -4,11 +4,11 @@ package compiler import ( "context" "fmt" - "time" "github.com/dop251/goja" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v3/pkg/types" contextutil "github.com/projectdiscovery/utils/context" "github.com/projectdiscovery/utils/errkit" stringsutil "github.com/projectdiscovery/utils/strings" @@ -38,13 +38,13 @@ type ExecuteOptions struct { // Cleanup is extra cleanup function to be called after execution Cleanup func(runtime *goja.Runtime) - /// Timeout for this script execution - Timeout int // Source is original source of the script Source *string Context context.Context + TimeoutVariants *types.Timeouts + // Manually exported objects exports map[string]interface{} } @@ -79,15 +79,6 @@ func (e ExecuteResult) GetSuccess() bool { return val } -// Execute executes a script with the default options. -func (c *Compiler) Execute(code string, args *ExecuteArgs) (ExecuteResult, error) { - p, err := WrapScriptNCompile(code, false) - if err != nil { - return nil, err - } - return c.ExecuteWithOptions(p, args, &ExecuteOptions{Context: context.Background()}) -} - // ExecuteWithOptions executes a script with the provided options. func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, opts *ExecuteOptions) (ExecuteResult, error) { if opts == nil { @@ -106,14 +97,9 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, // merge all args into templatectx args.TemplateCtx = generators.MergeMaps(args.TemplateCtx, args.Args) - if opts.Timeout <= 0 || opts.Timeout > 180 { - // some js scripts can take longer time so allow configuring timeout - // from template but keep it within sane limits (180s) - opts.Timeout = JsProtocolTimeout - } - // execute with context and timeout - ctx, cancel := context.WithTimeoutCause(opts.Context, time.Duration(opts.Timeout)*time.Second, ErrJSExecDeadline) + + ctx, cancel := context.WithTimeoutCause(opts.Context, opts.TimeoutVariants.JsCompilerExecutionTimeout, ErrJSExecDeadline) defer cancel() // execute the script results, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) { diff --git a/pkg/js/compiler/compiler_test.go b/pkg/js/compiler/compiler_test.go index ca09e7782e..3a43a4016e 100644 --- a/pkg/js/compiler/compiler_test.go +++ b/pkg/js/compiler/compiler_test.go @@ -1,11 +1,14 @@ package compiler import ( + "context" "strings" "testing" + "time" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/nuclei/v3/pkg/types" ) func TestNewCompilerConsoleDebug(t *testing.T) { @@ -18,23 +21,19 @@ func TestNewCompilerConsoleDebug(t *testing.T) { }) compiler := New() - _, err := compiler.Execute("console.log('hello world');", NewExecuteArgs()) + p, err := WrapScriptNCompile("console.log('hello world');", false) if err != nil { t.Fatal(err) } - if !strings.HasSuffix(gotString, "hello world") { - t.Fatalf("console.log not working, got=%v", gotString) - } -} -func TestExecuteResultGetSuccess(t *testing.T) { - compiler := New() - result, err := compiler.Execute("1+1 == 2", NewExecuteArgs()) + _, err = compiler.ExecuteWithOptions(p, NewExecuteArgs(), &ExecuteOptions{Context: context.Background(), + TimeoutVariants: &types.Timeouts{JsCompilerExecutionTimeout: time.Duration(20) * time.Second}}, + ) if err != nil { t.Fatal(err) } - if result.GetSuccess() != true { - t.Fatalf("expected true, got=%v", result.GetSuccess()) + if !strings.HasSuffix(gotString, "hello world") { + t.Fatalf("console.log not working, got=%v", gotString) } } diff --git a/pkg/js/compiler/init.go b/pkg/js/compiler/init.go index 8a171dc3da..92301df5e4 100644 --- a/pkg/js/compiler/init.go +++ b/pkg/js/compiler/init.go @@ -7,26 +7,17 @@ import ( // jsprotocolInit var ( - // Per Execution Javascript timeout in seconds - JsProtocolTimeout = 10 PoolingJsVmConcurrency = 100 NonPoolingVMConcurrency = 20 - JsTimeoutMultiplier = 1.5 ) // Init initializes the javascript protocol func Init(opts *types.Options) error { - if opts.Timeout < 10 { - // keep existing 10s timeout - return nil - } + if opts.JsConcurrency < 100 { // 100 is reasonable default opts.JsConcurrency = 100 } - // we have dialer timeout set to 10s so js needs to be at least - // 15s to return the actual error if not it will be a dialer timeout - JsProtocolTimeout = int(float64(opts.Timeout) * JsTimeoutMultiplier) PoolingJsVmConcurrency = opts.JsConcurrency PoolingJsVmConcurrency -= NonPoolingVMConcurrency return nil diff --git a/pkg/js/compiler/pool.go b/pkg/js/compiler/pool.go index 17d9f746f5..3407d97369 100644 --- a/pkg/js/compiler/pool.go +++ b/pkg/js/compiler/pool.go @@ -241,5 +241,5 @@ func stringify(gojaValue goja.Value, runtime *goja.Runtime) string { } } // for everything else stringify - return fmt.Sprintf("%v", value) + return fmt.Sprintf("%+v", value) } diff --git a/pkg/js/devtools/scrapefuncs/README.md b/pkg/js/devtools/scrapefuncs/README.md index de7c29ce2c..ecde2d5ab4 100644 --- a/pkg/js/devtools/scrapefuncs/README.md +++ b/pkg/js/devtools/scrapefuncs/README.md @@ -42,7 +42,11 @@ Description: getNetworkPort registers defaultPort and returns defaultPort if it Name: isPortOpen Signatures: "isPortOpen(host string, port string, [timeout int]) bool" -Description: isPortOpen checks if given port is open on host. timeout is optional and defaults to 5 seconds +Description: isPortOpen checks if given TCP port is open on host. timeout is optional and defaults to 5 seconds + +Name: isUDPPortOpen +Signatures: "isUDPPortOpen(host string, port string, [timeout int]) bool" +Description: isUDPPortOpen checks if the given UDP port is open on the host. Timeout is optional and defaults to 5 seconds. Name: ToBytes Signatures: "ToBytes(...interface{}) []byte" diff --git a/pkg/js/generated/go/libldap/ldap.go b/pkg/js/generated/go/libldap/ldap.go index 8d0c2702e4..978ded0c09 100644 --- a/pkg/js/generated/go/libldap/ldap.go +++ b/pkg/js/generated/go/libldap/ldap.go @@ -53,10 +53,12 @@ func init() { "FilterWorkstationTrustAccount": lib_ldap.FilterWorkstationTrustAccount, // Objects / Classes - "ADObject": gojs.GetClassConstructor[lib_ldap.ADObject](&lib_ldap.ADObject{}), - "Client": lib_ldap.NewClient, - "Config": gojs.GetClassConstructor[lib_ldap.Config](&lib_ldap.Config{}), - "Metadata": gojs.GetClassConstructor[lib_ldap.Metadata](&lib_ldap.Metadata{}), + "Client": lib_ldap.NewClient, + "Config": gojs.GetClassConstructor[lib_ldap.Config](&lib_ldap.Config{}), + "LdapAttributes": gojs.GetClassConstructor[lib_ldap.LdapAttributes](&lib_ldap.LdapAttributes{}), + "LdapEntry": gojs.GetClassConstructor[lib_ldap.LdapEntry](&lib_ldap.LdapEntry{}), + "Metadata": gojs.GetClassConstructor[lib_ldap.Metadata](&lib_ldap.Metadata{}), + "SearchResult": gojs.GetClassConstructor[lib_ldap.SearchResult](&lib_ldap.SearchResult{}), }, ).Register() } diff --git a/pkg/js/generated/ts/kerberos.ts b/pkg/js/generated/ts/kerberos.ts index 8f9ce98f99..d0af2d0e7e 100755 --- a/pkg/js/generated/ts/kerberos.ts +++ b/pkg/js/generated/ts/kerberos.ts @@ -188,9 +188,9 @@ export class Config { */ export interface AuthorizationDataEntry { - ADType?: number, - ADData?: Uint8Array, + + ADType?: number, } @@ -200,9 +200,9 @@ export interface AuthorizationDataEntry { */ export interface BitString { - BitLength?: number, - Bytes?: Uint8Array, + + BitLength?: number, } @@ -212,9 +212,9 @@ export interface BitString { */ export interface BitString { - BitLength?: number, - Bytes?: Uint8Array, + + BitLength?: number, } @@ -236,19 +236,15 @@ export interface Config { */ export interface EncTicketPart { - AuthTime?: Date, - - StartTime?: Date, - EndTime?: Date, RenewTill?: Date, CRealm?: string, - CAddr?: HostAddress, + AuthTime?: Date, - AuthorizationData?: AuthorizationDataEntry, + StartTime?: Date, Flags?: BitString, @@ -257,6 +253,10 @@ export interface EncTicketPart { CName?: PrincipalName, Transited?: TransitedEncoding, + + CAddr?: HostAddress, + + AuthorizationData?: AuthorizationDataEntry, } @@ -266,11 +266,11 @@ export interface EncTicketPart { */ export interface EncryptedData { + EType?: number, + KVNO?: number, Cipher?: Uint8Array, - - EType?: number, } @@ -318,41 +318,33 @@ export interface HostAddress { */ export interface LibDefaults { - NoAddresses?: boolean, + CCacheType?: number, - RealmTryDomains?: number, + K5LoginAuthoritative?: boolean, - DNSLookupKDC?: boolean, + Proxiable?: boolean, - DefaultRealm?: string, + RDNS?: boolean, - SafeChecksumType?: number, + K5LoginDirectory?: string, - VerifyAPReqNofail?: boolean, + KDCTimeSync?: number, - AllowWeakCrypto?: boolean, + VerifyAPReqNofail?: boolean, DefaultTGSEnctypes?: string[], - DefaultTktEnctypeIDs?: number[], - - IgnoreAcceptorHostname?: boolean, + DefaultTGSEnctypeIDs?: number[], - K5LoginAuthoritative?: boolean, + DNSCanonicalizeHostname?: boolean, - PermittedEnctypes?: string[], + Forwardable?: boolean, /** * time in nanoseconds */ - Clockskew?: number, - - DNSCanonicalizeHostname?: boolean, - - Proxiable?: boolean, - - RDNS?: boolean, + RenewLifetime?: number, /** * time in nanoseconds @@ -362,37 +354,45 @@ export interface LibDefaults { DefaultClientKeytabName?: string, + DefaultTktEnctypeIDs?: number[], + + DNSLookupRealm?: boolean, + + ExtraAddresses?: Uint8Array, + + DefaultRealm?: string, + + NoAddresses?: boolean, + + PreferredPreauthTypes?: number[], + PermittedEnctypeIDs?: number[], - UDPPreferenceLimit?: number, + RealmTryDomains?: number, - DefaultTGSEnctypeIDs?: number[], + DefaultKeytabName?: string, DefaultTktEnctypes?: string[], - CCacheType?: number, - - DNSLookupRealm?: boolean, + DNSLookupKDC?: boolean, - ExtraAddresses?: Uint8Array, + IgnoreAcceptorHostname?: boolean, - PreferredPreauthTypes?: number[], + AllowWeakCrypto?: boolean, Canonicalize?: boolean, - Forwardable?: boolean, - - K5LoginDirectory?: string, + SafeChecksumType?: number, - KDCTimeSync?: number, + UDPPreferenceLimit?: number, /** * time in nanoseconds */ - RenewLifetime?: number, + Clockskew?: number, - DefaultKeytabName?: string, + PermittedEnctypes?: string[], KDCDefaultOptions?: BitString, } @@ -416,6 +416,8 @@ export interface PrincipalName { */ export interface Realm { + Realm?: string, + AdminServer?: string[], DefaultDomain?: string, @@ -425,8 +427,6 @@ export interface Realm { KPasswdServer?: string[], MasterKDC?: string[], - - Realm?: string, } diff --git a/pkg/js/generated/ts/ldap.ts b/pkg/js/generated/ts/ldap.ts index daca4f86b9..02a10075b9 100755 --- a/pkg/js/generated/ts/ldap.ts +++ b/pkg/js/generated/ts/ldap.ts @@ -209,8 +209,8 @@ export class Client { * log(to_json(users)); * ``` */ - public FindADObjects(filter: string): ADObject[] { - return []; + public FindADObjects(filter: string): SearchResult | null { + return null; } @@ -225,8 +225,8 @@ export class Client { * log(to_json(users)); * ``` */ - public GetADUsers(): ADObject[] { - return []; + public GetADUsers(): SearchResult | null { + return null; } @@ -241,8 +241,8 @@ export class Client { * log(to_json(users)); * ``` */ - public GetADActiveUsers(): ADObject[] { - return []; + public GetADActiveUsers(): SearchResult | null { + return null; } @@ -257,8 +257,8 @@ export class Client { * log(to_json(users)); * ``` */ - public GetADUserWithNeverExpiringPasswords(): ADObject[] { - return []; + public GetADUserWithNeverExpiringPasswords(): SearchResult | null { + return null; } @@ -273,8 +273,8 @@ export class Client { * log(to_json(users)); * ``` */ - public GetADUserTrustedForDelegation(): ADObject[] { - return []; + public GetADUserTrustedForDelegation(): SearchResult | null { + return null; } @@ -289,8 +289,8 @@ export class Client { * log(to_json(users)); * ``` */ - public GetADUserWithPasswordNotRequired(): ADObject[] { - return []; + public GetADUserWithPasswordNotRequired(): SearchResult | null { + return null; } @@ -305,8 +305,8 @@ export class Client { * log(to_json(groups)); * ``` */ - public GetADGroups(): ADObject[] { - return []; + public GetADGroups(): SearchResult | null { + return null; } @@ -321,8 +321,8 @@ export class Client { * log(to_json(dcs)); * ``` */ - public GetADDCList(): ADObject[] { - return []; + public GetADDCList(): SearchResult | null { + return null; } @@ -337,8 +337,8 @@ export class Client { * log(to_json(admins)); * ``` */ - public GetADAdmins(): ADObject[] { - return []; + public GetADAdmins(): SearchResult | null { + return null; } @@ -353,8 +353,8 @@ export class Client { * log(to_json(kerberoastable)); * ``` */ - public GetADUserKerberoastable(): ADObject[] { - return []; + public GetADUserKerberoastable(): SearchResult | null { + return null; } @@ -369,8 +369,8 @@ export class Client { * log(to_json(AsRepRoastable)); * ``` */ - public GetADUserAsRepRoastable(): ADObject[] { - return []; + public GetADUserAsRepRoastable(): SearchResult | null { + return null; } @@ -428,8 +428,8 @@ export class Client { * const results = client.Search('(objectClass=*)', 'cn', 'mail'); * ``` */ - public Search(filter: string, attributes: any): Record[] { - return []; + public Search(filter: string, attributes: any): SearchResult | null { + return null; } @@ -481,33 +481,6 @@ export class Client { -/** - * ADObject represents an Active Directory object - * @example - * ```javascript - * const ldap = require('nuclei/ldap'); - * const client = new ldap.Client('ldap://ldap.example.com', 'acme.com'); - * const users = client.GetADUsers(); - * log(to_json(users)); - * ``` - */ -export interface ADObject { - - DistinguishedName?: string, - - SAMAccountName?: string, - - PWDLastSet?: string, - - LastLogon?: string, - - MemberOf?: string[], - - ServicePrincipalName?: string[], -} - - - /** * Config is extra configuration for the ldap client * @example @@ -535,27 +508,186 @@ export interface Config { /** - * Entry Interface + * LdapAttributes represents all LDAP attributes of a particular + * ldap entry */ -export interface Entry { +export interface LdapAttributes { - DN?: string, + /** + * CurrentTime contains current time + */ + + CurrentTime?: string[], + + /** + * SubschemaSubentry contains subschema subentry + */ + + SubschemaSubentry?: string[], + + /** + * DsServiceName contains ds service name + */ + + DsServiceName?: string[], + + /** + * NamingContexts contains naming contexts + */ + + NamingContexts?: string[], + + /** + * DefaultNamingContext contains default naming context + */ + + DefaultNamingContext?: string[], + + /** + * SchemaNamingContext contains schema naming context + */ + + SchemaNamingContext?: string[], + + /** + * ConfigurationNamingContext contains configuration naming context + */ + + ConfigurationNamingContext?: string[], + + /** + * RootDomainNamingContext contains root domain naming context + */ + + RootDomainNamingContext?: string[], + + /** + * SupportedLDAPVersion contains supported LDAP version + */ + + SupportedLDAPVersion?: string[], + + /** + * HighestCommittedUSN contains highest committed USN + */ + + HighestCommittedUSN?: string[], + + /** + * SupportedSASLMechanisms contains supported SASL mechanisms + */ + + SupportedSASLMechanisms?: string[], + + /** + * DnsHostName contains DNS host name + */ + + DnsHostName?: string[], - Attributes?: EntryAttribute, + /** + * LdapServiceName contains LDAP service name + */ + + LdapServiceName?: string[], + + /** + * ServerName contains server name + */ + + ServerName?: string[], + + /** + * IsSynchronized contains is synchronized + */ + + IsSynchronized?: string[], + + /** + * IsGlobalCatalogReady contains is global catalog ready + */ + + IsGlobalCatalogReady?: string[], + + /** + * DomainFunctionality contains domain functionality + */ + + DomainFunctionality?: string[], + + /** + * ForestFunctionality contains forest functionality + */ + + ForestFunctionality?: string[], + + /** + * DomainControllerFunctionality contains domain controller functionality + */ + + DomainControllerFunctionality?: string[], + + /** + * DistinguishedName contains the distinguished name + */ + + DistinguishedName?: string[], + + /** + * SAMAccountName contains the SAM account name + */ + + SAMAccountName?: string[], + + /** + * PWDLastSet contains the password last set time + */ + + PWDLastSet?: string[], + + /** + * LastLogon contains the last logon time + */ + + LastLogon?: string[], + + /** + * MemberOf contains the groups the entry is a member of + */ + + MemberOf?: string[], + + /** + * ServicePrincipalName contains the service principal names + */ + + ServicePrincipalName?: string[], + + /** + * Extra contains other extra fields which might be present + */ + + Extra?: Record, } /** - * EntryAttribute Interface + * LdapEntry represents a single LDAP entry */ -export interface EntryAttribute { +export interface LdapEntry { - Name?: string, + /** + * DN contains distinguished name + */ - Values?: string[], + DN?: string, - ByteValues?: Uint8Array, + /** + * Attributes contains list of attributes + */ + + Attributes?: LdapAttributes, } @@ -584,12 +716,32 @@ export interface Metadata { /** - * SearchResult Interface + * SearchResult contains search result of any / all ldap search request + * @example + * ```javascript + * const ldap = require('nuclei/ldap'); + * const client = new ldap.Client('ldap://ldap.example.com', 'acme.com'); + * const results = client.Search('(objectinterface=*)', 'cn', 'mail'); + * ``` */ export interface SearchResult { + /** + * Referrals contains list of referrals + */ + Referrals?: string[], - Entries?: Entry, + /** + * Controls contains list of controls + */ + + Controls?: string[], + + /** + * Entries contains list of entries + */ + + Entries?: LdapEntry[], } diff --git a/pkg/js/generated/ts/rdp.ts b/pkg/js/generated/ts/rdp.ts index 77dfeafb13..7858b78319 100755 --- a/pkg/js/generated/ts/rdp.ts +++ b/pkg/js/generated/ts/rdp.ts @@ -78,8 +78,6 @@ export interface IsRDPResponse { */ export interface ServiceRDP { - DNSDomainName?: string, - ForestName?: string, OSFingerprint?: string, @@ -93,5 +91,7 @@ export interface ServiceRDP { NetBIOSDomainName?: string, DNSComputerName?: string, + + DNSDomainName?: string, } diff --git a/pkg/js/generated/ts/smb.ts b/pkg/js/generated/ts/smb.ts index b0ca997ac5..c9b769cb5d 100755 --- a/pkg/js/generated/ts/smb.ts +++ b/pkg/js/generated/ts/smb.ts @@ -137,6 +137,10 @@ export interface NegotiationLog { */ export interface SMBCapabilities { + DFSSupport?: boolean, + + Leasing?: boolean, + LargeMTU?: boolean, MultiChan?: boolean, @@ -146,10 +150,6 @@ export interface SMBCapabilities { DirLeasing?: boolean, Encryption?: boolean, - - DFSSupport?: boolean, - - Leasing?: boolean, } @@ -159,16 +159,16 @@ export interface SMBCapabilities { */ export interface SMBLog { - SupportV1?: boolean, - - NativeOs?: string, - NTLM?: string, GroupName?: string, HasNTLM?: boolean, + SupportV1?: boolean, + + NativeOs?: string, + Version?: SMBVersions, Capabilities?: SMBCapabilities, @@ -185,13 +185,13 @@ export interface SMBLog { */ export interface SMBVersions { - VerString?: string, - Major?: number, Minor?: number, Revision?: number, + + VerString?: string, } @@ -225,12 +225,12 @@ export interface ServiceSMB { */ export interface SessionSetupLog { - NegotiateFlags?: number, - SetupFlags?: number, TargetName?: string, + NegotiateFlags?: number, + HeaderLog?: HeaderLog, } diff --git a/pkg/js/generated/ts/ssh.ts b/pkg/js/generated/ts/ssh.ts index 21dacf4054..058df03c5b 100755 --- a/pkg/js/generated/ts/ssh.ts +++ b/pkg/js/generated/ts/ssh.ts @@ -133,9 +133,9 @@ export interface Algorithms { HostKey?: string, - R?: DirectionAlgorithms, - W?: DirectionAlgorithms, + + R?: DirectionAlgorithms, } @@ -159,13 +159,13 @@ export interface DirectionAlgorithms { */ export interface EndpointId { - Raw?: string, - - ProtoVersion?: string, - SoftwareVersion?: string, Comment?: string, + + Raw?: string, + + ProtoVersion?: string, } @@ -197,34 +197,34 @@ export interface HandshakeLog { */ export interface KexInitMsg { - Reserved?: number, - - /** - * fixed size array of length: [16] - */ - - Cookie?: Uint8Array, + KexAlgos?: string[], CiphersClientServer?: string[], - MACsClientServer?: string[], - MACsServerClient?: string[], + LanguagesClientServer?: string[], + + CompressionClientServer?: string[], + CompressionServerClient?: string[], - LanguagesClientServer?: string[], + Reserved?: number, - FirstKexFollows?: boolean, + MACsClientServer?: string[], - KexAlgos?: string[], + /** + * fixed size array of length: [16] + */ - CiphersServerClient?: string[], + Cookie?: Uint8Array, - CompressionClientServer?: string[], + ServerHostKeyAlgos?: string[], + + CiphersServerClient?: string[], LanguagesServerClient?: string[], - ServerHostKeyAlgos?: string[], + FirstKexFollows?: boolean, } diff --git a/pkg/js/global/scripts.go b/pkg/js/global/scripts.go index 64d6b25c17..c6771fadf3 100644 --- a/pkg/js/global/scripts.go +++ b/pkg/js/global/scripts.go @@ -109,7 +109,7 @@ func initBuiltInFunc(runtime *goja.Runtime) { Signatures: []string{ "isPortOpen(host string, port string, [timeout int]) bool", }, - Description: "isPortOpen checks if given port is open on host. timeout is optional and defaults to 5 seconds", + Description: "isPortOpen checks if given TCP port is open on host. timeout is optional and defaults to 5 seconds", FuncDecl: func(host string, port string, timeout ...int) (bool, error) { timeoutInSec := 5 if len(timeout) > 0 { @@ -124,6 +124,27 @@ func initBuiltInFunc(runtime *goja.Runtime) { }, }) + _ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "isUDPPortOpen", + Signatures: []string{ + "isUDPPortOpen(host string, port string, [timeout int]) bool", + }, + Description: "isUDPPortOpen checks if the given UDP port is open on the host. Timeout is optional and defaults to 5 seconds.", + FuncDecl: func(host string, port string, timeout ...int) (bool, error) { + timeoutInSec := 5 + if len(timeout) > 0 { + timeoutInSec = timeout[0] + } + conn, err := net.DialTimeout("udp", net.JoinHostPort(host, port), time.Duration(timeoutInSec)*time.Second) + if err != nil { + return false, err + } + _ = conn.Close() + + return true, nil + }, + }) + _ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ Name: "ToBytes", Signatures: []string{ diff --git a/pkg/js/libs/ldap/adenum.go b/pkg/js/libs/ldap/adenum.go index 9aea98be97..849e4d6a15 100644 --- a/pkg/js/libs/ldap/adenum.go +++ b/pkg/js/libs/ldap/adenum.go @@ -74,25 +74,6 @@ func NegativeFilter(filter string) string { return fmt.Sprintf("(!%s)", filter) } -type ( - // ADObject represents an Active Directory object - // @example - // ```javascript - // const ldap = require('nuclei/ldap'); - // const client = new ldap.Client('ldap://ldap.example.com', 'acme.com'); - // const users = client.GetADUsers(); - // log(to_json(users)); - // ``` - ADObject struct { - DistinguishedName string - SAMAccountName string - PWDLastSet string - LastLogon string - MemberOf []string - ServicePrincipalName []string - } -) - // FindADObjects finds AD objects based on a filter // and returns them as a list of ADObject // @example @@ -102,7 +83,7 @@ type ( // const users = client.FindADObjects(ldap.FilterIsPerson); // log(to_json(users)); // ``` -func (c *Client) FindADObjects(filter string) []ADObject { +func (c *Client) FindADObjects(filter string) SearchResult { c.nj.Require(c.conn != nil, "no existing connection") sr := ldap.NewSearchRequest( c.BaseDN, ldap.ScopeWholeSubtree, @@ -121,19 +102,7 @@ func (c *Client) FindADObjects(filter string) []ADObject { res, err := c.conn.Search(sr) c.nj.HandleError(err, "ldap search request failed") - - var objects []ADObject - for _, obj := range res.Entries { - objects = append(objects, ADObject{ - DistinguishedName: obj.GetAttributeValue("distinguishedName"), - SAMAccountName: obj.GetAttributeValue("sAMAccountName"), - PWDLastSet: DecodeADTimestamp(obj.GetAttributeValue("pwdLastSet")), - LastLogon: DecodeADTimestamp(obj.GetAttributeValue("lastLogon")), - MemberOf: obj.GetAttributeValues("memberOf"), - ServicePrincipalName: obj.GetAttributeValues("servicePrincipalName"), - }) - } - return objects + return *getSearchResult(res) } // GetADUsers returns all AD users @@ -145,7 +114,7 @@ func (c *Client) FindADObjects(filter string) []ADObject { // const users = client.GetADUsers(); // log(to_json(users)); // ``` -func (c *Client) GetADUsers() []ADObject { +func (c *Client) GetADUsers() SearchResult { return c.FindADObjects(FilterIsPerson) } @@ -158,7 +127,7 @@ func (c *Client) GetADUsers() []ADObject { // const users = client.GetADActiveUsers(); // log(to_json(users)); // ``` -func (c *Client) GetADActiveUsers() []ADObject { +func (c *Client) GetADActiveUsers() SearchResult { return c.FindADObjects(JoinFilters(FilterIsPerson, FilterAccountEnabled)) } @@ -171,7 +140,7 @@ func (c *Client) GetADActiveUsers() []ADObject { // const users = client.GetADUserWithNeverExpiringPasswords(); // log(to_json(users)); // ``` -func (c *Client) GetADUserWithNeverExpiringPasswords() []ADObject { +func (c *Client) GetADUserWithNeverExpiringPasswords() SearchResult { return c.FindADObjects(JoinFilters(FilterIsPerson, FilterDontExpirePassword)) } @@ -184,7 +153,7 @@ func (c *Client) GetADUserWithNeverExpiringPasswords() []ADObject { // const users = client.GetADUserTrustedForDelegation(); // log(to_json(users)); // ``` -func (c *Client) GetADUserTrustedForDelegation() []ADObject { +func (c *Client) GetADUserTrustedForDelegation() SearchResult { return c.FindADObjects(JoinFilters(FilterIsPerson, FilterTrustedForDelegation)) } @@ -197,7 +166,7 @@ func (c *Client) GetADUserTrustedForDelegation() []ADObject { // const users = client.GetADUserWithPasswordNotRequired(); // log(to_json(users)); // ``` -func (c *Client) GetADUserWithPasswordNotRequired() []ADObject { +func (c *Client) GetADUserWithPasswordNotRequired() SearchResult { return c.FindADObjects(JoinFilters(FilterIsPerson, FilterPasswordNotRequired)) } @@ -210,7 +179,7 @@ func (c *Client) GetADUserWithPasswordNotRequired() []ADObject { // const groups = client.GetADGroups(); // log(to_json(groups)); // ``` -func (c *Client) GetADGroups() []ADObject { +func (c *Client) GetADGroups() SearchResult { return c.FindADObjects(FilterIsGroup) } @@ -223,7 +192,7 @@ func (c *Client) GetADGroups() []ADObject { // const dcs = client.GetADDCList(); // log(to_json(dcs)); // ``` -func (c *Client) GetADDCList() []ADObject { +func (c *Client) GetADDCList() SearchResult { return c.FindADObjects(JoinFilters(FilterIsComputer, FilterAccountEnabled, FilterServerTrustAccount)) } @@ -236,7 +205,7 @@ func (c *Client) GetADDCList() []ADObject { // const admins = client.GetADAdmins(); // log(to_json(admins)); // ``` -func (c *Client) GetADAdmins() []ADObject { +func (c *Client) GetADAdmins() SearchResult { return c.FindADObjects(JoinFilters(FilterIsPerson, FilterAccountEnabled, FilterIsAdmin)) } @@ -249,7 +218,7 @@ func (c *Client) GetADAdmins() []ADObject { // const kerberoastable = client.GetADUserKerberoastable(); // log(to_json(kerberoastable)); // ``` -func (c *Client) GetADUserKerberoastable() []ADObject { +func (c *Client) GetADUserKerberoastable() SearchResult { return c.FindADObjects(JoinFilters(FilterIsPerson, FilterAccountEnabled, FilterHasServicePrincipalName)) } @@ -262,7 +231,7 @@ func (c *Client) GetADUserKerberoastable() []ADObject { // const AsRepRoastable = client.GetADUserAsRepRoastable(); // log(to_json(AsRepRoastable)); // ``` -func (c *Client) GetADUserAsRepRoastable() []ADObject { +func (c *Client) GetADUserAsRepRoastable() SearchResult { return c.FindADObjects(JoinFilters(FilterIsPerson, FilterDontRequirePreauth)) } @@ -276,7 +245,16 @@ func (c *Client) GetADUserAsRepRoastable() []ADObject { // ``` func (c *Client) GetADDomainSID() string { r := c.Search(FilterServerTrustAccount, "objectSid") - c.nj.Require(len(r) > 0, "no result from GetADDomainSID query") - c.nj.Require(len(r[0]["objectSid"]) > 0, "could not grab DomainSID") - return DecodeSID(r[0]["objectSid"][0]) + c.nj.Require(len(r.Entries) > 0, "no result from GetADDomainSID query") + for _, entry := range r.Entries { + if sid, ok := entry.Attributes.Extra["objectSid"]; ok { + if sid, ok := sid.([]string); ok { + return DecodeSID(sid[0]) + } else { + c.nj.HandleError(fmt.Errorf("invalid objectSid type: %T", entry.Attributes.Extra["objectSid"]), "invalid objectSid type") + } + } + } + c.nj.HandleError(fmt.Errorf("no objectSid found"), "no objectSid found") + return "" } diff --git a/pkg/js/libs/ldap/ldap.go b/pkg/js/libs/ldap/ldap.go index 5f03611cb2..3961fa7b7e 100644 --- a/pkg/js/libs/ldap/ldap.go +++ b/pkg/js/libs/ldap/ldap.go @@ -203,38 +203,24 @@ func (c *Client) AuthenticateWithNTLMHash(username, hash string) { // const client = new ldap.Client('ldap://ldap.example.com', 'acme.com'); // const results = client.Search('(objectClass=*)', 'cn', 'mail'); // ``` -func (c *Client) Search(filter string, attributes ...string) []map[string][]string { +func (c *Client) Search(filter string, attributes ...string) SearchResult { c.nj.Require(c.conn != nil, "no existing connection") + c.nj.Require(c.BaseDN != "", "base dn cannot be empty") + c.nj.Require(len(attributes) > 0, "attributes cannot be empty") res, err := c.conn.Search( ldap.NewSearchRequest( - c.BaseDN, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, - 0, 0, false, filter, attributes, nil, + "", + ldap.ScopeWholeSubtree, + ldap.NeverDerefAliases, + 0, 0, false, + filter, + attributes, + nil, ), ) c.nj.HandleError(err, "ldap search request failed") - if len(res.Entries) == 0 { - // return empty list - return nil - } - - // convert ldap.Entry to []map[string][]string - var out []map[string][]string - for _, r := range res.Entries { - app := make(map[string][]string) - empty := true - for _, a := range attributes { - v := r.GetAttributeValues(a) - if len(v) > 0 { - app[a] = v - empty = false - } - } - if !empty { - out = append(out, app) - } - } - return out + return *getSearchResult(res) } // AdvancedSearch accepts all values of search request type and return Ldap Entry @@ -250,7 +236,7 @@ func (c *Client) AdvancedSearch( TypesOnly bool, Filter string, Attributes []string, - Controls []ldap.Control) ldap.SearchResult { + Controls []ldap.Control) SearchResult { c.nj.Require(c.conn != nil, "no existing connection") if c.BaseDN == "" { c.BaseDN = fmt.Sprintf("dc=%s", strings.Join(strings.Split(c.Realm, "."), ",dc=")) @@ -259,7 +245,7 @@ func (c *Client) AdvancedSearch( res, err := c.conn.Search(req) c.nj.HandleError(err, "ldap search request failed") c.nj.Require(res != nil, "ldap search request failed got nil response") - return *res + return *getSearchResult(res) } type ( @@ -295,7 +281,7 @@ func (c *Client) CollectMetadata() Metadata { srMetadata := ldap.NewSearchRequest( "", - ldap.ScopeBaseObject, + ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, "(objectClass=*)", diff --git a/pkg/js/libs/ldap/utils.go b/pkg/js/libs/ldap/utils.go index 96a243bbc7..bcff41dc49 100644 --- a/pkg/js/libs/ldap/utils.go +++ b/pkg/js/libs/ldap/utils.go @@ -5,8 +5,189 @@ import ( "strconv" "strings" "time" + + "github.com/go-ldap/ldap/v3" ) +type ( + // SearchResult contains search result of any / all ldap search request + // @example + // ```javascript + // const ldap = require('nuclei/ldap'); + // const client = new ldap.Client('ldap://ldap.example.com', 'acme.com'); + // const results = client.Search('(objectClass=*)', 'cn', 'mail'); + // ``` + SearchResult struct { + // Referrals contains list of referrals + Referrals []string `json:"referrals"` + // Controls contains list of controls + Controls []string `json:"controls"` + // Entries contains list of entries + Entries []LdapEntry `json:"entries"` + } + + // LdapEntry represents a single LDAP entry + LdapEntry struct { + // DN contains distinguished name + DN string `json:"dn"` + // Attributes contains list of attributes + Attributes LdapAttributes `json:"attributes"` + } + + // LdapAttributes represents all LDAP attributes of a particular + // ldap entry + LdapAttributes struct { + // CurrentTime contains current time + CurrentTime []string `json:"currentTime,omitempty"` + // SubschemaSubentry contains subschema subentry + SubschemaSubentry []string `json:"subschemaSubentry,omitempty"` + // DsServiceName contains ds service name + DsServiceName []string `json:"dsServiceName,omitempty"` + // NamingContexts contains naming contexts + NamingContexts []string `json:"namingContexts,omitempty"` + // DefaultNamingContext contains default naming context + DefaultNamingContext []string `json:"defaultNamingContext,omitempty"` + // SchemaNamingContext contains schema naming context + SchemaNamingContext []string `json:"schemaNamingContext,omitempty"` + // ConfigurationNamingContext contains configuration naming context + ConfigurationNamingContext []string `json:"configurationNamingContext,omitempty"` + // RootDomainNamingContext contains root domain naming context + RootDomainNamingContext []string `json:"rootDomainNamingContext,omitempty"` + // SupportedLDAPVersion contains supported LDAP version + SupportedLDAPVersion []string `json:"supportedLDAPVersion,omitempty"` + // HighestCommittedUSN contains highest committed USN + HighestCommittedUSN []string `json:"highestCommittedUSN,omitempty"` + // SupportedSASLMechanisms contains supported SASL mechanisms + SupportedSASLMechanisms []string `json:"supportedSASLMechanisms,omitempty"` + // DnsHostName contains DNS host name + DnsHostName []string `json:"dnsHostName,omitempty"` + // LdapServiceName contains LDAP service name + LdapServiceName []string `json:"ldapServiceName,omitempty"` + // ServerName contains server name + ServerName []string `json:"serverName,omitempty"` + // IsSynchronized contains is synchronized + IsSynchronized []string `json:"isSynchronized,omitempty"` + // IsGlobalCatalogReady contains is global catalog ready + IsGlobalCatalogReady []string `json:"isGlobalCatalogReady,omitempty"` + // DomainFunctionality contains domain functionality + DomainFunctionality []string `json:"domainFunctionality,omitempty"` + // ForestFunctionality contains forest functionality + ForestFunctionality []string `json:"forestFunctionality,omitempty"` + // DomainControllerFunctionality contains domain controller functionality + DomainControllerFunctionality []string `json:"domainControllerFunctionality,omitempty"` + // DistinguishedName contains the distinguished name + DistinguishedName []string `json:"distinguishedName,omitempty"` + // SAMAccountName contains the SAM account name + SAMAccountName []string `json:"sAMAccountName,omitempty"` + // PWDLastSet contains the password last set time + PWDLastSet []string `json:"pwdLastSet,omitempty"` + // LastLogon contains the last logon time + LastLogon []string `json:"lastLogon,omitempty"` + // MemberOf contains the groups the entry is a member of + MemberOf []string `json:"memberOf,omitempty"` + // ServicePrincipalName contains the service principal names + ServicePrincipalName []string `json:"servicePrincipalName,omitempty"` + // Extra contains other extra fields which might be present + Extra map[string]any `json:"extra,omitempty"` + } +) + +// getSearchResult converts a ldap.SearchResult to a SearchResult +func getSearchResult(sr *ldap.SearchResult) *SearchResult { + t := &SearchResult{ + Referrals: []string{}, + Controls: []string{}, + Entries: []LdapEntry{}, + } + // add referrals + t.Referrals = append(t.Referrals, sr.Referrals...) + // add controls + for _, ctrl := range sr.Controls { + t.Controls = append(t.Controls, ctrl.String()) + } + // add entries + for _, entry := range sr.Entries { + t.Entries = append(t.Entries, parseLdapEntry(entry)) + } + return t +} + +func parseLdapEntry(entry *ldap.Entry) LdapEntry { + e := LdapEntry{ + DN: entry.DN, + } + attrs := LdapAttributes{ + Extra: make(map[string]any), + } + for _, attr := range entry.Attributes { + switch attr.Name { + case "currentTime": + attrs.CurrentTime = decodeTimestamps(attr.Values) + case "subschemaSubentry": + attrs.SubschemaSubentry = attr.Values + case "dsServiceName": + attrs.DsServiceName = attr.Values + case "namingContexts": + attrs.NamingContexts = attr.Values + case "defaultNamingContext": + attrs.DefaultNamingContext = attr.Values + case "schemaNamingContext": + attrs.SchemaNamingContext = attr.Values + case "configurationNamingContext": + attrs.ConfigurationNamingContext = attr.Values + case "rootDomainNamingContext": + attrs.RootDomainNamingContext = attr.Values + case "supportedLDAPVersion": + attrs.SupportedLDAPVersion = attr.Values + case "highestCommittedUSN": + attrs.HighestCommittedUSN = attr.Values + case "supportedSASLMechanisms": + attrs.SupportedSASLMechanisms = attr.Values + case "dnsHostName": + attrs.DnsHostName = attr.Values + case "ldapServiceName": + attrs.LdapServiceName = attr.Values + case "serverName": + attrs.ServerName = attr.Values + case "isSynchronized": + attrs.IsSynchronized = attr.Values + case "isGlobalCatalogReady": + attrs.IsGlobalCatalogReady = attr.Values + case "domainFunctionality": + attrs.DomainFunctionality = attr.Values + case "forestFunctionality": + attrs.ForestFunctionality = attr.Values + case "domainControllerFunctionality": + attrs.DomainControllerFunctionality = attr.Values + case "distinguishedName": + attrs.DistinguishedName = attr.Values + case "sAMAccountName": + attrs.SAMAccountName = attr.Values + case "pwdLastSet": + attrs.PWDLastSet = decodeTimestamps(attr.Values) + case "lastLogon": + attrs.LastLogon = decodeTimestamps(attr.Values) + case "memberOf": + attrs.MemberOf = attr.Values + case "servicePrincipalName": + attrs.ServicePrincipalName = attr.Values + default: + attrs.Extra[attr.Name] = attr.Values + } + } + e.Attributes = attrs + return e +} + +// decodeTimestamps decodes multiple timestamps +func decodeTimestamps(timestamps []string) []string { + res := []string{} + for _, timestamp := range timestamps { + res = append(res, DecodeADTimestamp(timestamp)) + } + return res +} + // DecodeSID decodes a SID string // @example // ```javascript diff --git a/pkg/js/libs/mysql/mysql.go b/pkg/js/libs/mysql/mysql.go index 0fd2f24200..4566059260 100644 --- a/pkg/js/libs/mysql/mysql.go +++ b/pkg/js/libs/mysql/mysql.go @@ -36,6 +36,7 @@ type ( // const isMySQL = mysql.IsMySQL('acme.com', 3306); // ``` func (c *MySQLClient) IsMySQL(host string, port int) (bool, error) { + // todo: why this is exposed? Service fingerprint should be automatic return memoizedisMySQL(host, port) } @@ -77,6 +78,16 @@ func (c *MySQLClient) Connect(host string, port int, username, password string) // host is not valid according to network policy return false, protocolstate.ErrHostDenied.Msgf(host) } + + // executing queries implies the remote mysql service + ok, err := c.IsMySQL(host, port) + if err != nil { + return false, err + } + if !ok { + return false, fmt.Errorf("not a mysql service") + } + dsn, err := BuildDSN(MySQLOptions{ Host: host, Port: port, @@ -182,6 +193,16 @@ func (c *MySQLClient) ExecuteQueryWithOpts(opts MySQLOptions, query string) (*ut // host is not valid according to network policy return nil, protocolstate.ErrHostDenied.Msgf(opts.Host) } + + // executing queries implies the remote mysql service + ok, err := c.IsMySQL(opts.Host, opts.Port) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("not a mysql service") + } + dsn, err := BuildDSN(opts) if err != nil { return nil, err @@ -220,6 +241,15 @@ func (c *MySQLClient) ExecuteQueryWithOpts(opts MySQLOptions, query string) (*ut // log(to_json(result)); // ``` func (c *MySQLClient) ExecuteQuery(host string, port int, username, password, query string) (*utils.SQLResult, error) { + // executing queries implies the remote mysql service + ok, err := c.IsMySQL(host, port) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("not a mysql service") + } + return c.ExecuteQueryWithOpts(MySQLOptions{ Host: host, Port: port, diff --git a/pkg/js/libs/postgres/postgres.go b/pkg/js/libs/postgres/postgres.go index f20ea85a73..e3093c49d3 100644 --- a/pkg/js/libs/postgres/postgres.go +++ b/pkg/js/libs/postgres/postgres.go @@ -37,6 +37,7 @@ type ( // const isPostgres = postgres.IsPostgres('acme.com', 5432); // ``` func (c *PGClient) IsPostgres(host string, port int) (bool, error) { + // todo: why this is exposed? Service fingerprint should be automatic return memoizedisPostgres(host, port) } @@ -74,6 +75,13 @@ func isPostgres(host string, port int) (bool, error) { // const connected = client.Connect('acme.com', 5432, 'username', 'password'); // ``` func (c *PGClient) Connect(host string, port int, username, password string) (bool, error) { + ok, err := c.IsPostgres(host, port) + if err != nil { + return false, err + } + if !ok { + return false, fmt.Errorf("not a postgres service") + } return memoizedconnect(host, port, username, password, "postgres") } @@ -88,6 +96,14 @@ func (c *PGClient) Connect(host string, port int, username, password string) (bo // log(to_json(result)); // ``` func (c *PGClient) ExecuteQuery(host string, port int, username, password, dbName, query string) (*utils.SQLResult, error) { + ok, err := c.IsPostgres(host, port) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("not a postgres service") + } + return memoizedexecuteQuery(host, port, username, password, dbName, query) } @@ -129,6 +145,14 @@ func executeQuery(host string, port int, username string, password string, dbNam // const connected = client.ConnectWithDB('acme.com', 5432, 'username', 'password', 'dbname'); // ``` func (c *PGClient) ConnectWithDB(host string, port int, username, password, dbName string) (bool, error) { + ok, err := c.IsPostgres(host, port) + if err != nil { + return false, err + } + if !ok { + return false, fmt.Errorf("not a postgres service") + } + return memoizedconnect(host, port, username, password, dbName) } @@ -149,10 +173,10 @@ func connect(host string, port int, username string, password string, dbName str defer cancel() db := pg.Connect(&pg.Options{ - Addr: target, - User: username, - Password: password, - Database: dbName, + Addr: target, + User: username, + Password: password, + Database: dbName, Dialer: func(network, addr string) (net.Conn, error) { return protocolstate.Dialer.Dial(context.Background(), network, addr) }, diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go index f103f50d0c..7736113dca 100644 --- a/pkg/protocols/code/code.go +++ b/pkg/protocols/code/code.go @@ -37,8 +37,7 @@ import ( ) const ( - pythonEnvRegex = `os\.getenv\(['"]([^'"]+)['"]\)` - TimeoutMultiplier = 6 // timeout multiplier for code protocol + pythonEnvRegex = `os\.getenv\(['"]([^'"]+)['"]\)` ) var ( @@ -179,9 +178,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa metaSrc.AddVariable(gozerotypes.Variable{Name: name, Value: v}) } - // set timeout using multiplier - timeout := TimeoutMultiplier * request.options.Options.Timeout - if request.PreCondition != "" { if request.options.Options.Debug || request.options.Options.DebugRequests { gologger.Debug().Msgf("[%s] Executing Precondition for Code request\n", request.TemplateID) @@ -199,11 +195,11 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa result, err := request.options.JsCompiler.ExecuteWithOptions(request.preConditionCompiled, args, &compiler.ExecuteOptions{ - Timeout: timeout, - Source: &request.PreCondition, - Callback: registerPreConditionFunctions, - Cleanup: cleanUpPreConditionFunctions, - Context: input.Context(), + TimeoutVariants: request.options.Options.GetTimeouts(), + Source: &request.PreCondition, + Callback: registerPreConditionFunctions, + Cleanup: cleanUpPreConditionFunctions, + Context: input.Context(), }) if err != nil { return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err) @@ -218,7 +214,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } } - ctx, cancel := context.WithTimeoutCause(input.Context(), time.Duration(timeout)*time.Second, ErrCodeExecutionDeadline) + ctx, cancel := context.WithTimeoutCause(input.Context(), request.options.Options.GetTimeouts().CodeExecutionTimeout, ErrCodeExecutionDeadline) defer cancel() // Note: we use contextutil despite the fact that gozero accepts context as argument gOutput, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (*gozerotypes.Result, error) { diff --git a/pkg/protocols/common/contextargs/contextargs.go b/pkg/protocols/common/contextargs/contextargs.go index ed004ae370..af55595498 100644 --- a/pkg/protocols/common/contextargs/contextargs.go +++ b/pkg/protocols/common/contextargs/contextargs.go @@ -50,9 +50,11 @@ func NewWithInput(ctx context.Context, input string) *Context { if err != nil { gologger.Error().Msgf("contextargs: could not create cookie jar: %s\n", err) } + metaInput := NewMetaInput() + metaInput.Input = input return &Context{ ctx: ctx, - MetaInput: &MetaInput{Input: input}, + MetaInput: metaInput, CookieJar: jar, args: &mapsutil.SyncLockMap[string, interface{}]{ Map: make(map[string]interface{}), diff --git a/pkg/protocols/common/contextargs/metainput.go b/pkg/protocols/common/contextargs/metainput.go index 72eac6247d..b3c0dabe7a 100644 --- a/pkg/protocols/common/contextargs/metainput.go +++ b/pkg/protocols/common/contextargs/metainput.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "strings" + "sync" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" @@ -24,6 +25,12 @@ type MetaInput struct { // ReqResp is the raw request for the input ReqResp *types.RequestResponse `json:"raw-request,omitempty"` + + mu *sync.Mutex +} + +func NewMetaInput() *MetaInput { + return &MetaInput{mu: &sync.Mutex{}} } func (metaInput *MetaInput) marshalToBuffer() (bytes.Buffer, error) { @@ -135,10 +142,9 @@ func (metaInput *MetaInput) Unmarshal(data string) error { } func (metaInput *MetaInput) Clone() *MetaInput { - input := &MetaInput{ - Input: metaInput.Input, - CustomIP: metaInput.CustomIP, - } + input := NewMetaInput() + input.Input = metaInput.Input + input.CustomIP = metaInput.CustomIP if metaInput.ReqResp != nil { input.ReqResp = metaInput.ReqResp.Clone() } @@ -160,6 +166,9 @@ func (metaInput *MetaInput) GetScanHash(templateId string) string { // there may be some cases where metainput is changed ex: while executing self-contained template etc // but that totally changes the scanID/hash so to avoid that we compute hash only once // and reuse it for all subsequent calls + metaInput.mu.Lock() + defer metaInput.mu.Unlock() + if metaInput.hash == "" { var rawRequest string if metaInput.ReqResp != nil { diff --git a/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/pkg/protocols/common/hosterrorscache/hosterrorscache.go index 38a7ff3290..1630e97c7d 100644 --- a/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr" "github.com/projectdiscovery/utils/errkit" + stringsutil "github.com/projectdiscovery/utils/strings" ) // CacheInterface defines the signature of the hosterrorscache so that @@ -200,7 +201,7 @@ func (c *Cache) checkError(err error) bool { errX := errkit.FromError(err) tmp := errX.Cause() cause := tmp.Error() - if strings.Contains(cause, "ReadStatusLine:") && strings.Contains(cause, "read: connection reset by peer") { + if stringsutil.ContainsAll(cause, "ReadStatusLine:", "read: connection reset by peer") { // this is a FP and should not be counted as a host error // because server closes connection when it reads corrupted bytes which we send via rawhttp return false diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index cf6f5234ec..b7c7796fa8 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -34,9 +34,7 @@ func Init(options *types.Options) error { lfaAllowed = options.AllowLocalFileAccess opts := fastdialer.DefaultOptions - if options.DialerTimeout > 0 { - opts.DialerTimeout = options.DialerTimeout - } + opts.DialerTimeout = options.GetTimeouts().DialTimeout if options.DialerKeepAlive > 0 { opts.DialerKeepAlive = options.DialerKeepAlive } diff --git a/pkg/protocols/dns/cluster.go b/pkg/protocols/dns/cluster.go index f2f5734785..86852b4d78 100644 --- a/pkg/protocols/dns/cluster.go +++ b/pkg/protocols/dns/cluster.go @@ -1,27 +1,24 @@ package dns -// CanCluster returns true if the request can be clustered. -// -// This used by the clustering engine to decide whether two requests -// are similar enough to be considered one and can be checked by -// just adding the matcher/extractors for the request and the correct IDs. -func (request *Request) CanCluster(other *Request) bool { - if len(request.Resolvers) > 0 || request.Trace || request.ID != "" { - return false - } - if request.Name != other.Name || - request.class != other.class || - request.Retries != other.Retries || - request.question != other.question { - return false - } +import ( + "fmt" + + "github.com/cespare/xxhash" +) + + +// TmplClusterKey generates a unique key for the request +// to be used in the clustering process. +func (request *Request) TmplClusterKey() uint64 { + recursion := "" if request.Recursion != nil { - if other.Recursion == nil { - return false - } - if *request.Recursion != *other.Recursion { - return false - } + recursion = fmt.Sprintf("%t", *request.Recursion) } - return true + inp := fmt.Sprintf("%s-%d-%d-%d-%s", request.Name, request.class, request.Retries, request.question, recursion) + return xxhash.Sum64String(inp) +} + +// IsClusterable returns true if the request is eligible to be clustered. +func (request *Request) IsClusterable() bool { + return !(len(request.Resolvers) > 0 || request.Trace || request.ID != "") } diff --git a/pkg/protocols/headless/request.go b/pkg/protocols/headless/request.go index 05bf5d3548..973dfa642a 100644 --- a/pkg/protocols/headless/request.go +++ b/pkg/protocols/headless/request.go @@ -215,6 +215,10 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p } dumpResponse(event, request.options, responseBody, input.MetaInput.Input) + shouldStopAtFirstMatch := request.StopAtFirstMatch || request.options.StopAtFirstMatch || request.options.Options.StopAtFirstMatch + if shouldStopAtFirstMatch && event.HasOperatorResult() { + return types.ErrNoMoreRequests + } return nil } diff --git a/pkg/protocols/http/cluster.go b/pkg/protocols/http/cluster.go index 3839817454..aa95c32bae 100644 --- a/pkg/protocols/http/cluster.go +++ b/pkg/protocols/http/cluster.go @@ -1,30 +1,21 @@ package http import ( - sliceutil "github.com/projectdiscovery/utils/slice" - "golang.org/x/exp/maps" + "fmt" + "strings" + + "github.com/cespare/xxhash" + "github.com/projectdiscovery/nuclei/v3/pkg/utils" ) -// CanCluster returns true if the request can be clustered. -// -// This used by the clustering engine to decide whether two requests -// are similar enough to be considered one and can be checked by -// just adding the matcher/extractors for the request and the correct IDs. -func (request *Request) CanCluster(other *Request) bool { - if len(request.Payloads) > 0 || len(request.Fuzzing) > 0 || len(request.Raw) > 0 || len(request.Body) > 0 || request.Unsafe || request.NeedsRequestCondition() || request.Name != "" { - return false - } - if request.Method != other.Method || - request.MaxRedirects != other.MaxRedirects || - request.DisableCookie != other.DisableCookie || - request.Redirects != other.Redirects { - return false - } - if !sliceutil.Equal(request.Path, other.Path) { - return false - } - if !maps.Equal(request.Headers, other.Headers) { - return false - } - return true +// TmplClusterKey generates a unique key for the request +// to be used in the clustering process. +func (request *Request) TmplClusterKey() uint64 { + inp := fmt.Sprintf("%s-%d-%t-%t-%s-%d", request.Method.String(), request.MaxRedirects, request.DisableCookie, request.Redirects, strings.Join(request.Path, "-"), utils.MapHash(request.Headers)) + return xxhash.Sum64String(inp) +} + +// IsClusterable returns true if the request is eligible to be clustered. +func (request *Request) IsClusterable() bool { + return !(len(request.Payloads) > 0 || len(request.Fuzzing) > 0 || len(request.Raw) > 0 || len(request.Body) > 0 || request.Unsafe || request.NeedsRequestCondition() || request.Name != "") } diff --git a/pkg/protocols/http/cluster_test.go b/pkg/protocols/http/cluster_test.go index a16db41ee0..da9c1e5a76 100644 --- a/pkg/protocols/http/cluster_test.go +++ b/pkg/protocols/http/cluster_test.go @@ -8,8 +8,11 @@ import ( func TestCanCluster(t *testing.T) { req := &Request{Unsafe: true} - require.False(t, req.CanCluster(&Request{}), "could cluster unsafe request") + require.False(t, req.IsClusterable(), "could cluster unsafe request") req = &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}} - require.True(t, req.CanCluster(&Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}), "could not cluster GET request") + newReq := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}} + require.True(t, req.IsClusterable(), "could not cluster GET request") + require.True(t, req.IsClusterable(), "could not cluster GET request") + require.Equal(t, req.TmplClusterKey(), newReq.TmplClusterKey(), "cluster keys should be equal") } diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go index c20cbbfa66..fc347ce88d 100644 --- a/pkg/protocols/http/http.go +++ b/pkg/protocols/http/http.go @@ -336,7 +336,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { request.Raw[i] = strings.ReplaceAll(raw, "\n", "\r\n") } } - request.rawhttpClient = httpclientpool.GetRawHTTP(options.Options) + request.rawhttpClient = httpclientpool.GetRawHTTP(options) } if len(request.Matchers) > 0 || len(request.Extractors) > 0 { compiled := &request.Operators diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go index b4ca5d40cf..2c244556ae 100644 --- a/pkg/protocols/http/httpclientpool/clientpool.go +++ b/pkg/protocols/http/httpclientpool/clientpool.go @@ -17,6 +17,7 @@ import ( "golang.org/x/net/publicsuffix" "github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v3/pkg/types" @@ -31,28 +32,14 @@ var ( forceMaxRedirects int normalClient *retryablehttp.Client clientPool *mapsutil.SyncLockMap[string, *retryablehttp.Client] - // MaxResponseHeaderTimeout is the timeout for response headers - // to be read from the server (this prevents infinite hang started by server if any) - // Note: this will be overridden temporarily when using @timeout request annotation - MaxResponseHeaderTimeout = time.Duration(10) * time.Second - // HttpTimeoutMultiplier is the multiplier for the http timeout - HttpTimeoutMultiplier = 3 ) -// GetHttpTimeout returns the http timeout for the client -func GetHttpTimeout(opts *types.Options) time.Duration { - return time.Duration(opts.Timeout*HttpTimeoutMultiplier) * time.Second -} - // Init initializes the clientpool implementation func Init(options *types.Options) error { // Don't create clients if already created in the past. if normalClient != nil { return nil } - if options.Timeout > 10 { - MaxResponseHeaderTimeout = time.Duration(options.Timeout) * time.Second - } if options.ShouldFollowHTTPRedirects() { forceMaxRedirects = options.MaxRedirects } @@ -143,7 +130,7 @@ func (c *Configuration) HasStandardOptions() bool { } // GetRawHTTP returns the rawhttp request client -func GetRawHTTP(options *types.Options) *rawhttp.Client { +func GetRawHTTP(options *protocols.ExecutorOptions) *rawhttp.Client { if rawHttpClient == nil { rawHttpOptions := rawhttp.DefaultOptions if types.ProxyURL != "" { @@ -153,7 +140,7 @@ func GetRawHTTP(options *types.Options) *rawhttp.Client { } else if protocolstate.Dialer != nil { rawHttpOptions.FastDialer = protocolstate.Dialer } - rawHttpOptions.Timeout = GetHttpTimeout(options) + rawHttpOptions.Timeout = options.Options.GetTimeouts().HttpTimeout rawHttpClient = rawhttp.NewClient(rawHttpOptions) } return rawHttpClient @@ -239,7 +226,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl } // responseHeaderTimeout is max timeout for response headers to be read - responseHeaderTimeout := MaxResponseHeaderTimeout + responseHeaderTimeout := options.GetTimeouts().HttpResponseHeaderTimeout if configuration.ResponseHeaderTimeout != 0 { responseHeaderTimeout = configuration.ResponseHeaderTimeout } @@ -308,7 +295,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl CheckRedirect: makeCheckRedirectFunc(redirectFlow, maxRedirects), } if !configuration.NoTimeout { - httpclient.Timeout = GetHttpTimeout(options) + httpclient.Timeout = options.GetTimeouts().HttpTimeout } client := retryablehttp.NewWithHTTPClient(httpclient, retryableHttpOptions) if jar != nil { diff --git a/pkg/protocols/http/httpclientpool/options.go b/pkg/protocols/http/httpclientpool/options.go index f24e421b86..92e42fe96b 100644 --- a/pkg/protocols/http/httpclientpool/options.go +++ b/pkg/protocols/http/httpclientpool/options.go @@ -7,6 +7,7 @@ import ( "time" "github.com/projectdiscovery/rawhttp" + stringsutil "github.com/projectdiscovery/utils/strings" urlutil "github.com/projectdiscovery/utils/url" ) @@ -36,7 +37,7 @@ func SendRawRequest(client *rawhttp.Client, opts *RawHttpRequestOpts) (*http.Res resp, err := client.DoRawWithOptions(opts.Method, opts.URL, opts.Path, opts.Headers, opts.Body, opts.Options) if err != nil { cause := err.Error() - if strings.Contains(cause, "ReadStatusLine: ") && strings.Contains(cause, "read: connection reset by peer") { + if stringsutil.ContainsAll(cause, "ReadStatusLine: ", "read: connection reset by peer") { // this error is caused when rawhttp client sends a corrupted or malformed request packet to server // some servers may attempt gracefully shutdown but most will just abruptly close the connection which results // in a connection reset by peer error and this can be safely assumed as 400 Bad Request in terms of normal http flow diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 7e5e069348..0df13b8872 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -483,7 +483,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa request.options.RateLimitTake() ctx := request.newContext(input) - ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, httpclientpool.GetHttpTimeout(request.options.Options), ErrHttpEngineRequestDeadline) + ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, request.options.Options.GetTimeouts().HttpTimeout, ErrHttpEngineRequestDeadline) defer cancel() generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue) @@ -844,6 +844,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ outputEvent["ip"] = input.MetaInput.CustomIP } else { outputEvent["ip"] = protocolstate.Dialer.GetDialedIP(hostname) + // try getting cname + request.addCNameIfAvailable(hostname, outputEvent) } if len(generatedRequest.interactshURLs) > 0 { @@ -940,6 +942,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ outputEvent["ip"] = input.MetaInput.CustomIP } else { outputEvent["ip"] = protocolstate.Dialer.GetDialedIP(hostname) + // try getting cname + request.addCNameIfAvailable(hostname, outputEvent) } if request.options.Interactsh != nil { request.options.Interactsh.MakePlaceholders(generatedRequest.interactshURLs, outputEvent) @@ -1031,6 +1035,23 @@ func (request *Request) validateNFixEvent(input *contextargs.Context, gr *genera } } +// addCNameIfAvailable adds the cname to the event if available +func (request *Request) addCNameIfAvailable(hostname string, outputEvent map[string]interface{}) { + data, err := protocolstate.Dialer.GetDNSData(hostname) + if err == nil { + switch len(data.CNAME) { + case 0: + return + case 1: + outputEvent["cname"] = data.CNAME[0] + default: + // add 1st and put others in cname_all + outputEvent["cname"] = data.CNAME[0] + outputEvent["cname_all"] = data.CNAME + } + } +} + // handleSignature of the http request func (request *Request) handleSignature(generatedRequest *generatedRequest) error { switch request.Signature.Value { diff --git a/pkg/protocols/http/request_annotations.go b/pkg/protocols/http/request_annotations.go index c0f4b69774..fa2a5eaedd 100644 --- a/pkg/protocols/http/request_annotations.go +++ b/pkg/protocols/http/request_annotations.go @@ -130,6 +130,10 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req if duration := reTimeoutAnnotation.FindStringSubmatch(rawRequest); len(duration) > 0 { value := strings.TrimSpace(duration[1]) if parsed, err := time.ParseDuration(value); err == nil { + // to avoid dos via timeout request annotation in http template we set it to maximum of 2 minutes + if parsed > 2*time.Minute { + parsed = 2 * time.Minute + } //nolint:govet // cancelled automatically by withTimeout // global timeout is overridden by annotation by replacing context ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), parsed, ErrTimeoutAnnotationDeadline) @@ -140,7 +144,7 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req } else { //nolint:govet // cancelled automatically by withTimeout // global timeout is overridden by annotation by replacing context - ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), httpclientpool.GetHttpTimeout(r.options.Options), ErrRequestTimeoutDeadline) + ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), r.options.Options.GetTimeouts().HttpTimeout, ErrRequestTimeoutDeadline) request = request.Clone(ctx) } } diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index 71f9f53413..2e43c61cd9 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -65,9 +65,6 @@ type Request struct { // Code contains code to execute for the javascript request. Code string `yaml:"code,omitempty" json:"code,omitempty" jsonschema:"title=code to execute in javascript,description=Executes inline javascript code for the request"` // description: | - // Timeout in seconds is optional timeout for each javascript script execution (i.e init, pre-condition, code) - Timeout int `yaml:"timeout,omitempty" json:"timeout,omitempty" jsonschema:"title=timeout for javascript execution,description=Timeout in seconds is optional timeout for entire javascript script execution"` - // description: | // StopAtFirstMatch stops processing the request at first match. StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" json:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop the execution after a match is found"` // description: | @@ -153,9 +150,9 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } opts := &compiler.ExecuteOptions{ - Timeout: request.Timeout, - Source: &request.Init, - Context: context.Background(), + TimeoutVariants: request.options.Options.GetTimeouts(), + Source: &request.Init, + Context: context.Background(), } // register 'export' function to export variables from init code // these are saved in args and are available in pre-condition and request code @@ -345,7 +342,10 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV argsCopy.TemplateCtx = templateCtx.GetAll() result, err := request.options.JsCompiler.ExecuteWithOptions(request.preConditionCompiled, argsCopy, - &compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.PreCondition, Context: target.Context()}) + &compiler.ExecuteOptions{ + TimeoutVariants: requestOptions.Options.GetTimeouts(), + Source: &request.PreCondition, Context: target.Context(), + }) if err != nil { return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err) } @@ -500,7 +500,11 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte } results, err := request.options.JsCompiler.ExecuteWithOptions(request.scriptCompiled, argsCopy, - &compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.Code, Context: input.Context()}) + &compiler.ExecuteOptions{ + TimeoutVariants: requestOptions.Options.GetTimeouts(), + Source: &request.Code, + Context: input.Context(), + }) if err != nil { // shouldn't fail even if it returned error instead create a failure event results = compiler.ExecuteResult{"success": false, "error": err.Error()} diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go index 62807abf93..90390e53c3 100644 --- a/pkg/protocols/network/request.go +++ b/pkg/protocols/network/request.go @@ -327,7 +327,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac } if input.Read > 0 { - buffer, err := ConnReadNWithTimeout(conn, int64(input.Read), request.options.Options.ResponseReadTimeout) + buffer, err := ConnReadNWithTimeout(conn, int64(input.Read), request.options.Options.GetTimeouts().TcpReadTimeout) if err != nil { return errorutil.NewWithErr(err).Msgf("could not read response from connection") } @@ -377,7 +377,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac bufferSize = -1 } - final, err := ConnReadNWithTimeout(conn, int64(bufferSize), request.options.Options.ResponseReadTimeout) + final, err := ConnReadNWithTimeout(conn, int64(bufferSize), request.options.Options.GetTimeouts().TcpReadTimeout) if err != nil { request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) gologger.Verbose().Msgf("could not read more data from %s: %s", actualAddress, err) diff --git a/pkg/protocols/ssl/ssl.go b/pkg/protocols/ssl/ssl.go index 49d612b882..1abde973aa 100644 --- a/pkg/protocols/ssl/ssl.go +++ b/pkg/protocols/ssl/ssl.go @@ -3,8 +3,10 @@ package ssl import ( "fmt" "net" + "strings" "time" + "github.com/cespare/xxhash" "github.com/fatih/structs" jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" @@ -99,15 +101,15 @@ type Request struct { options *protocols.ExecutorOptions } -// CanCluster returns true if the request can be clustered. -func (request *Request) CanCluster(other *Request) bool { - if len(request.CipherSuites) > 0 || request.MinVersion != "" || request.MaxVersion != "" { - return false - } - if request.Address != other.Address || request.ScanMode != other.ScanMode { - return false - } - return true +// TmplClusterKey generates a unique key for the request +// to be used in the clustering process. +func (request *Request) TmplClusterKey() uint64 { + inp := fmt.Sprintf("%s-%s-%t-%t-%s", request.Address, request.ScanMode, request.TLSCiphersEnum, request.TLSVersionsEnum, strings.Join(request.TLSCipherTypes, ",")) + return xxhash.Sum64String(inp) +} + +func (request *Request) IsClusterable() bool { + return !(len(request.CipherSuites) > 0 || request.MinVersion != "" || request.MaxVersion != "") } // Compile compiles the request generators preparing any requests possible. diff --git a/pkg/scan/scan_context.go b/pkg/scan/scan_context.go index 69fbfde4c0..b8f59ac7d4 100644 --- a/pkg/scan/scan_context.go +++ b/pkg/scan/scan_context.go @@ -77,6 +77,9 @@ func (s *ScanContext) LogEvent(e *output.InternalWrappedEvent) { s.events = append(s.events, e) } + e.RLock() + defer e.RUnlock() + s.results = append(s.results, e.Results...) } diff --git a/pkg/templates/cluster.go b/pkg/templates/cluster.go index 603378e986..a4a4ac4bad 100644 --- a/pkg/templates/cluster.go +++ b/pkg/templates/cluster.go @@ -14,7 +14,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" cryptoutil "github.com/projectdiscovery/utils/crypto" - mapsutil "github.com/projectdiscovery/utils/maps" ) // Cluster clusters a list of templates into a lesser number if possible based @@ -42,91 +41,68 @@ import ( // Finally, the engine creates a single executer with a clusteredexecuter for all templates // in a cluster. func Cluster(list []*Template) [][]*Template { + http := make(map[uint64][]*Template) + dns := make(map[uint64][]*Template) + ssl := make(map[uint64][]*Template) + final := [][]*Template{} - skip := mapsutil.NewSyncLockMap[string, struct{}]() + // Split up templates that might be clusterable for _, template := range list { - key := template.Path - - if skip.Has(key) { - continue - } - - // We only cluster http, dns and ssl requests as of now. - // Take care of requests that can't be clustered first. - if len(template.RequestsHTTP) == 0 && len(template.RequestsDNS) == 0 && len(template.RequestsSSL) == 0 { - _ = skip.Set(key, struct{}{}) - final = append(final, []*Template{template}) - continue - } - // it is not possible to cluster flow and multiprotocol due to dependent execution if template.Flow != "" || template.Options.IsMultiProtocol { - _ = skip.Set(key, struct{}{}) final = append(final, []*Template{template}) continue } - _ = skip.Set(key, struct{}{}) - - var templateType types.ProtocolType switch { case len(template.RequestsDNS) == 1: - templateType = types.DNSProtocol - case len(template.RequestsHTTP) == 1: - templateType = types.HTTPProtocol - case len(template.RequestsSSL) == 1: - templateType = types.SSLProtocol - } - - // Find any/all similar matching request that is identical to - // this one and cluster them together for http protocol only. - cluster := []*Template{} - for _, other := range list { - otherKey := other.Path - - if skip.Has(otherKey) { - continue - } - - // it is not possible to cluster flow and multiprotocol due to dependent execution - if other.Flow != "" || other.Options.IsMultiProtocol { - _ = skip.Set(otherKey, struct{}{}) - final = append(final, []*Template{other}) - continue + if template.RequestsDNS[0].IsClusterable() { + hash := template.RequestsDNS[0].TmplClusterKey() + if dns[hash] == nil { + dns[hash] = []*Template{} + } + dns[hash] = append(dns[hash], template) + } else { + final = append(final, []*Template{template}) } - switch templateType { - case types.DNSProtocol: - if len(other.RequestsDNS) != 1 { - continue - } else if template.RequestsDNS[0].CanCluster(other.RequestsDNS[0]) { - _ = skip.Set(otherKey, struct{}{}) - cluster = append(cluster, other) - } - case types.HTTPProtocol: - if len(other.RequestsHTTP) != 1 { - continue - } else if template.RequestsHTTP[0].CanCluster(other.RequestsHTTP[0]) { - _ = skip.Set(otherKey, struct{}{}) - cluster = append(cluster, other) + case len(template.RequestsHTTP) == 1: + if template.RequestsHTTP[0].IsClusterable() { + hash := template.RequestsHTTP[0].TmplClusterKey() + if http[hash] == nil { + http[hash] = []*Template{} } - case types.SSLProtocol: - if len(other.RequestsSSL) != 1 { - continue - } else if template.RequestsSSL[0].CanCluster(other.RequestsSSL[0]) { - _ = skip.Set(otherKey, struct{}{}) - cluster = append(cluster, other) + http[hash] = append(http[hash], template) + } else { + final = append(final, []*Template{template}) + } + case len(template.RequestsSSL) == 1: + if template.RequestsSSL[0].IsClusterable() { + hash := template.RequestsSSL[0].TmplClusterKey() + if ssl[hash] == nil { + ssl[hash] = []*Template{} } + ssl[hash] = append(ssl[hash], template) + } else { + final = append(final, []*Template{template}) } - } - if len(cluster) > 0 { - cluster = append(cluster, template) - final = append(final, cluster) - } else { + default: final = append(final, []*Template{template}) } } + + // add all clusterd templates + for _, templates := range http { + final = append(final, templates) + } + for _, templates := range dns { + final = append(final, templates) + } + for _, templates := range ssl { + final = append(final, templates) + } + return final } diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index 39d6a6ea0f..54708e7ea1 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -342,6 +342,11 @@ func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, e return nil, errors.New("no template author field provided") } + numberOfWorkflows := len(template.Workflows) + if numberOfWorkflows > 0 && numberOfWorkflows != template.Requests() { + return nil, errors.New("workflows cannot have other protocols") + } + // use default unknown severity if len(template.Workflows) == 0 { if template.Info.SeverityHolder.Severity == severity.Undefined { diff --git a/pkg/templates/compile_test.go b/pkg/templates/compile_test.go index 3a230cc527..91a858bd7e 100644 --- a/pkg/templates/compile_test.go +++ b/pkg/templates/compile_test.go @@ -39,15 +39,15 @@ func setup() { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts = protocols.ExecutorOptions{ - Output: testutils.NewMockOutputWriter(options.OmitTemplate), - Options: options, - Progress: progressImpl, - ProjectFile: nil, - IssuesClient: nil, - Browser: nil, - Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), - RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), - Parser: templates.NewParser(), + Output: testutils.NewMockOutputWriter(options.OmitTemplate), + Options: options, + Progress: progressImpl, + ProjectFile: nil, + IssuesClient: nil, + Browser: nil, + Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), + RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), + Parser: templates.NewParser(), } workflowLoader, err := workflow.NewLoader(&executerOpts) if err != nil { @@ -197,3 +197,12 @@ func Test_WrongTemplate(t *testing.T) { require.Nil(t, got, "could not parse template") require.ErrorContains(t, err, "no requests defined ") } + +func TestWrongWorkflow(t *testing.T) { + setup() + + filePath := "tests/workflow-invalid.yaml" + got, err := templates.Parse(filePath, nil, executerOpts) + require.Nil(t, got, "could not parse template") + require.ErrorContains(t, err, "workflows cannot have other protocols") +} diff --git a/pkg/templates/preprocessors.go b/pkg/templates/preprocessors.go index 054a0bc41e..7cf6f8e21a 100644 --- a/pkg/templates/preprocessors.go +++ b/pkg/templates/preprocessors.go @@ -5,6 +5,7 @@ import ( "regexp" "strings" + stringsutil "github.com/projectdiscovery/utils/strings" "github.com/segmentio/ksuid" ) @@ -48,7 +49,7 @@ func (r *randStrPreprocessor) ProcessNReturnData(data []byte) ([]byte, map[strin continue } value := expression[1] - if strings.Contains(value, "(") || strings.Contains(value, ")") { + if stringsutil.ContainsAny(value, "(", ")") { continue } diff --git a/pkg/templates/templates.go b/pkg/templates/templates.go index 045f947c68..df0c47648d 100644 --- a/pkg/templates/templates.go +++ b/pkg/templates/templates.go @@ -177,8 +177,6 @@ func (template *Template) Type() types.ProtocolType { return types.HeadlessProtocol case len(template.RequestsNetwork) > 0: return types.NetworkProtocol - case len(template.Workflow.Workflows) > 0: - return types.WorkflowProtocol case len(template.RequestsSSL) > 0: return types.SSLProtocol case len(template.RequestsWebsocket) > 0: @@ -189,6 +187,8 @@ func (template *Template) Type() types.ProtocolType { return types.CodeProtocol case len(template.RequestsJavascript) > 0: return types.JavascriptProtocol + case len(template.Workflow.Workflows) > 0: + return types.WorkflowProtocol default: return types.InvalidProtocol } diff --git a/pkg/templates/templates_doc.go b/pkg/templates/templates_doc.go index 536b80ad7f..53864516a7 100644 --- a/pkg/templates/templates_doc.go +++ b/pkg/templates/templates_doc.go @@ -1929,7 +1929,7 @@ func init() { Value: "Matched is the input which was matched upon", }, } - JAVASCRIPTRequestDoc.Fields = make([]encoder.Doc, 10) + JAVASCRIPTRequestDoc.Fields = make([]encoder.Doc, 9) JAVASCRIPTRequestDoc.Fields[0].Name = "id" JAVASCRIPTRequestDoc.Fields[0].Type = "string" JAVASCRIPTRequestDoc.Fields[0].Note = "" @@ -1955,33 +1955,28 @@ func init() { JAVASCRIPTRequestDoc.Fields[4].Note = "" JAVASCRIPTRequestDoc.Fields[4].Description = "Code contains code to execute for the javascript request." JAVASCRIPTRequestDoc.Fields[4].Comments[encoder.LineComment] = "Code contains code to execute for the javascript request." - JAVASCRIPTRequestDoc.Fields[5].Name = "timeout" - JAVASCRIPTRequestDoc.Fields[5].Type = "int" + JAVASCRIPTRequestDoc.Fields[5].Name = "stop-at-first-match" + JAVASCRIPTRequestDoc.Fields[5].Type = "bool" JAVASCRIPTRequestDoc.Fields[5].Note = "" - JAVASCRIPTRequestDoc.Fields[5].Description = "Timeout in seconds is optional timeout for each javascript script execution (i.e init, pre-condition, code)" - JAVASCRIPTRequestDoc.Fields[5].Comments[encoder.LineComment] = "Timeout in seconds is optional timeout for each javascript script execution (i.e init, pre-condition, code)" - JAVASCRIPTRequestDoc.Fields[6].Name = "stop-at-first-match" - JAVASCRIPTRequestDoc.Fields[6].Type = "bool" + JAVASCRIPTRequestDoc.Fields[5].Description = "StopAtFirstMatch stops processing the request at first match." + JAVASCRIPTRequestDoc.Fields[5].Comments[encoder.LineComment] = "StopAtFirstMatch stops processing the request at first match." + JAVASCRIPTRequestDoc.Fields[6].Name = "attack" + JAVASCRIPTRequestDoc.Fields[6].Type = "generators.AttackTypeHolder" JAVASCRIPTRequestDoc.Fields[6].Note = "" - JAVASCRIPTRequestDoc.Fields[6].Description = "StopAtFirstMatch stops processing the request at first match." - JAVASCRIPTRequestDoc.Fields[6].Comments[encoder.LineComment] = "StopAtFirstMatch stops processing the request at first match." - JAVASCRIPTRequestDoc.Fields[7].Name = "attack" - JAVASCRIPTRequestDoc.Fields[7].Type = "generators.AttackTypeHolder" + JAVASCRIPTRequestDoc.Fields[6].Description = "Attack is the type of payload combinations to perform.\n\nSniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." + JAVASCRIPTRequestDoc.Fields[6].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." + JAVASCRIPTRequestDoc.Fields[7].Name = "threads" + JAVASCRIPTRequestDoc.Fields[7].Type = "int" JAVASCRIPTRequestDoc.Fields[7].Note = "" - JAVASCRIPTRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nSniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." - JAVASCRIPTRequestDoc.Fields[7].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." - JAVASCRIPTRequestDoc.Fields[8].Name = "threads" - JAVASCRIPTRequestDoc.Fields[8].Type = "int" + JAVASCRIPTRequestDoc.Fields[7].Description = "Payload concurreny i.e threads for sending requests." + JAVASCRIPTRequestDoc.Fields[7].Comments[encoder.LineComment] = "Payload concurreny i.e threads for sending requests." + + JAVASCRIPTRequestDoc.Fields[7].AddExample("Send requests using 10 concurrent threads", 10) + JAVASCRIPTRequestDoc.Fields[8].Name = "payloads" + JAVASCRIPTRequestDoc.Fields[8].Type = "map[string]interface{}" JAVASCRIPTRequestDoc.Fields[8].Note = "" - JAVASCRIPTRequestDoc.Fields[8].Description = "Payload concurreny i.e threads for sending requests." - JAVASCRIPTRequestDoc.Fields[8].Comments[encoder.LineComment] = "Payload concurreny i.e threads for sending requests." - - JAVASCRIPTRequestDoc.Fields[8].AddExample("Send requests using 10 concurrent threads", 10) - JAVASCRIPTRequestDoc.Fields[9].Name = "payloads" - JAVASCRIPTRequestDoc.Fields[9].Type = "map[string]interface{}" - JAVASCRIPTRequestDoc.Fields[9].Note = "" - JAVASCRIPTRequestDoc.Fields[9].Description = "Payloads contains any payloads for the current request.\n\nPayloads support both key-values combinations where a list\nof payloads is provided, or optionally a single file can also\nbe provided as payload which will be read on run-time." - JAVASCRIPTRequestDoc.Fields[9].Comments[encoder.LineComment] = "Payloads contains any payloads for the current request." + JAVASCRIPTRequestDoc.Fields[8].Description = "Payloads contains any payloads for the current request.\n\nPayloads support both key-values combinations where a list\nof payloads is provided, or optionally a single file can also\nbe provided as payload which will be read on run-time." + JAVASCRIPTRequestDoc.Fields[8].Comments[encoder.LineComment] = "Payloads contains any payloads for the current request." HTTPSignatureTypeHolderDoc.Type = "http.SignatureTypeHolder" HTTPSignatureTypeHolderDoc.Comments[encoder.LineComment] = " SignatureTypeHolder is used to hold internal type of the signature" diff --git a/pkg/templates/tests/workflow-invalid.yaml b/pkg/templates/tests/workflow-invalid.yaml new file mode 100644 index 0000000000..981823aa0c --- /dev/null +++ b/pkg/templates/tests/workflow-invalid.yaml @@ -0,0 +1,18 @@ +id: workflow-example + +info: + name: Test Invalid Workflow Template + author: pdteam + severity: info + +http: + - raw: + - | + POST /re HTTP/1.1 + Host: {{Hostname}} + + {{code_response}} + +workflows: + - template: tests/match-1.yaml + - template: tests/match-1.yaml diff --git a/pkg/testutils/integration.go b/pkg/testutils/integration.go index 8b5a71b1ab..c8a67e41cb 100644 --- a/pkg/testutils/integration.go +++ b/pkg/testutils/integration.go @@ -14,6 +14,7 @@ import ( "github.com/gobwas/ws" "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/utils/conversion" ) // ExtraArgs @@ -70,20 +71,26 @@ func RunNucleiBareArgsAndGetResults(debug bool, env []string, extra ...string) ( } else { cmd.Args = append(cmd.Args, "-silent") } - data, err := cmd.Output() - if debug { - fmt.Println(string(data)) + output, err := cmd.Output() + var data string + if len(output) > 0 { + data = strings.TrimSpace(conversion.String(output)) } - if len(data) < 1 && err != nil { - return nil, fmt.Errorf("%v: %v", err.Error(), string(data)) + if debug { + fmt.Println(data) } var parts []string - items := strings.Split(string(data), "\n") + items := strings.Split(data, "\n") for _, i := range items { if i != "" { parts = append(parts, i) } } + + if (data == "" || len(parts) == 0) && err != nil { + return nil, fmt.Errorf("%v: %v", err.Error(), data) + } + return parts, nil } @@ -98,20 +105,25 @@ func RunNucleiWithArgsAndGetResults(debug bool, args ...string) ([]string, error } else { cmd.Args = append(cmd.Args, "-silent") } - data, err := cmd.Output() - if debug { - fmt.Println(string(data)) + output, err := cmd.Output() + var data string + if len(output) > 0 { + data = strings.TrimSpace(conversion.String(output)) } - if len(data) < 1 && err != nil { - return nil, fmt.Errorf("%v: %v", err.Error(), string(data)) + if debug { + fmt.Println(data) } var parts []string - items := strings.Split(string(data), "\n") + items := strings.Split(data, "\n") for _, i := range items { if i != "" { parts = append(parts, i) } } + + if (data == "" || len(parts) == 0) && err != nil { + return nil, fmt.Errorf("%v: %v", err.Error(), data) + } return parts, nil } @@ -127,12 +139,16 @@ func RunNucleiArgsAndGetErrors(debug bool, env []string, extra ...string) ([]str cmd.Args = append(cmd.Args, "-allow-local-file-access") cmd.Args = append(cmd.Args, "-nc") // disable color cmd.Env = append(cmd.Env, ExtraEnvVars...) - data, err := cmd.CombinedOutput() + dataOutput, err := cmd.CombinedOutput() if debug { - fmt.Println(string(data)) + fmt.Println(string(dataOutput)) + } + var data string + if len(dataOutput) > 0 { + data = strings.TrimSpace(conversion.String(dataOutput)) } results := []string{} - for _, v := range strings.Split(string(data), "\n") { + for _, v := range strings.Split(data, "\n") { line := strings.TrimSpace(v) switch { case strings.HasPrefix(line, "[ERR]"): @@ -164,20 +180,24 @@ func RunNucleiArgsWithEnvAndGetResults(debug bool, env []string, extra ...string } else { cmd.Args = append(cmd.Args, "-silent") } - data, err := cmd.Output() - if debug { - fmt.Println(string(data)) + dataOutput, err := cmd.Output() + var data string + if len(dataOutput) > 0 { + data = strings.TrimSpace(conversion.String(dataOutput)) } - if len(data) < 1 && err != nil { - return nil, fmt.Errorf("%v: %v", err.Error(), string(data)) + if debug { + fmt.Println(data) } var parts []string - items := strings.Split(string(data), "\n") + items := strings.Split(data, "\n") for _, i := range items { if i != "" { parts = append(parts, i) } } + if (data == "" || len(parts) == 0) && err != nil { + return nil, fmt.Errorf("%v: %v", err.Error(), data) + } return parts, nil } diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index 6e6f94d9eb..930787aab9 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -76,7 +76,6 @@ var DefaultOptions = &types.Options{ InteractionsPollDuration: 5, GitHubTemplateRepo: []string{}, GitHubToken: "", - ResponseReadTimeout: time.Second * 5, } // TemplateInfo contains info for a mock executed template. @@ -90,17 +89,17 @@ type TemplateInfo struct { func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protocols.ExecutorOptions { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts := &protocols.ExecutorOptions{ - TemplateID: info.ID, - TemplateInfo: info.Info, - TemplatePath: info.Path, - Output: NewMockOutputWriter(options.OmitTemplate), - Options: options, - Progress: progressImpl, - ProjectFile: nil, - IssuesClient: nil, - Browser: nil, - Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), - RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), + TemplateID: info.ID, + TemplateInfo: info.Info, + TemplatePath: info.Path, + Output: NewMockOutputWriter(options.OmitTemplate), + Options: options, + Progress: progressImpl, + ProjectFile: nil, + IssuesClient: nil, + Browser: nil, + Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), + RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), } executerOpts.CreateTemplateCtxStore() return executerOpts diff --git a/pkg/tmplexec/flow/flow_executor_test.go b/pkg/tmplexec/flow/flow_executor_test.go index b47b38a2a0..cf7b1790a6 100644 --- a/pkg/tmplexec/flow/flow_executor_test.go +++ b/pkg/tmplexec/flow/flow_executor_test.go @@ -27,15 +27,15 @@ func setup() { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts = protocols.ExecutorOptions{ - Output: testutils.NewMockOutputWriter(options.OmitTemplate), - Options: options, - Progress: progressImpl, - ProjectFile: nil, - IssuesClient: nil, - Browser: nil, - Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), - RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), - Parser: templates.NewParser(), + Output: testutils.NewMockOutputWriter(options.OmitTemplate), + Options: options, + Progress: progressImpl, + ProjectFile: nil, + IssuesClient: nil, + Browser: nil, + Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), + RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), + Parser: templates.NewParser(), } workflowLoader, err := workflow.NewLoader(&executerOpts) if err != nil { diff --git a/pkg/types/types.go b/pkg/types/types.go index f49da194cc..48e9039d37 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -278,8 +278,6 @@ type Options struct { SNI string // InputFileMode specifies the mode of input file (jsonl, burp, openapi, swagger, etc) InputFileMode string - // DialerTimeout sets the timeout for network requests. - DialerTimeout time.Duration // DialerKeepAlive sets the keep alive duration for network requests. DialerKeepAlive time.Duration // Interface to use for network scan @@ -292,8 +290,6 @@ type Options struct { ResponseReadSize int // ResponseSaveSize is the maximum size of response to save ResponseSaveSize int - // ResponseReadTimeout is response read timeout in seconds - ResponseReadTimeout time.Duration // Health Check HealthCheck bool // Time to wait between each input read operation before closing the stream @@ -406,6 +402,79 @@ type Options struct { HttpApiEndpoint string // ListTemplateProfiles lists all available template profiles ListTemplateProfiles bool + // timeouts contains various types of timeouts used in nuclei + // these timeouts are derived from dial-timeout (-timeout) with known multipliers + // This is internally managed and does not need to be set by user by explicitly setting + // this overrides the default/derived one + timeouts *Timeouts +} + +// SetTimeouts sets the timeout variants to use for the executor +func (opts *Options) SetTimeouts(t *Timeouts) { + opts.timeouts = t +} + +// GetTimeouts returns the timeout variants to use for the executor +func (eo *Options) GetTimeouts() *Timeouts { + if eo.timeouts != nil { + // redundant but apply to avoid any potential issues + eo.timeouts.ApplyDefaults() + return eo.timeouts + } + // set timeout variant value + eo.timeouts = NewTimeoutVariant(eo.Timeout) + eo.timeouts.ApplyDefaults() + return eo.timeouts +} + +// Timeouts is a struct that contains all the timeout variants for nuclei +// dialer timeout is used to derive other timeouts +type Timeouts struct { + // DialTimeout for fastdialer (default 10s) + DialTimeout time.Duration + // Tcp(Network Protocol) Read From Connection Timeout (default 5s) + TcpReadTimeout time.Duration + // Http Response Header Timeout (default 10s) + // this timeout prevents infinite hangs started by server if any + // this is temporarily overridden when using @timeout request annotation + HttpResponseHeaderTimeout time.Duration + // HttpTimeout for http client (default -> 3 x dial-timeout = 30s) + HttpTimeout time.Duration + // JsCompilerExec timeout/deadline (default -> 2 x dial-timeout = 20s) + JsCompilerExecutionTimeout time.Duration + // CodeExecutionTimeout for code execution (default -> 3 x dial-timeout = 30s) + CodeExecutionTimeout time.Duration +} + +// NewTimeoutVariant creates a new timeout variant with the given dial timeout in seconds +func NewTimeoutVariant(dialTimeoutSec int) *Timeouts { + tv := &Timeouts{ + DialTimeout: time.Duration(dialTimeoutSec) * time.Second, + } + tv.ApplyDefaults() + return tv +} + +// ApplyDefaults applies default values to timeout variants when missing +func (tv *Timeouts) ApplyDefaults() { + if tv.DialTimeout == 0 { + tv.DialTimeout = 10 * time.Second + } + if tv.TcpReadTimeout == 0 { + tv.TcpReadTimeout = 5 * time.Second + } + if tv.HttpResponseHeaderTimeout == 0 { + tv.HttpResponseHeaderTimeout = 10 * time.Second + } + if tv.HttpTimeout == 0 { + tv.HttpTimeout = 3 * tv.DialTimeout + } + if tv.JsCompilerExecutionTimeout == 0 { + tv.JsCompilerExecutionTimeout = 2 * tv.DialTimeout + } + if tv.CodeExecutionTimeout == 0 { + tv.CodeExecutionTimeout = 3 * tv.DialTimeout + } } // ShouldLoadResume resume file @@ -444,7 +513,6 @@ func DefaultOptions() *Options { MaxHostError: 30, ResponseReadSize: 10 * unitutils.Mega, ResponseSaveSize: unitutils.Mega, - ResponseReadTimeout: 5 * time.Second, } } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 0d95b13f46..dfe37ae99e 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -2,12 +2,16 @@ package utils import ( "errors" + "fmt" "io" "net/url" "strings" + "github.com/cespare/xxhash" "github.com/projectdiscovery/nuclei/v3/pkg/catalog" "github.com/projectdiscovery/retryablehttp-go" + mapsutil "github.com/projectdiscovery/utils/maps" + "golang.org/x/exp/constraints" ) func IsBlank(value string) bool { @@ -57,3 +61,13 @@ func StringSliceContains(slice []string, item string) bool { } return false } + +// MapHash generates a hash for any give map +func MapHash[K constraints.Ordered, V any](m map[K]V) uint64 { + keys := mapsutil.GetSortedKeys(m) + var sb strings.Builder + for _, k := range keys { + sb.WriteString(fmt.Sprintf("%v:%v\n", k, m[k])) + } + return xxhash.Sum64([]byte(sb.String())) +}