Skip to content

Commit b7f76cf

Browse files
Mzack9999Mzack9999
authored
Merge pull request #4833 from projectdiscovery/maint-memory
Adding memguardian + various optimizations
2 parents d292ac8 + 117d4a3 commit b7f76cf

File tree

13 files changed

+221
-72
lines changed

13 files changed

+221
-72
lines changed

‎go.mod‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ require (
3030
github.com/remeh/sizedwaitgroup v1.0.0
3131
github.com/rs/xid v1.5.0
3232
github.com/segmentio/ksuid v1.0.4
33-
github.com/shirou/gopsutil/v3 v3.23.7 // indirect
33+
github.com/shirou/gopsutil/v3 v3.24.2 // indirect
3434
github.com/spaolacci/murmur3 v1.1.0 // indirect
3535
github.com/spf13/cast v1.5.1
3636
github.com/syndtr/goleveldb v1.0.0
@@ -94,7 +94,7 @@ require (
9494
github.com/projectdiscovery/tlsx v1.1.6
9595
github.com/projectdiscovery/uncover v1.0.7
9696
github.com/projectdiscovery/useragent v0.0.40
97-
github.com/projectdiscovery/utils v0.0.84-0.20240312214300-d3ba70dbb9ca
97+
github.com/projectdiscovery/utils v0.0.84-0.20240313184656-e3ec80f4dd42
9898
github.com/projectdiscovery/wappalyzergo v0.0.112
9999
github.com/redis/go-redis/v9 v9.1.0
100100
github.com/sashabaranov/go-openai v1.15.3
@@ -204,6 +204,7 @@ require (
204204
github.com/projectdiscovery/stringsutil v0.0.2 // indirect
205205
github.com/quic-go/quic-go v0.40.1 // indirect
206206
github.com/refraction-networking/utls v1.6.1 // indirect
207+
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
207208
github.com/shoenig/go-m1cpu v0.1.6 // indirect
208209
github.com/sirupsen/logrus v1.9.3 // indirect
209210
github.com/skeema/knownhosts v1.2.1 // indirect
@@ -227,6 +228,7 @@ require (
227228
github.com/yuin/goldmark-emoji v1.0.1 // indirect
228229
github.com/zeebo/blake3 v0.2.3 // indirect
229230
golang.org/x/arch v0.3.0 // indirect
231+
golang.org/x/sync v0.6.0 // indirect
230232
gopkg.in/djherbis/times.v1 v1.3.0 // indirect
231233
mellium.im/sasl v0.3.1 // indirect
232234
)
@@ -281,16 +283,16 @@ require (
281283
github.com/projectdiscovery/networkpolicy v0.0.8
282284
github.com/rivo/uniseg v0.4.6 // indirect
283285
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
284-
github.com/tklauser/go-sysconf v0.3.11 // indirect
285-
github.com/tklauser/numcpus v0.6.0 // indirect
286+
github.com/tklauser/go-sysconf v0.3.12 // indirect
287+
github.com/tklauser/numcpus v0.6.1 // indirect
286288
github.com/trivago/tgo v1.0.7
287289
github.com/ulikunitz/xz v0.5.11 // indirect
288290
github.com/valyala/bytebufferpool v1.0.0 // indirect
289291
github.com/yl2chen/cidranger v1.0.2 // indirect
290292
github.com/ysmood/goob v0.4.0 // indirect
291293
github.com/ysmood/gson v0.7.3 // indirect
292294
github.com/ysmood/leakless v0.8.0 // indirect
293-
github.com/yusufpapurcu/wmi v1.2.3 // indirect
295+
github.com/yusufpapurcu/wmi v1.2.4 // indirect
294296
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
295297
github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c // indirect
296298
go.etcd.io/bbolt v1.3.8 // indirect

‎go.sum‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -890,8 +890,10 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
890890
github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
891891
github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo=
892892
github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4=
893-
github.com/projectdiscovery/utils v0.0.84-0.20240312214300-d3ba70dbb9ca h1:GY9lUYDlENXPSFPJH01Bm1BfhrUF2jpnUBR+K4VPJIs=
894-
github.com/projectdiscovery/utils v0.0.84-0.20240312214300-d3ba70dbb9ca/go.mod h1:wzMfHBq2I9oy+DEiMfUYV86g1D7eXKaQsgWnqFpmMtI=
893+
github.com/projectdiscovery/utils v0.0.84-0.20240311212130-16ce15974a4a h1:njYY24OsTQJ80L8O+QxcYFljiVl83xp/BWz3dsIJF30=
894+
github.com/projectdiscovery/utils v0.0.84-0.20240311212130-16ce15974a4a/go.mod h1:bvcudEteeZ5MIZeBCXEfpcgj9h3tyB9qtnmc7zQR92w=
895+
github.com/projectdiscovery/utils v0.0.84-0.20240313184656-e3ec80f4dd42 h1:l22rSOP8i6HXu1QfAtIot8NvmJgUmBHEn6Mih7s8Gak=
896+
github.com/projectdiscovery/utils v0.0.84-0.20240313184656-e3ec80f4dd42/go.mod h1:VsoXXTuNAAziuodKWakLyurVXaV4tNTJU4Eo8umyr3Q=
895897
github.com/projectdiscovery/wappalyzergo v0.0.112 h1:QPpp5jmj1lqLd5mFdFKQ9VvcYhQNqyU9Mr+IB0US2zA=
896898
github.com/projectdiscovery/wappalyzergo v0.0.112/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8=
897899
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
@@ -964,8 +966,10 @@ github.com/seh-msft/burpxml v1.0.1/go.mod h1:lTViCHPtGGS0scK0B4krm6Ld1kVZLWzQccw
964966
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
965967
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
966968
github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
967-
github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4=
968-
github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4=
969+
github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
970+
github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
971+
github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y=
972+
github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk=
969973
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
970974
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
971975
github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
@@ -1044,10 +1048,10 @@ github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaym
10441048
github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw=
10451049
github.com/tim-ywliu/nested-logrus-formatter v1.3.2 h1:jugNJ2/CNCI79SxOJCOhwUHeN3O7/7/bj+ZRGOFlCSw=
10461050
github.com/tim-ywliu/nested-logrus-formatter v1.3.2/go.mod h1:oGPmcxZB65j9Wo7mCnQKSrKEJtVDqyjD666SGmyStXI=
1047-
github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
1048-
github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
1049-
github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
1050-
github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
1051+
github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
1052+
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
1053+
github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
1054+
github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
10511055
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
10521056
github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM=
10531057
github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
@@ -1116,8 +1120,8 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU=
11161120
github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
11171121
github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os=
11181122
github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ=
1119-
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
1120-
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
1123+
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
1124+
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
11211125
github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
11221126
github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
11231127
github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg=
@@ -1387,6 +1391,7 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
13871391
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
13881392
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
13891393
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1394+
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
13901395
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
13911396
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
13921397
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

‎pkg/operators/operators.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,22 @@ func (operators *Operators) Compile() error {
6464
return nil
6565
}
6666

67+
func (operators *Operators) HasDSL() bool {
68+
for _, matcher := range operators.Matchers {
69+
if len(matcher.DSL) > 0 {
70+
return true
71+
}
72+
}
73+
74+
for _, extractor := range operators.Extractors {
75+
if len(extractor.DSL) > 0 {
76+
return true
77+
}
78+
}
79+
80+
return false
81+
}
82+
6783
// GetMatchersCondition returns the condition for the matchers
6884
func (operators *Operators) GetMatchersCondition() matchers.ConditionType {
6985
return operators.matchersCondition

‎pkg/output/output.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,10 @@ var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
7373
// InternalEvent is an internal output generation structure for nuclei.
7474
type InternalEvent map[string]interface{}
7575

76+
func (ie InternalEvent) Set(k string, v interface{}) {
77+
ie[k] = v
78+
}
79+
7680
// InternalWrappedEvent is a wrapped event with operators result added to it.
7781
type InternalWrappedEvent struct {
7882
// Mutex is internal field which is implicitly used

‎pkg/protocols/common/protocolstate/memguardian.go‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
package protocolstate
2+
3+
import (
4+
"sync"
5+
"time"
6+
7+
"github.com/projectdiscovery/gologger"
8+
"github.com/projectdiscovery/utils/env"
9+
httputil "github.com/projectdiscovery/utils/http"
10+
"github.com/projectdiscovery/utils/memguardian"
11+
)
12+
13+
var (
14+
MaxThreadsOnLowMemory = env.GetEnvOrDefault("MEMGUARDIAN_THREADS", 0)
15+
MaxBytesBufferAllocOnLowMemory = env.GetEnvOrDefault("MEMGUARDIAN_ALLOC", 0)
16+
memTimer *time.Ticker
17+
)
18+
19+
func StartActiveMemGuardian() {
20+
if memguardian.DefaultMemGuardian == nil {
21+
return
22+
}
23+
24+
memTimer := time.NewTicker(memguardian.DefaultInterval)
25+
go func() {
26+
for range memTimer.C {
27+
if IsLowOnMemory() {
28+
_ = GlobalGuardBytesBufferAlloc()
29+
} else {
30+
GlobalRestoreBytesBufferAlloc()
31+
}
32+
}
33+
}()
34+
}
35+
36+
func StopActiveMemGuardian() {
37+
if memguardian.DefaultMemGuardian == nil {
38+
return
39+
}
40+
41+
memTimer.Stop()
42+
}
43+
44+
func IsLowOnMemory() bool {
45+
if memguardian.DefaultMemGuardian != nil && memguardian.DefaultMemGuardian.Warning.Load() {
46+
return true
47+
}
48+
return false
49+
}
50+
51+
// GuardThreads on caller
52+
func GuardThreadsOrDefault(current int) int {
53+
if MaxThreadsOnLowMemory > 0 {
54+
return MaxThreadsOnLowMemory
55+
}
56+
57+
fraction := int(current / 5)
58+
if fraction > 0 {
59+
return fraction
60+
}
61+
62+
return 1
63+
}
64+
65+
var muGlobalChange sync.Mutex
66+
67+
// Global setting
68+
func GlobalGuardBytesBufferAlloc() error {
69+
if muGlobalChange.TryLock() {
70+
return nil
71+
72+
}
73+
defer muGlobalChange.Unlock()
74+
75+
// if current capacity was not reduced decrease it
76+
if MaxBytesBufferAllocOnLowMemory > 0 && httputil.DefaultBytesBufferAlloc == httputil.GetPoolSize() {
77+
gologger.Debug().Msgf("reducing bytes.buffer pool size to: %d", MaxBytesBufferAllocOnLowMemory)
78+
delta := httputil.GetPoolSize() - int64(MaxBytesBufferAllocOnLowMemory)
79+
return httputil.ChangePoolSize(-delta)
80+
}
81+
82+
return nil
83+
}
84+
85+
// Global setting
86+
func GlobalRestoreBytesBufferAlloc() {
87+
if muGlobalChange.TryLock() {
88+
return
89+
90+
}
91+
defer muGlobalChange.Unlock()
92+
93+
if httputil.DefaultBytesBufferAlloc != httputil.GetPoolSize() {
94+
delta := httputil.DefaultBytesBufferAlloc - httputil.GetPoolSize()
95+
gologger.Debug().Msgf("restoring bytes.buffer pool size to: %d", httputil.DefaultBytesBufferAlloc)
96+
_ = httputil.ChangePoolSize(delta)
97+
}
98+
}

‎pkg/protocols/common/protocolstate/state.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,16 @@ import (
1818
)
1919

2020
// Dialer is a shared fastdialer instance for host DNS resolution
21-
var Dialer *fastdialer.Dialer
21+
var (
22+
Dialer *fastdialer.Dialer
23+
)
2224

2325
// Init creates the Dialer instance based on user configuration
2426
func Init(options *types.Options) error {
2527
if Dialer != nil {
2628
return nil
2729
}
30+
2831
lfaAllowed = options.AllowLocalFileAccess
2932
opts := fastdialer.DefaultOptions
3033
if options.DialerTimeout > 0 {
@@ -142,6 +145,8 @@ func Init(options *types.Options) error {
142145
return Dialer.Dial(ctx, "tcp", addr)
143146
})
144147

148+
StartActiveMemGuardian()
149+
145150
return nil
146151
}
147152

@@ -202,4 +207,5 @@ func Close() {
202207
if Dialer != nil {
203208
Dialer.Close()
204209
}
210+
StopActiveMemGuardian()
205211
}

‎pkg/protocols/common/tostring/tostring.go‎

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎pkg/protocols/http/http.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
1515
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
1616
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
17+
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
1718
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
1819
httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
1920
"github.com/projectdiscovery/rawhttp"
@@ -411,6 +412,10 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
411412
}
412413
}
413414
if len(request.Payloads) > 0 {
415+
// specifically for http requests high concurrency and and threads will lead to memory exausthion, hence reduce the maximum parallelism
416+
if protocolstate.IsLowOnMemory() {
417+
request.Threads = protocolstate.GuardThreadsOrDefault(request.Threads)
418+
}
414419
// if we have payloads, adjust threads if none specified
415420
request.Threads = options.GetThreadsForNPayloadRequests(request.Requests(), request.Threads)
416421
}

‎pkg/protocols/http/operators.go‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -112,21 +112,21 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw
112112
data[k] = v
113113
}
114114
for _, cookie := range resp.Cookies() {
115-
data[strings.ToLower(cookie.Name)] = cookie.Value
115+
request.setHashOrDefault(data, strings.ToLower(cookie.Name), cookie.Value)
116116
}
117117
for k, v := range resp.Header {
118118
k = strings.ToLower(strings.ReplaceAll(strings.TrimSpace(k), "-", "_"))
119-
data[k] = strings.Join(v, " ")
119+
request.setHashOrDefault(data, k, strings.Join(v, " "))
120120
}
121121
data["host"] = host
122122
data["type"] = request.Type().String()
123123
data["matched"] = matched
124-
data["request"] = rawReq
125-
data["response"] = rawResp
124+
request.setHashOrDefault(data, "request", rawReq)
125+
request.setHashOrDefault(data, "response", rawResp)
126126
data["status_code"] = resp.StatusCode
127-
data["body"] = body
128-
data["all_headers"] = headers
129-
data["header"] = headers
127+
request.setHashOrDefault(data, "body", body)
128+
request.setHashOrDefault(data, "all_headers", headers)
129+
request.setHashOrDefault(data, "header", headers)
130130
data["duration"] = duration.Seconds()
131131
data["template-id"] = request.options.TemplateID
132132
data["template-info"] = request.options.TemplateInfo
@@ -140,6 +140,15 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw
140140
return data
141141
}
142142

143+
// TODO: disabling hdd storage while testing backpressure mechanism
144+
func (request *Request) setHashOrDefault(data output.InternalEvent, k string, v string) {
145+
// if hash, err := request.options.Storage.SetString(v); err == nil {
146+
// data[k] = hash
147+
// } else {
148+
data[k] = v
149+
//}
150+
}
151+
143152
// MakeResultEvent creates a result event from internal wrapped event
144153
func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
145154
return protocols.MakeDefaultResultEvent(request, wrapped)

‎pkg/protocols/http/request.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import (
2828
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
2929
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
3030
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
31-
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/tostring"
31+
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
3232
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
3333
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httputils"
3434
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signer"
@@ -168,6 +168,10 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
168168
// Workers that keeps enqueuing new requests
169169
maxWorkers := request.Threads
170170

171+
if protocolstate.IsLowOnMemory() {
172+
maxWorkers = protocolstate.GuardThreadsOrDefault(request.Threads)
173+
}
174+
171175
// Stop-at-first-match logic while executing requests
172176
// parallely using threads
173177
shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch)
@@ -684,7 +688,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
684688
// In case of interactsh markers and request times out, still send
685689
// a callback event so in case we receive an interaction, correlation is possible.
686690
// Also, to log failed use-cases.
687-
outputEvent := request.responseToDSLMap(&http.Response{}, input.MetaInput.Input, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta)
691+
outputEvent := request.responseToDSLMap(&http.Response{}, input.MetaInput.Input, formedURL, convUtil.String(dumpedRequest), "", "", "", 0, generatedRequest.meta)
688692
if i := strings.LastIndex(hostname, ":"); i != -1 {
689693
hostname = hostname[:i]
690694
}
@@ -695,8 +699,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
695699
outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
696700
}
697701

698-
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
699-
if request.CompiledOperators != nil {
702+
event := &output.InternalWrappedEvent{}
703+
if request.CompiledOperators != nil && request.CompiledOperators.HasDSL() {
700704
event.InternalEvent = outputEvent
701705
}
702706
callback(event)

0 commit comments

Comments
 (0)