Add HAR file writing support (#439)

cmd/root.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/hahwul/dalfox/v2/pkg/har"
 	"io/ioutil"
 	"os"
 	"time"
@@ -18,6 +19,7 @@ var optionsStr = make(map[string]string)
 var optionsBool = make(map[string]bool)
 var header, p, ignoreParams []string
 var config, cookie, data, customPayload, userAgent, blind, output, format, foundAction, proxy, grep, cookieFromRaw string
+var harFilePath string
 var ignoreReturn, miningWord, method, customAlertValue, customAlertType, remotePayloads, remoteWordlists string
 var timeout, concurrence, delay int
 var onlyDiscovery, silence, followRedirect, mining, findingDOM, noColor, noSpinner, onlyCustomPayload, debug, useDeepDXSS, outputAll bool
@@ -35,6 +37,12 @@ var rootCmd = &cobra.Command{
 
 // Execute is run rootCmd
 func Execute() {
+	defer func() {
+		if options.HarWriter != nil {
+			options.HarWriter.Close()
+		}
+	}()
+
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Println(err)
 		os.Exit(1)
@@ -72,6 +80,7 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(&onlyPoC, "only-poc", "", "Shows only the PoC code for the specified pattern (g: grep / r: reflected / v: verified)\n * Example: --only-poc='g,v'")
 	rootCmd.PersistentFlags().StringVar(&pocType, "poc-type", "plain", "Select PoC type \n * Supported: plain/curl/httpie/http-request\n * Example: --poc-type='curl'")
 	rootCmd.PersistentFlags().StringVar(&reportFormat, "report-format", "plain", "Format of --report flag [plain/json]")
+	rootCmd.PersistentFlags().StringVar(&harFilePath, "har-file-path", "", "Path to save HAR of scan requests to")
 
 	//Int
 	rootCmd.PersistentFlags().IntVar(&timeout, "timeout", 10, "Second of timeout")
@@ -158,6 +167,18 @@ func initConfig() {
 	}
 	// var skipMiningDom, skipMiningDict, skipMiningAll, skipXSSScan, skipBAV bool
 
+	if harFilePath != "" {
+		f, err := os.Create(harFilePath)
+		if err != nil {
+			fmt.Println(err)
+		} else {
+			options.HarWriter, err = har.NewWriter(f, &har.Creator{Name: "dalfox", Version: printing.VERSION})
+			if err != nil {
+				fmt.Println(err)
+			}
+		}
+	}
+
 	if skipMiningAll {
 		options.FindingDOM = false
 		options.Mining = false

go.mod

@@ -17,7 +17,9 @@ require (
 	github.com/stretchr/testify v1.8.1
 	github.com/swaggo/echo-swagger v1.3.5
 	github.com/swaggo/swag v1.8.9
+	github.com/tidwall/sjson v1.2.5
 	github.com/tylerb/graceful v1.2.15
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
 )
 
@@ -48,6 +50,9 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/swaggo/files v0.0.0-20220728132757-551d4a08d97a // indirect
+	github.com/tidwall/gjson v1.14.2 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.1 // indirect
 	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect

go.sum

@@ -133,6 +133,14 @@ github.com/swaggo/files v0.0.0-20220728132757-551d4a08d97a/go.mod h1:lKJPbtWzJ9J
 github.com/swaggo/swag v1.8.1/go.mod h1:ugemnJsPZm/kRwFUnzBlbHRd0JY9zE1M4F+uy2pAaPQ=
 github.com/swaggo/swag v1.8.9 h1:kHtaBe/Ob9AZzAANfcn5c6RyCke9gG9QpH0jky0I/sA=
 github.com/swaggo/swag v1.8.9/go.mod h1:ezQVUUhly8dludpVk+/PuwJWvLLanB13ygV5Pr9enSk=
+github.com/tidwall/gjson v1.14.2 h1:6BBkirS0rAHjumnjHF6qgy5d2YAJ1TLIaFE2lzfOLqo=
+github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
+github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/tylerb/graceful v1.2.15 h1:B0x01Y8fsJpogzZTkDg6BDi6eMf03s01lEKGdrv83oA=
 github.com/tylerb/graceful v1.2.15/go.mod h1:LPYTbOYmUTdabwRt0TGhLllQ0MUNbs0Y5q1WXJOI9II=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
@@ -167,6 +175,7 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgk
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

pkg/har/client_tracer.go

@@ -0,0 +1,79 @@
+package har
+
+import (
+	"crypto/tls"
+	"net/http/httptrace"
+	"time"
+)
+
+func newClientTracer() (*clientTracer, *httptrace.ClientTrace) {
+	ct1 := &clientTracer{
+		startAt: time.Now(),
+	}
+
+	ct2 := &httptrace.ClientTrace{
+		GetConn:              ct1.GetConn,
+		GotConn:              ct1.GotConn,
+		PutIdleConn:          nil,
+		GotFirstResponseByte: ct1.GotFirstResponseByte,
+		Got100Continue:       nil,
+		Got1xxResponse:       nil,
+		DNSStart:             ct1.DNSStart,
+		DNSDone:              ct1.DNSDone,
+		ConnectStart:         nil,
+		ConnectDone:          nil,
+		TLSHandshakeStart:    ct1.TLSHandshakeStart,
+		TLSHandshakeDone:     ct1.TLSHandshakeDone,
+		WroteHeaderField:     nil,
+		WroteHeaders:         nil,
+		Wait100Continue:      nil,
+		WroteRequest:         ct1.WroteRequest,
+	}
+
+	return ct1, ct2
+}
+
+type clientTracer struct {
+	startAt           time.Time
+	connStart         time.Time
+	connObtained      time.Time
+	firstResponseByte time.Time
+	dnsStart          time.Time
+	dnsEnd            time.Time
+	tlsHandshakeStart time.Time
+	tlsHandshakeEnd   time.Time
+	writeRequest      time.Time
+	endAt             time.Time
+}
+
+func (ct *clientTracer) GetConn(hostPort string) {
+	ct.connStart = time.Now()
+}
+
+func (ct *clientTracer) GotConn(info httptrace.GotConnInfo) {
+	ct.connObtained = time.Now()
+}
+
+func (ct *clientTracer) GotFirstResponseByte() {
+	ct.firstResponseByte = time.Now()
+}
+
+func (ct *clientTracer) DNSStart(info httptrace.DNSStartInfo) {
+	ct.dnsStart = time.Now()
+}
+
+func (ct *clientTracer) DNSDone(info httptrace.DNSDoneInfo) {
+	ct.dnsEnd = time.Now()
+}
+
+func (ct *clientTracer) TLSHandshakeStart() {
+	ct.tlsHandshakeStart = time.Now()
+}
+
+func (ct *clientTracer) TLSHandshakeDone(tls.ConnectionState, error) {
+	ct.tlsHandshakeEnd = time.Now()
+}
+
+func (ct *clientTracer) WroteRequest(info httptrace.WroteRequestInfo) {
+	ct.writeRequest = time.Now()
+}

pkg/har/har_test.go

@@ -0,0 +1,120 @@
+package har_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"github.com/hahwul/dalfox/v2/pkg/har"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/tidwall/sjson"
+	"golang.org/x/sync/errgroup"
+	"net/http"
+	"testing"
+)
+
+var creator = &har.Creator{Name: "dalfox tests", Version: "0.0"}
+
+func TestSingleRequest(t *testing.T) {
+	buf := &bytes.Buffer{}
+
+	hw, err := har.NewWriter(buf, creator)
+	require.NoError(t, err)
+
+	rt := har.NewRoundTripper(nil, hw, nil)
+	c := &http.Client{Transport: rt}
+
+	resp, err := c.Get("https://example.com")
+	require.NoError(t, err)
+	assert.NotNil(t, resp)
+	assert.Equal(t, 200, resp.StatusCode)
+
+	err = hw.Close()
+	require.NoError(t, err)
+
+	assert.True(t, json.Valid(buf.Bytes()), "HAR file is not valid json")
+}
+
+func TestMultipleRequests(t *testing.T) {
+	buf := &bytes.Buffer{}
+
+	hw, err := har.NewWriter(buf, creator)
+	require.NoError(t, err)
+
+	rt := har.NewRoundTripper(nil, hw, nil)
+	c := &http.Client{Transport: rt}
+
+	g, _ := errgroup.WithContext(context.Background())
+	for idx := 0; idx < 5; idx++ {
+		g.Go(func() error {
+			resp, err := c.Get("https://example.com")
+			require.NoError(t, err)
+			assert.NotNil(t, resp)
+			assert.Equal(t, 200, resp.StatusCode)
+			return nil
+		})
+	}
+
+	err = g.Wait()
+	require.NoError(t, err)
+
+	err = hw.Close()
+	require.NoError(t, err)
+
+	harfile := har.File{}
+	err = json.Unmarshal(buf.Bytes(), &harfile)
+	require.NoError(t, err)
+
+	assert.Len(t, harfile.Log.Entries, 5)
+}
+
+func TestRewrite(t *testing.T) {
+	buf := &bytes.Buffer{}
+	ctx := context.Background()
+
+	hw, err := har.NewWriter(buf, creator)
+	require.NoError(t, err)
+
+	rt := har.NewRoundTripper(nil, hw, func(request *http.Request, response *http.Response, entry json.RawMessage) json.RawMessage {
+		messageID := har.MessageIDFromRequest(request)
+		entry, _ = sjson.SetBytes(entry, "_messageId", messageID)
+		return entry
+	})
+	require.NoError(t, err)
+
+	c := &http.Client{Transport: rt}
+
+	req, _ := http.NewRequestWithContext(ctx, "GET", "https://example.com", nil)
+	req = har.AddMessageIDToRequest(req)
+	firstMessageID := har.MessageIDFromRequest(req)
+
+	resp, err := c.Do(req)
+	require.NoError(t, err)
+	assert.NotNil(t, resp)
+
+	req, _ = http.NewRequestWithContext(ctx, "GET", "https://example.com/?a=b&c=d", nil)
+	req = har.AddMessageIDToRequest(req)
+	secondMessageID := har.MessageIDFromRequest(req)
+
+	resp, err = c.Do(req)
+	require.NoError(t, err)
+	assert.NotNil(t, resp)
+
+	err = hw.Close()
+	require.NoError(t, err)
+
+	// assert valid HAR files
+	harfile := har.File{}
+	err = json.Unmarshal(buf.Bytes(), &harfile)
+	require.NoError(t, err)
+
+	anything := map[string]interface{}{}
+	err = json.Unmarshal(buf.Bytes(), &anything)
+	require.NoError(t, err)
+
+	entries := anything["log"].(map[string]interface{})["entries"].([]interface{})
+	assert.Len(t, entries, 2)
+
+	assert.EqualValues(t, firstMessageID, entries[0].(map[string]interface{})["_messageId"])
+	assert.EqualValues(t, secondMessageID, entries[1].(map[string]interface{})["_messageId"])
+}

pkg/har/message_id.go

@@ -0,0 +1,31 @@
+package har
+
+import (
+	"context"
+	"net/http"
+	"sync/atomic"
+)
+
+var nextMessageID int64
+
+// NewMessageID returns an incrementing message ID. It is used for
+// correlating Dalfox PoCs with entries in a HAR file.
+func NewMessageID() int64 {
+	return atomic.AddInt64(&nextMessageID, 1)
+}
+
+type ctxkeyType int
+
+const ctxkey ctxkeyType = iota
+
+// MessageIDFromRequest returns the message ID associated with a *http.Request
+func MessageIDFromRequest(req *http.Request) int64 {
+	return req.Context().Value(ctxkey).(int64)
+}
+
+// AddMessageIDToRequest returns a new *http.Request with a message ID associated to it
+func AddMessageIDToRequest(req *http.Request) *http.Request {
+	messageID := NewMessageID()
+	ctx := context.WithValue(req.Context(), ctxkey, messageID)
+	return req.WithContext(ctx)
+}