metrics: add metrics monitoring system CPU and memory

metrics/cpu.go

@@ -0,0 +1,24 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package metrics
+
+// CPUStats is the system and process CPU stats.
+type CPUStats struct {
+	GlobalTime int64 // Time spent by the CPU working on all processes
+	GlobalWait int64 // Time spent by waiting on disk for all processes
+	LocalTime  int64 // Time spent by the CPU working on this process
+}

metrics/cpu_disabled.go

@@ -0,0 +1,24 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+//go:build ios
+// +build ios
+
+package metrics
+
+// ReadCPUStats retrieves the current CPU stats. Internally this uses `gosigar`,
+// which is not supported on the platforms in this file.
+func ReadCPUStats(stats *CPUStats) {}

metrics/cpu_enabled.go

@@ -0,0 +1,40 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+//go:build !ios
+// +build !ios
+
+package metrics
+
+import (
+	"github.com/shirou/gopsutil/cpu"
+	"github.com/tomochain/tomochain/log"
+)
+
+// ReadCPUStats retrieves the current CPU stats.
+func ReadCPUStats(stats *CPUStats) {
+	// passing false to request all cpu times
+	timeStats, err := cpu.Times(false)
+	if err != nil {
+		log.Error("Could not read cpu stats", "err", err)
+		return
+	}
+	// requesting all cpu times will always return an array with only one time stats entry
+	timeStat := timeStats[0]
+	stats.GlobalTime = int64((timeStat.User + timeStat.Nice + timeStat.System) * cpu.ClocksPerSec)
+	stats.GlobalWait = int64((timeStat.Iowait) * cpu.ClocksPerSec)
+	stats.LocalTime = getProcessCPUTime()
+}

metrics/cputime_nop.go

@@ -0,0 +1,26 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+//go:build windows || js
+// +build windows js
+
+package metrics
+
+// getProcessCPUTime returns 0 on Windows as there is no system call to resolve
+// the actual process' CPU time.
+func getProcessCPUTime() int64 {
+	return 0
+}

metrics/cputime_unix.go

@@ -0,0 +1,36 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+//go:build !windows && !js
+// +build !windows,!js
+
+package metrics
+
+import (
+	syscall "golang.org/x/sys/unix"
+
+	"github.com/tomochain/tomochain/log"
+)
+
+// getProcessCPUTime retrieves the process' CPU time since program startup.
+func getProcessCPUTime() int64 {
+	var usage syscall.Rusage
+	if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil {
+		log.Warn("Failed to retrieve CPU time", "err", err)
+		return 0
+	}
+	return int64(usage.Utime.Sec+usage.Stime.Sec)*100 + int64(usage.Utime.Usec+usage.Stime.Usec)/10000 //nolint:unconvert
+}

metrics/metrics.go

@@ -61,41 +61,56 @@ func CollectProcessMetrics(refresh time.Duration) {
 	if !Enabled {
 		return
 	}
+	refreshFreq := int64(refresh / time.Second)
 	// Create the various data collectors
+	cpuStats := make([]*CPUStats, 2)
 	memstats := make([]*runtime.MemStats, 2)
 	diskstats := make([]*DiskStats, 2)
 	for i := 0; i < len(memstats); i++ {
+		cpuStats[i] = new(CPUStats)
 		memstats[i] = new(runtime.MemStats)
 		diskstats[i] = new(DiskStats)
 	}
 	// Define the various metrics to collect
-	memAllocs := GetOrRegisterMeter("system/memory/allocs", DefaultRegistry)
-	memFrees := GetOrRegisterMeter("system/memory/frees", DefaultRegistry)
-	memInuse := GetOrRegisterMeter("system/memory/inuse", DefaultRegistry)
-	memPauses := GetOrRegisterMeter("system/memory/pauses", DefaultRegistry)
-
-	var diskReads, diskReadBytes, diskWrites, diskWriteBytes Meter
-	var diskReadBytesCounter, diskWriteBytesCounter Counter
-	if err := ReadDiskStats(diskstats[0]); err == nil {
-		diskReads = GetOrRegisterMeter("system/disk/readcount", DefaultRegistry)
-		diskReadBytes = GetOrRegisterMeter("system/disk/readdata", DefaultRegistry)
-		diskReadBytesCounter = GetOrRegisterCounter("system/disk/readbytes", DefaultRegistry)
-		diskWrites = GetOrRegisterMeter("system/disk/writecount", DefaultRegistry)
-		diskWriteBytes = GetOrRegisterMeter("system/disk/writedata", DefaultRegistry)
+	var (
+		cpuSysLoad    = GetOrRegisterGauge("system/cpu/sysload", DefaultRegistry)
+		cpuSysWait    = GetOrRegisterGauge("system/cpu/syswait", DefaultRegistry)
+		cpuProcLoad   = GetOrRegisterGauge("system/cpu/procload", DefaultRegistry)
+		cpuThreads    = GetOrRegisterGauge("system/cpu/threads", DefaultRegistry)
+		cpuGoroutines = GetOrRegisterGauge("system/cpu/goroutines", DefaultRegistry)
+
+		memPauses = GetOrRegisterMeter("system/memory/pauses", DefaultRegistry)
+		memAllocs = GetOrRegisterMeter("system/memory/allocs", DefaultRegistry)
+		memFrees  = GetOrRegisterMeter("system/memory/frees", DefaultRegistry)
+		memHeld   = GetOrRegisterGauge("system/memory/held", DefaultRegistry)
+		memUsed   = GetOrRegisterGauge("system/memory/used", DefaultRegistry)
+
+		diskReads             = GetOrRegisterMeter("system/disk/readcount", DefaultRegistry)
+		diskReadBytes         = GetOrRegisterMeter("system/disk/readdata", DefaultRegistry)
+		diskReadBytesCounter  = GetOrRegisterCounter("system/disk/readbytes", DefaultRegistry)
+		diskWrites            = GetOrRegisterMeter("system/disk/writecount", DefaultRegistry)
+		diskWriteBytes        = GetOrRegisterMeter("system/disk/writedata", DefaultRegistry)
 		diskWriteBytesCounter = GetOrRegisterCounter("system/disk/writebytes", DefaultRegistry)
-	} else {
-		log.Debug("Failed to read disk metrics", "err", err)
-	}
+	)
+
 	// Iterate loading the different stats and updating the meters
 	for i := 1; ; i++ {
 		location1 := i % 2
 		location2 := (i - 1) % 2
 
+		ReadCPUStats(cpuStats[location1])
+		cpuSysLoad.Update((cpuStats[location1].GlobalTime - cpuStats[location2].GlobalTime) / refreshFreq)
+		cpuSysWait.Update((cpuStats[location1].GlobalWait - cpuStats[location2].GlobalWait) / refreshFreq)
+		cpuProcLoad.Update((cpuStats[location1].LocalTime - cpuStats[location2].LocalTime) / refreshFreq)
+		cpuThreads.Update(int64(threadCreateProfile.Count()))
+		cpuGoroutines.Update(int64(runtime.NumGoroutine()))
+
 		runtime.ReadMemStats(memstats[location1])
+		memPauses.Mark(int64(memstats[location1].PauseTotalNs - memstats[location2].PauseTotalNs))
 		memAllocs.Mark(int64(memstats[location1].Mallocs - memstats[location2].Mallocs))
 		memFrees.Mark(int64(memstats[location1].Frees - memstats[location2].Frees))
-		memInuse.Mark(int64(memstats[location1].Alloc - memstats[location2].Alloc))
-		memPauses.Mark(int64(memstats[location1].PauseTotalNs - memstats[location2].PauseTotalNs))
+		memHeld.Update(int64(memstats[location1].HeapSys - memstats[location1].HeapReleased))
+		memUsed.Update(int64(memstats[location1].Alloc))
 
 		if ReadDiskStats(diskstats[location1]) == nil {
 			diskReads.Mark(diskstats[location1].ReadCount - diskstats[location2].ReadCount)