Merge pull request #180 from lyricnz/feature/resample-data

Resample portfolio data to consistent time-interval before charting

Author: lyricnz

cointop/chart.go

@@ -2,16 +2,24 @@ package cointop
 
 import (
 	"fmt"
+	"math"
 	"sort"
 	"sync"
 	"time"
 
 	"github.com/miguelmota/cointop/pkg/chartplot"
+	"github.com/miguelmota/cointop/pkg/timedata"
 	"github.com/miguelmota/cointop/pkg/timeutil"
 	"github.com/miguelmota/cointop/pkg/ui"
 	log "github.com/sirupsen/logrus"
 )
 
+// PriceData is the time-series data for a Coin used when building a Portfolio view for chart
+type PriceData struct {
+	coin *Coin
+	data [][]float64
+}
+
 // ChartView is structure for chart view
 type ChartView = ui.View
 
@@ -120,7 +128,7 @@ func (ct *Cointop) ChartPoints(symbol string, name string) error {
 	start := nowseconds - int64(rangeseconds.Seconds())
 	end := nowseconds
 
-	var data []float64
+	var cacheData [][]float64
 
 	keyname := symbol
 	if keyname == "" {
@@ -131,21 +139,18 @@ func (ct *Cointop) ChartPoints(symbol string, name string) error {
 	cached, found := ct.cache.Get(cachekey)
 	if found {
 		// cache hit
-		data, _ = cached.([]float64)
+		cacheData, _ = cached.([][]float64)
 		log.Debug("ChartPoints() soft cache hit")
 	}
 
-	if len(data) == 0 {
+	if len(cacheData) == 0 {
 		if symbol == "" {
 			convert := ct.State.currencyConversion
 			graphData, err := ct.api.GetGlobalMarketGraphData(convert, start, end)
 			if err != nil {
 				return nil
 			}
-			for i := range graphData.MarketCapByAvailableSupply {
-				price := graphData.MarketCapByAvailableSupply[i][1]
-				data = append(data, price)
-			}
+			cacheData = graphData.MarketCapByAvailableSupply
 		} else {
 			convert := ct.State.currencyConversion
 			graphData, err := ct.api.GetCoinGraphData(convert, symbol, name, start, end)
@@ -156,20 +161,33 @@ func (ct *Cointop) ChartPoints(symbol string, name string) error {
 			sort.Slice(sorted[:], func(i, j int) bool {
 				return sorted[i][0] < sorted[j][0]
 			})
-			for i := range sorted {
-				price := sorted[i][1]
-				data = append(data, price)
-			}
+			cacheData = sorted
 		}
 
-		ct.cache.Set(cachekey, data, 10*time.Second)
+		ct.cache.Set(cachekey, cacheData, 10*time.Second)
 		if ct.filecache != nil {
 			go func() {
-				ct.filecache.Set(cachekey, data, 24*time.Hour)
+				ct.filecache.Set(cachekey, cacheData, 24*time.Hour)
 			}()
 		}
 	}
 
+	// Resample cachedata
+	timeQuantum := timedata.CalculateTimeQuantum(cacheData)
+	newStart := time.Unix(start, 0).Add(timeQuantum)
+	newEnd := time.Unix(end, 0).Add(-timeQuantum)
+	timeData := timedata.ResampleTimeSeriesData(cacheData, float64(newStart.UnixMilli()), float64(newEnd.UnixMilli()), chart.GetChartDataSize(maxX))
+
+	// Extract just the values from the data
+	var data []float64
+	for i := range timeData {
+		value := timeData[i][1]
+		if math.IsNaN(value) {
+			value = 0.0
+		}
+		data = append(data, value)
+	}
+
 	chart.SetData(data)
 	ct.State.chartPoints = chart.GetChartPoints(maxX)
 
@@ -202,7 +220,7 @@ func (ct *Cointop) PortfolioChart() error {
 	start := nowseconds - int64(rangeseconds.Seconds())
 	end := nowseconds
 
-	var data []float64
+	var allCacheData []PriceData
 	portfolio := ct.GetPortfolioSlice()
 	chartname := ct.SelectedCoinName()
 	for _, p := range portfolio {
@@ -217,46 +235,65 @@ func (ct *Cointop) PortfolioChart() error {
 			continue
 		}
 
-		var graphData []float64
+		var cacheData [][]float64 // [][time,value]
 		cachekey := ct.CompositeCacheKey(p.Symbol, p.Name, convert, selectedChartRange)
 		cached, found := ct.cache.Get(cachekey)
 		if found {
 			// cache hit
-			graphData, _ = cached.([]float64)
+			cacheData, _ = cached.([][]float64)
 			log.Debug("PortfolioChart() soft cache hit")
 		} else {
 			if ct.filecache != nil {
-				ct.filecache.Get(cachekey, &graphData)
+				ct.filecache.Get(cachekey, &cacheData)
 			}
 
-			if len(graphData) == 0 {
+			if len(cacheData) == 0 {
 				time.Sleep(2 * time.Second)
 
 				apiGraphData, err := ct.api.GetCoinGraphData(convert, p.Symbol, p.Name, start, end)
 				if err != nil {
 					return err
 				}
-				sorted := apiGraphData.Price
-				sort.Slice(sorted[:], func(i, j int) bool {
-					return sorted[i][0] < sorted[j][0]
+
+				cacheData = apiGraphData.Price
+				sort.Slice(cacheData[:], func(i, j int) bool {
+					return cacheData[i][0] < cacheData[j][0]
 				})
-				for i := range sorted {
-					price := sorted[i][1]
-					graphData = append(graphData, price)
-				}
 			}
 
-			ct.cache.Set(cachekey, graphData, 10*time.Second)
+			ct.cache.Set(cachekey, cacheData, 10*time.Second)
 			if ct.filecache != nil {
 				go func() {
-					ct.filecache.Set(cachekey, graphData, 24*time.Hour)
+					ct.filecache.Set(cachekey, cacheData, 24*time.Hour)
 				}()
 			}
 		}
 
-		for i := range graphData {
-			price := graphData[i]
-			sum := p.Holdings * price
+		allCacheData = append(allCacheData, PriceData{p, cacheData})
+	}
+
+	// Use the gap between price samples to adjust start/end in by one interval
+	var timeQuantum time.Duration
+	for _, cacheData := range allCacheData {
+		timeQuantum = timedata.CalculateTimeQuantum(cacheData.data)
+		if timeQuantum != 0 {
+			break // use the first one
+		}
+	}
+	newStart := time.Unix(start, 0).Add(timeQuantum)
+	newEnd := time.Unix(end, 0).Add(-timeQuantum)
+
+	// Resample and sum data
+	var data []float64
+	for _, cacheData := range allCacheData {
+		coinData := timedata.ResampleTimeSeriesData(cacheData.data, float64(newStart.UnixMilli()), float64(newEnd.UnixMilli()), chart.GetChartDataSize(maxX))
+		// sum (excluding NaN)
+		for i := range coinData {
+			price := coinData[i][1]
+			if math.IsNaN(price) {
+				price = 0.0
+			}
+			sum := cacheData.coin.Holdings * price
 			if i < len(data) {
 				data[i] += sum
 			} else {

pkg/chartplot/chartplot.go

@@ -1,8 +1,6 @@
 package chartplot
 
 import (
-	"math"
-
 	"github.com/miguelmota/cointop/pkg/termui"
 )
 
@@ -53,13 +51,18 @@ func (c *ChartPlot) SetBorder(enabled bool) {
 func (c *ChartPlot) SetData(data []float64) {
 	// NOTE: edit `termui.LineChart.shortenFloatVal(float64)` to not
 	// use exponential notation.
+	// NOTE: data should be the correct width for rendering - see GetChartDataSize()
 	c.t.Data = data
 }
 
+// GetChartDataSize ...
+func (c *ChartPlot) GetChartDataSize(width int) int {
+	axisYWidth := 30
+	return (width * 2) - axisYWidth
+}
+
 // GetChartPoints ...
 func (c *ChartPlot) GetChartPoints(width int) [][]rune {
-	axisYWidth := 30
-	c.t.Data = interpolateData(c.t.Data, (width*2)-axisYWidth)
 	termui.Body = termui.NewGrid()
 	termui.Body.Width = width
 	termui.Body.AddRows(
@@ -86,24 +89,3 @@ func (c *ChartPlot) GetChartPoints(width int) [][]rune {
 
 	return points
 }
-
-func interpolateData(data []float64, width int) []float64 {
-	var res []float64
-	if len(data) == 0 {
-		return res
-	}
-	stepFactor := float64(len(data)-1) / float64(width-1)
-	res = append(res, data[0])
-	for i := 1; i < width-1; i++ {
-		step := float64(i) * stepFactor
-		before := math.Floor(step)
-		after := math.Ceil(step)
-		atPoint := step - before
-		pointBefore := data[int(before)]
-		pointAfter := data[int(after)]
-		interpolated := pointBefore + (pointAfter-pointBefore)*atPoint
-		res = append(res, interpolated)
-	}
-	res = append(res, data[len(data)-1])
-	return res
-}

pkg/timedata/timedata.go

@@ -0,0 +1,51 @@
+package timedata
+
+import (
+	"math"
+	"sort"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// Resample the [timestamp,value] data given to numsteps between start-end (returns numSteps+1 points).
+// If the data does not extend past start/end then there will likely be NaN in the output data.
+func ResampleTimeSeriesData(data [][]float64, start float64, end float64, numSteps int) [][]float64 {
+	var newData [][]float64
+	l := len(data)
+	step := (end - start) / float64(numSteps)
+	for pos := start; pos <= end; pos += step {
+		idx := sort.Search(l, func(i int) bool { return data[i][0] >= pos })
+		var val float64
+		if idx == 0 {
+			val = math.NaN() // off the left
+		} else if idx == l {
+			val = math.NaN() // off the right
+		} else {
+			// between two points - linear interpolation
+			left := data[idx-1]
+			right := data[idx]
+			dvdt := (right[1] - left[1]) / (right[0] - left[0])
+			val = left[1] + (pos-left[0])*dvdt
+		}
+		newData = append(newData, []float64{pos, val})
+	}
+	return newData
+}
+
+// Assuming that the [timestamp,value] data provided is roughly evenly spaced, calculate that interval.
+func CalculateTimeQuantum(data [][]float64) time.Duration {
+	if len(data) > 1 {
+		minTime := time.UnixMilli(int64(data[0][0]))
+		maxTime := time.UnixMilli(int64(data[len(data)-1][0]))
+		return time.Duration(int64(maxTime.Sub(minTime)) / int64(len(data)-1))
+	}
+	return 0
+}
+
+// Print out all the [timestamp,value] data provided
+func DebugLogPriceData(data [][]float64) {
+	for i := range data {
+		log.Debugf("%s %.2f", time.Unix(int64(data[i][0]/1000), 0), data[i][1])
+	}
+}