End-to-end testing

.github/workflows/test-e2e.yml

@@ -0,0 +1,59 @@
+name: End-to-end Test
+
+on:
+  pull_request:
+  push:
+    branches: ["main"]
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v5
+      - name: Cache test dependencies
+        uses: actions/cache@v4
+        id: cache-dependencies
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+            /usr/local/bin
+            $GOPATH/bin
+            /lib
+          key: ${{ runner.os }}-${{ hashFiles('/usr/local/bin', '$GOPATH/bin', '/lib', '**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-
+      - uses: actions/[email protected]
+        if: steps.cache-dependencies.outputs.cache-it != 'true'
+        with:
+          repository: JackalLabs/canine-chain
+          ref: v4.6.2-alpha.1
+          path: canine-chain
+      - name: Install canined
+        if: steps.cache-dependencies.outputs.cache-it != 'true'
+        working-directory: ./canine-chain
+        run: |
+          sudo wget -O /lib/libwasmvm.x86_64.so https://github.com/CosmWasm/wasmvm/raw/v1.2.6/internal/api/libwasmvm.x86_64.so
+          make install
+      - name: Install venom
+        if: steps.cache-dependencies.outputs.cache-it != 'true'
+        run: curl https://github.com/ovh/venom/releases/download/v1.2.0/venom.linux-amd64 -L -o /usr/local/bin/venom && chmod +x /usr/local/bin/venom
+      - uses: actions/[email protected]
+        with:
+          path: sequoia
+      - name: Get sequoia build pkgs
+        if: steps.cache-dependencies.outputs.cache-it != 'true'
+        working-directory: ./sequoia
+        run: go mod download
+      - name: Install sequoia
+        working-directory: ./sequoia
+        run: make install
+      - name: Test
+        shell: bash
+        working-directory: ./sequoia/test
+        run: venom run -vv e2e/init.yaml `find e2e/ -type f -name "*_test-*.yaml" | sort`
+      - name: Stop containers
+        working-directory: ./sequoia/test
+        run: docker compose down
+

Dockerfile

@@ -0,0 +1,25 @@
+FROM golang:1.24
+
+ARG RPC_ADDR="http://localhost:26657"
+ARG GRPC_ADDR="localhost:9090"
+
+WORKDIR /usr/src/app
+
+# pre-copy/cache go.mod for pre-downloading dependencies and only redownloading them in subsequent builds if they change
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+RUN make install
+
+# RPC GRPC api ipfs
+EXPOSE 26657 9090 3333 4005
+
+WORKDIR /root/.sequoia
+
+RUN sequoia init
+
+RUN sed -i -e "s/http:\/\/localhost:26657/${RPC_ADDR}/" config.yaml
+RUN sed -i -e "s/localhost:9090/${GRPC_ADDR}/" config.yaml
+
+CMD ["sequoia", "start"]

Makefile

@@ -10,7 +10,7 @@ COMMIT := $(shell git log -1 --format='%H')
 ldflags = -X github.com/JackalLabs/sequoia/config.COMMIT=$(COMMIT) \
 		  -X github.com/JackalLabs/sequoia/config.VERSION=$(VERSION)
 
-all: lint test-unit
+all: lint test
 
 install:
 	@go install -ldflags '$(ldflags)'
@@ -69,7 +69,21 @@ format:
 ###                           Tests & Simulation                            ###
 ###############################################################################
 
+test: test-unit test-e2e
+
 test-unit:
 	@echo "Executing unit tests..."
 	@go test -mod=readonly -v -coverprofile coverage.txt ./...
-.PHONY: test-unit
+
+test-e2e:
+	@echo "Executing e2e tests..."
+	@venom run ./test/e2e/*.yml
+
+test-e2e-dev-mode:
+	@echo "Executing e2e tests..."
+	@venom run --var dev-mode=true ./test/e2e/*.yml
+
+test-clean:
+	rm venom*.log
+
+.PHONY: test-unit test-e2e test-e2e-dev-mode test test-clean

go.mod

@@ -51,6 +51,7 @@ require (
 	github.com/adlio/schema v1.3.6 // indirect
 	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
 	github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect
+	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect

go.sum

@@ -140,6 +140,8 @@ github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
+github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0=

network/downloads.go

@@ -2,6 +2,8 @@ package network
 
 import (
 	"bytes"
+	"compress/flate"
+	"compress/gzip"
 	"context"
 	"encoding/json"
 	"errors"
@@ -11,6 +13,8 @@ import (
 	"os"
 	"time"
 
+	"github.com/andybalholm/brotli"
+
 	apiTypes "github.com/JackalLabs/sequoia/api/types"
 
 	ipfslite "github.com/hsanjuan/ipfs-lite"
@@ -77,7 +81,7 @@ func DownloadFile(f *file_system.FileSystem, merkle []byte, owner string, start
 // Returns the number of bytes written, or an error if the download or write fails. Timeout and HTTP errors are
 // reported with detailed messages.
 func DownloadFileFromURL(f *file_system.FileSystem, url string, merkle []byte, owner string, start int64, chunkSize int64, proofType int64, ipfsParams *ipfslite.AddParams, fileSize int64) (int, error) {
-	log.Info().Msg(fmt.Sprintf("Downloading %x from %s...", merkle, url))
+	log.Info().Msgf("Downloading %x from %s...", merkle, url)
 
 	// Calculate timeout based on file size
 	// Base timeout + additional time for large files
@@ -154,14 +158,37 @@ func DownloadFileFromURL(f *file_system.FileSystem, url string, merkle []byte, o
 		return 0, fmt.Errorf("could not get file, code: %d | msg: %s", resp.StatusCode, e.Error)
 	}
 
+	var bodyReader io.Reader = resp.Body
+	contentEncoding := resp.Header.Get("Content-Encoding")
+	log.Info().Str("merkle", fmt.Sprintf("%x", merkle)).Msgf("Downloads content encoding: %s", contentEncoding)
+	switch contentEncoding {
+	case "gzip":
+		gz, err := gzip.NewReader(resp.Body)
+		if err != nil {
+			return 0, fmt.Errorf("failed to create gzip reader: %w", err)
+		}
+		//nolint:errcheck
+		defer gz.Close()
+		bodyReader = gz
+	case "deflate":
+		deflateReader := flate.NewReader(resp.Body)
+		//nolint:errcheck
+		defer deflateReader.Close()
+		bodyReader = deflateReader
+	case "br":
+		bodyReader = brotli.NewReader(resp.Body)
+	default:
+		// No compression or unsupported; use raw body
+	}
+
 	buff := bytes.NewBuffer([]byte{})
 
 	// Use TeeReader to monitor for context cancellation while copying
 	doneCh := make(chan struct{})
 	errCh := make(chan error, 1)
 
 	go func() {
-		_, err := io.Copy(buff, resp.Body)
+		_, err := io.Copy(buff, bodyReader)
 		if err != nil {
 			errCh <- err
 		}

test/README.md

@@ -0,0 +1,34 @@
+# End-to-End Testing
+
+### Requirements
+[canined](https://github.com/JackalLabs/canine-chain/releases)  
+[venom](https://github.com/ovh/venom/tree/master?tab=readme-ov-file#installing)  
+docker  
+docker compose  
+
+### Running Tests
+
+> Add `--var dev-mode=true` to skip tests that alter current setup files  
+> Add `-vv` flag to debug tests  
+
+Setup canined and sequoia containers
+```bash
+venom run e2e/init.yaml
+```
+Run tests:
+```bash
+venom run ./test/e2e/*_test-*.yaml
+```
+Or all at once:
+```bash
+venom run e2e/init.yaml `find e2e/ -type f -name "*_test-*.yaml" | sort`
+```
+
+Stop and delete test containers
+```bash
+docker compose down
+```
+
+### Writing Tests
+
+If one test depends on the other, make the file name in [sort](https://en.wikipedia.org/wiki/Sort_(Unix))ed order.

test/compose.yaml

@@ -0,0 +1,27 @@
+services:
+  chain:
+    build: ./resources/canine
+    image: e2e-test-chain
+    ports: 
+    - "26656:26656"
+    - "26657:26657"
+    - "1317:1317"
+    - "9092:9092"
+    - "9090:9090"
+
+  sequoia:
+    build:
+      context: ../
+      args:
+        RPC_ADDR: 'http:\/\/chain:26657'
+        GRPC_ADDR: "chain:9090"
+    image: e2e-test-sequoia
+    volumes:
+    - $HOME/.sequoia/provider_wallet.json:/root/.sequoia/provider_wallet.json
+    depends_on:
+    - chain
+    entrypoint: ["/bin/sh", "-c", "sleep 10 && sequoia start"]
+    ports:
+    - "3333:3333"
+    - "4005:4005"
+

test/e2e/01_test-post-file.yaml

@@ -0,0 +1,21 @@
+name: Test Sequoia File Upload
+description: Test file upload to sequoia
+vars:
+  dev-mode: false
+
+testcases:
+- name: Start sequoia and canine-chain containers
+  steps:
+  - script: docker compose up -d; sleep 5
+
+- name: Upload file to sequoia
+
+  steps:
+  - name: Upload file using canined
+    script: 'canined tx storage post ../resources/logo.png 500000 --dest http://localhost:3333 --max_proofs 1 --from j1 -y'
+    retry: 3
+    delay: 10
+    assertions:
+    - result.systemerr ShouldBeBlank
+    - result.systemout ShouldContainSubstring 1ab5cd31e2d001fedad0b92abb01cf36f16e1c395c7880a67e2d18d39d81040d7732fa5a33d81bb33578243c54d60c8d5a56de604b1827243dc1feab39b4b3a6
+    - result.code ShouldEqual 0

test/e2e/init.yaml

@@ -0,0 +1,58 @@
+name: Test Sequoia Initialization
+description: Test provider Initialization on chain
+vars:
+  dev-mode: false
+  faucet: jkl1hj5fveer5cjtn4wd6wstzugjfdxzl0xpljur4u
+
+testcases:
+- name: Initialize canine-chain and sequoia configs
+  skip:
+  - dev-mode ShouldEqual false
+
+  steps:
+  - name: Delete existing docker containers
+    script: "docker compose down"
+
+  - name: Initialize sequoia config
+    script: "rm -rf $HOME/.sequoia; sequoia init"
+
+  - name: Initialize canined config
+    script: "rm -rf $HOME/.canine; ../resources/canine/setup-chain.sh"
+
+  - name: Build and start canined container
+    script: "docker compose up -d chain; sleep 5" # wait for the chain to open api
+
+- name: Fund sequoia provider wallet
+  skip:
+  - dev-mode ShouldEqual false
+
+  steps:
+  - name: Get provider address
+    script: "sequoia wallet address | sed 's/Provider Address: //'"
+    vars:
+      address:
+        from: result.systemout
+
+  - name: Fund provider
+    info: "provider address {{.address}}"
+    script: "canined tx bank send {{.faucet}} {{.address}} 100000000000ujkl -y"
+    retry: 3
+    delay: 10
+    assertions:
+    - result.code ShouldEqual 0
+
+- name: Start sequoia
+  steps:
+  - name: Build and start sequoia container
+    script: "docker compose up -d sequoia"
+    assertions:
+    - result.code ShouldEqual 0
+
+- name: Verify provider registration on chain
+  steps:
+  - script: "canined query storage show-providers {{.Fund-sequoia-provider-wallet.address}}"
+    retry: 3
+    delay: 10 # bump this number if provider takes longer to start up
+    assertions:
+    - result.systemout ShouldContainSubstring {{.Fund-sequoia-provider-wallet.address}}
+    - result.code ShouldEqual 0

test/resources/canine/Dockerfile

@@ -0,0 +1,48 @@
+# syntax=docker/dockerfile:1
+
+# Start from a base Golang image to build the binary
+FROM golang:1.23 AS builder
+
+RUN apt-get update && apt-get install -y jq
+
+WORKDIR /lib
+
+COPY ./setup-chain.sh .
+
+RUN wget https://github.com/CosmWasm/wasmvm/raw/v1.2.6/internal/api/libwasmvm.x86_64.so
+
+WORKDIR /app
+
+RUN git clone https://github.com/JackalLabs/canine-chain.git .
+
+RUN git checkout $(git describe --abbrev=0 --tags)
+
+RUN make build
+
+# Use a smaller image to run the binary
+FROM debian:bookworm-slim
+
+WORKDIR /root
+
+RUN apt-get update && apt-get install -y \
+	build-essential \
+	git \
+	curl \
+	bash \
+	wget \
+	jq
+
+COPY --from=builder /app/build/canined /usr/local/bin/canined
+COPY --from=builder /lib/libwasmvm.x86_64.so /lib/libwasmvm.x86_64.so
+COPY --from=builder /lib/setup-chain.sh /root/setup-chain.sh
+
+WORKDIR /root
+
+RUN ./setup-chain.sh
+
+# Expose default RPC, P2P, and REST ports
+EXPOSE 26656 26657 1317 9092 9090
+
+WORKDIR /root/.canine
+
+CMD ["canined", "start"]