Merge rc v1.8.0 feat governance fixes

.github/workflows/build_and_test.yml

@@ -1,17 +1,15 @@
-name: Build
+name: Build and smoke test
 
 on:
   pull_request:
-    branches: [ master, rc/* ]
-    types: [opened, ready_for_review]
-  push:
+    branches: [master, rc/*]
   workflow_dispatch:
 
 jobs:
   build:
     strategy:
       matrix:
-        runs-on: [ubuntu-latest]
+        runs-on: [ubuntu-latest, macos-13, macos-13-xlarge]
     runs-on: ${{ matrix.runs-on }}
     name: Build
     steps:
@@ -28,12 +26,23 @@ jobs:
         run: |
           go get -v -t -d ./...
           if [ -f Gopkg.toml ]; then
-              curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-              dep ensure
+            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+            dep ensure
           fi
+
       - name: Build
         run: |
           cd ${GITHUB_WORKSPACE}/cmd/node && go build .
+          cd ${GITHUB_WORKSPACE}/cmd/seednode && go build .
           cd ${GITHUB_WORKSPACE}/cmd/keygenerator && go build .
           cd ${GITHUB_WORKSPACE}/cmd/logviewer && go build .
           cd ${GITHUB_WORKSPACE}/cmd/termui && go build .
+
+      # On GitHub, we only run the short tests, and we only run them for some OS/ARCH combinations.
+      - name: Run tests
+        run: |
+          GOOS=$(go env GOOS)
+
+          if [[ "$GOOS" == darwin ]]; then
+            go test -short -v ./...
+          fi

.github/workflows/create_release.yml

@@ -15,7 +15,7 @@ jobs:
   build:
     strategy:
       matrix:
-        runs-on: [ubuntu-latest] # TODO add macos-latest when builds are possible on macs
+        runs-on: [ubuntu-latest, macos-13, macos-13-xlarge]
     runs-on: ${{ matrix.runs-on }}
     name: Build
     steps:
@@ -45,21 +45,23 @@ jobs:
           GOOS=$(go env GOOS)
           GOARCH=$(go env GOARCH)
           GOPATH=$(go env GOPATH)
-          ARCHIVE="multiversx_""$APP_VER_SHORT""_""$GOOS""_""$GOARCH"".tgz"
+          ARCHIVE="multiversx_""$APP_VER_SHORT""_""$GOOS""_""$GOARCH"".zip"
           BUILD_DIR=${GITHUB_WORKSPACE}/build
-          WASM_VERSION=$(cat go.mod | grep mx-chain-vm-v | sort -n | tail -n -1| awk -F '/' '{print$3}'| sed 's/ /@/g')
-          WASMER_DIR=${GOPATH}/pkg/mod/github.com/multiversx/${WASM_VERSION}/wasmer
+          VM_GO_VERSION=$(cat go.mod | grep mx-chain-vm-go | sort -n | tail -n -1| awk -F '/' '{print$3}'| sed 's/ /@/g')
+          VM_GO_DIR=${GOPATH}/pkg/mod/github.com/multiversx/${VM_GO_VERSION}
 
           echo "GOOS=${GOOS}" >> $GITHUB_ENV
           echo "GOARCH=${GOARCH}" >> $GITHUB_ENV
           echo "ARCHIVE=${ARCHIVE}" >> $GITHUB_ENV
           echo "BUILD_DIR=${BUILD_DIR}" >> $GITHUB_ENV
-          echo "WASMER_DIR=${WASMER_DIR}" >> $GITHUB_ENV
+          echo "VM_GO_VERSION=${VM_GO_VERSION}" >> $GITHUB_ENV
+          echo "VM_GO_DIR=${VM_GO_DIR}" >> $GITHUB_ENV
 
       - name: Build
         run: |
           mkdir -p ${BUILD_DIR}
           cd ${GITHUB_WORKSPACE}/cmd/node && go build -o "${BUILD_DIR}/node" -a -ldflags="-X main.appVersion=${APP_VER}"
+          cd ${GITHUB_WORKSPACE}/cmd/seednode && go build -o "${BUILD_DIR}/seednode" -a -ldflags="-X main.appVersion=${APP_VER}"
           cd ${GITHUB_WORKSPACE}/cmd/keygenerator && go build -o "${BUILD_DIR}/keygenerator" -a -ldflags="-X main.appVersion=${APP_VER}"
           cd ${GITHUB_WORKSPACE}/cmd/logviewer && go build -o "${BUILD_DIR}/logviewer" -a -ldflags="-X main.appVersion=${APP_VER}"
           cd ${GITHUB_WORKSPACE}/cmd/termui && go build -o "${BUILD_DIR}/termui" -a -ldflags="-X main.appVersion=${APP_VER}"
@@ -69,24 +71,68 @@ jobs:
           cd ${GITHUB_WORKSPACE}
 
           if [[ "$GOOS" == linux && "$GOARCH" == amd64 ]]; then
-            cp -f ${WASMER_DIR}/libwasmer_linux_amd64.so ${BUILD_DIR};
+            cp --verbose --no-preserve=mode,ownership ${VM_GO_DIR}/wasmer2/libvmexeccapi.so ${BUILD_DIR}/libvmexeccapi.so
+            cp --verbose --no-preserve=mode,ownership ${VM_GO_DIR}/wasmer/libwasmer_linux_amd64.so ${BUILD_DIR}/libwasmer_linux_amd64.so
           fi
+
+          # Actually, there's no runner for this combination (as of March 2024).
           if [[ "$GOOS" == linux && "$GOARCH" == arm64 ]]; then
-            cp -f ${WASMER_DIR}/libwasmer_linux_arm64.so ${BUILD_DIR};
+            cp --verbose --no-preserve=mode,ownership ${VM_GO_DIR}/wasmer2/libvmexeccapi_arm.so ${BUILD_DIR}/libvmexeccapi_arm.so
+            cp --verbose --no-preserve=mode,ownership ${VM_GO_DIR}/wasmer/libwasmer_linux_arm64_shim.so ${BUILD_DIR}/libwasmer_linux_arm64_shim.so
           fi
+
           if [[ "$GOOS" == darwin && "$GOARCH" == amd64 ]]; then
-            cp -f ${WASMER_DIR}/libwasmer_darwin_amd64.dylib ${BUILD_DIR};
+            cp -v ${VM_GO_DIR}/wasmer2/libvmexeccapi.dylib ${BUILD_DIR}/libvmexeccapi.dylib
+            cp -v ${VM_GO_DIR}/wasmer/libwasmer_darwin_amd64.dylib ${BUILD_DIR}/libwasmer_darwin_amd64.dylib
+          fi
+
+          if [[ "$GOOS" == darwin && "$GOARCH" == arm64 ]]; then
+            cp -v ${VM_GO_DIR}/wasmer2/libvmexeccapi_arm.dylib ${BUILD_DIR}/libvmexeccapi_arm.dylib
+            cp -v ${VM_GO_DIR}/wasmer/libwasmer_darwin_arm64_shim.dylib ${BUILD_DIR}/libwasmer_darwin_arm64_shim.dylib
           fi
 
-          cd ${BUILD_DIR}
-          tar czvf "${GITHUB_WORKSPACE}/${ARCHIVE}" *
-          stat ${GITHUB_WORKSPACE}/${ARCHIVE}
+          if [[ "$GOOS" == linux ]]; then
+            patchelf --set-rpath "\$ORIGIN" ${BUILD_DIR}/node
+            patchelf --set-rpath "\$ORIGIN" ${BUILD_DIR}/seednode
+
+            ldd ${BUILD_DIR}/node
+            ldd ${BUILD_DIR}/seednode
+          fi
+
+          if [[ "$GOOS" == darwin ]]; then
+            install_name_tool -add_rpath "@loader_path" ${BUILD_DIR}/node
+            install_name_tool -add_rpath "@loader_path" ${BUILD_DIR}/seednode
+
+            otool -L ${BUILD_DIR}/node
+            otool -L ${BUILD_DIR}/seednode
+          fi
+
+      - name: Smoke test
+        run: |
+          # Remove all downloaded Go packages, so that we can test the binary's independence from them (think of Wasmer libraries).
+          sudo rm -rf ${GOPATH}/pkg/mod
+
+          # Test binaries in different current directories.
+          cd ${BUILD_DIR} && ./node --version
+          cd ${GITHUB_WORKSPACE} && ${BUILD_DIR}/node --version
+          cd / && ${BUILD_DIR}/node --version
+
+          cd ${BUILD_DIR} && ./seednode --version
+          cd ${GITHUB_WORKSPACE} && ${BUILD_DIR}/seednode --version
+          cd / && ${BUILD_DIR}/seednode --version
+
+      - name: Package build output
+        run: |
+          sudo chown -R $USER: ${BUILD_DIR}
+          chmod -R 755 ${BUILD_DIR}
+          ls -al ${BUILD_DIR}
+          zip -r -j ${ARCHIVE} ${BUILD_DIR}
 
       - name: Save artifacts
         uses: actions/upload-artifact@v3
         with:
           name: ${{ env.ARCHIVE }}
-          path: ${{ github.workspace }}/${{ env.ARCHIVE }}
+          path: ${{ env.ARCHIVE }}
           if-no-files-found: error
 
   release:
@@ -113,6 +159,6 @@ jobs:
         run: |
           gh release create --draft --notes="Release draft from Github Actions" vNext
           sleep 10
-          for i in $(find ./assets -name '*.tgz' -type f); do
+          for i in $(find ./assets -name '*.zip' -type f); do
             gh release upload vNext ${i}
           done

.github/workflows/docker-keygenerator.yaml

@@ -0,0 +1,36 @@
+name: Build & push keygenerator docker image
+
+on:
+  workflow_dispatch:
+  pull_request:
+
+jobs:
+  build-docker-image:
+    strategy:
+      matrix:
+        runs-on: [ubuntu-latest]
+    runs-on: ${{ matrix.runs-on }}
+
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log into Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push image to Docker Hub
+        id: push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./docker/keygenerator/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: multiversx/chain-keygenerator:latest

.github/workflows/golangci-lint.yml

@@ -5,6 +5,7 @@ on:
       - master
   pull_request:
     branches: [ master, feat/*, rc/* ]
+  workflow_dispatch:
 
 permissions:
   contents: read

api/errors/errors.go

@@ -174,3 +174,6 @@ var ErrGetWaitingManagedKeys = errors.New("error getting the waiting managed key
 
 // ErrGetWaitingEpochsLeftForPublicKey signals that an error occurred while getting the waiting epochs left for public key
 var ErrGetWaitingEpochsLeftForPublicKey = errors.New("error getting the waiting epochs left for public key")
+
+// ErrRecursiveRelayedTxIsNotAllowed signals that recursive relayed tx is not allowed
+var ErrRecursiveRelayedTxIsNotAllowed = errors.New("recursive relayed tx is not allowed")

api/groups/addressGroup.go

@@ -38,6 +38,7 @@ const (
 	urlParamBlockHash              = "blockHash"
 	urlParamBlockRootHash          = "blockRootHash"
 	urlParamHintEpoch              = "hintEpoch"
+	urlParamWithKeys               = "withKeys"
 )
 
 // addressFacadeHandler defines the methods to be implemented by a facade for handling address requests
@@ -71,9 +72,11 @@ type esdtTokenData struct {
 	Properties      string `json:"properties"`
 }
 
-type esdtNFTTokenData struct {
+// ESDTNFTTokenData defines the exposed nft token data structure
+type ESDTNFTTokenData struct {
 	TokenIdentifier string   `json:"tokenIdentifier"`
 	Balance         string   `json:"balance"`
+	Type            string   `json:"type"`
 	Properties      string   `json:"properties,omitempty"`
 	Name            string   `json:"name,omitempty"`
 	Nonce           uint64   `json:"nonce,omitempty"`
@@ -185,6 +188,14 @@ func (ag *addressGroup) getAccount(c *gin.Context) {
 		return
 	}
 
+	withKeys, err := parseBoolUrlParam(c, urlParamWithKeys)
+	if err != nil {
+		shared.RespondWithValidationError(c, errors.ErrCouldNotGetAccount, err)
+		return
+	}
+
+	options.WithKeys = withKeys
+
 	accountResponse, blockInfo, err := ag.getFacade().GetAccount(addr, options)
 	if err != nil {
 		shared.RespondWithInternalError(c, errors.ErrCouldNotGetAccount, err)
@@ -439,7 +450,7 @@ func (ag *addressGroup) getAllESDTData(c *gin.Context) {
 		return
 	}
 
-	formattedTokens := make(map[string]*esdtNFTTokenData)
+	formattedTokens := make(map[string]*ESDTNFTTokenData)
 	for tokenID, esdtData := range tokens {
 		tokenData := buildTokenDataApiResponse(tokenID, esdtData)
 
@@ -472,12 +483,14 @@ func (ag *addressGroup) isDataTrieMigrated(c *gin.Context) {
 	shared.RespondWithSuccess(c, gin.H{"isMigrated": isMigrated})
 }
 
-func buildTokenDataApiResponse(tokenIdentifier string, esdtData *esdt.ESDigitalToken) *esdtNFTTokenData {
-	tokenData := &esdtNFTTokenData{
+func buildTokenDataApiResponse(tokenIdentifier string, esdtData *esdt.ESDigitalToken) *ESDTNFTTokenData {
+	tokenData := &ESDTNFTTokenData{
 		TokenIdentifier: tokenIdentifier,
 		Balance:         esdtData.Value.String(),
 		Properties:      hex.EncodeToString(esdtData.Properties),
 	}
+
+	tokenType := core.ESDTType(esdtData.Type).String()
 	if esdtData.TokenMetaData != nil {
 		tokenData.Name = string(esdtData.TokenMetaData.Name)
 		tokenData.Nonce = esdtData.TokenMetaData.Nonce
@@ -486,11 +499,25 @@ func buildTokenDataApiResponse(tokenIdentifier string, esdtData *esdt.ESDigitalT
 		tokenData.Hash = esdtData.TokenMetaData.Hash
 		tokenData.URIs = esdtData.TokenMetaData.URIs
 		tokenData.Attributes = esdtData.TokenMetaData.Attributes
+
+		tokenType = getTokenType(esdtData.GetType(), tokenData.Nonce)
 	}
 
+	tokenData.Type = tokenType
+
 	return tokenData
 }
 
+func getTokenType(tokenType uint32, tokenNonce uint64) string {
+	isNotFungible := tokenNonce != 0
+	tokenTypeNotSet := isNotFungible && core.ESDTType(tokenType) == core.NonFungible
+	if tokenTypeNotSet {
+		return ""
+	}
+
+	return core.ESDTType(tokenType).String()
+}
+
 func (ag *addressGroup) getFacade() addressFacadeHandler {
 	ag.mutFacade.RLock()
 	defer ag.mutFacade.RUnlock()

api/groups/nodeGroup.go

@@ -28,6 +28,7 @@ const (
 	bootstrapStatusPath       = "/bootstrapstatus"
 	connectedPeersRatingsPath = "/connected-peers-ratings"
 	managedKeys               = "/managed-keys"
+	loadedKeys                = "/loaded-keys"
 	managedKeysCount          = "/managed-keys/count"
 	eligibleManagedKeys       = "/managed-keys/eligible"
 	waitingManagedKeys        = "/managed-keys/waiting"
@@ -44,6 +45,7 @@ type nodeFacadeHandler interface {
 	GetConnectedPeersRatingsOnMainNetwork() (string, error)
 	GetManagedKeysCount() int
 	GetManagedKeys() []string
+	GetLoadedKeys() []string
 	GetEligibleManagedKeys() ([]string, error)
 	GetWaitingManagedKeys() ([]string, error)
 	GetWaitingEpochsLeftForPublicKey(publicKey string) (uint32, error)
@@ -129,6 +131,11 @@ func NewNodeGroup(facade nodeFacadeHandler) (*nodeGroup, error) {
 			Method:  http.MethodGet,
 			Handler: ng.managedKeys,
 		},
+		{
+			Path:    loadedKeys,
+			Method:  http.MethodGet,
+			Handler: ng.loadedKeys,
+		},
 		{
 			Path:    eligibleManagedKeys,
 			Method:  http.MethodGet,
@@ -411,6 +418,19 @@ func (ng *nodeGroup) managedKeys(c *gin.Context) {
 	)
 }
 
+// loadedKeys returns all keys loaded by the current node
+func (ng *nodeGroup) loadedKeys(c *gin.Context) {
+	keys := ng.getFacade().GetLoadedKeys()
+	c.JSON(
+		http.StatusOK,
+		shared.GenericAPIResponse{
+			Data:  gin.H{"loadedKeys": keys},
+			Error: "",
+			Code:  shared.ReturnCodeSuccess,
+		},
+	)
+}
+
 // managedKeysEligible returns the node's eligible managed keys
 func (ng *nodeGroup) managedKeysEligible(c *gin.Context) {
 	keys, err := ng.getFacade().GetEligibleManagedKeys()