Merge branch 'andromeda-to-barnard' into andromeda-to-governance

# Conflicts:
#	common/constants.go

Author: sasurobert

.github/workflows/build_and_run_chain_simulator_and_execute_system_test.yml

@@ -62,20 +62,27 @@ jobs:
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
-            // Get the latest comment
+            // Get all comments
             const comments = await github.rest.issues.listComments({
               owner: context.repo.owner,
               repo: context.repo.repo,
               issue_number: context.issue.number,
             });
 
-            const lastComment = comments.data.pop(); // Get the last comment
+            // Find the last comment that contains 'Run Tests:'
+            let lastTestComment = null;
+            for (let i = comments.data.length - 1; i >= 0; i--) {
+              if (comments.data[i].body.includes('Run Tests:')) {
+                lastTestComment = comments.data[i];
+                break;
+              }
+            }
 
-            if (lastComment && lastComment.body.includes('Run Tests:')) {
-              const body = lastComment.body.trim();
+            if (lastTestComment) {
+              const body = lastTestComment.body.trim();
               core.setOutput('latest_comment', body);
 
-              // Parse the branches from the last comment
+              // Parse the branches from the last test comment
               const simulatorBranchMatch = body.match(/mx-chain-simulator-go:\s*(\S+)/);
               const testingSuiteBranchMatch = body.match(/mx-chain-testing-suite:\s*(\S+)/);
 
@@ -86,8 +93,11 @@ jobs:
               if (testingSuiteBranchMatch) {
                 core.exportVariable('MX_CHAIN_TESTING_SUITE_TARGET_BRANCH', testingSuiteBranchMatch[1]);
               }
+
+              // Log which comment was used for configuration
+              core.info(`Found 'Run Tests:' comment from ${lastTestComment.user.login} at ${lastTestComment.created_at}`);
             } else {
-              core.info('The last comment does not contain "Run Tests:". Skipping branch override.');
+              core.info('No comment containing "Run Tests:" was found. Using default branch settings.');
             }
       
 
@@ -146,6 +156,82 @@ jobs:
           go build
           echo "CHAIN_SIMULATOR_BUILD_PATH=$(pwd)" >> $GITHUB_ENV
 
+      - name: Initialize Chain Simulator
+        run: |
+          cd mx-chain-simulator-go/cmd/chainsimulator
+          
+          # Start ChainSimulator with minimal args to initialize configs
+          INIT_LOG_FILE="/tmp/chainsim_init.log"
+          echo "Starting ChainSimulator initialization process..."
+          ./chainsimulator > $INIT_LOG_FILE 2>&1 &
+          INIT_PROCESS_PID=$!
+          
+          # Verify the process is running
+          if ! ps -p $INIT_PROCESS_PID > /dev/null; then
+            echo "Failed to start ChainSimulator process"
+            cat $INIT_LOG_FILE
+            exit 1
+          fi
+          
+          # Wait for the initialization to complete - look for multiple possible success patterns
+          INIT_COMPLETED=false
+          RETRY_COUNT=0
+          MAX_RETRIES=60  # Increase timeout to 60 seconds
+          
+          echo "Waiting for ChainSimulator initialization..."
+          while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
+            # Check for any of these success patterns
+            if grep -q "starting as observer node" $INIT_LOG_FILE || \
+               grep -q "ChainSimulator started successfully" $INIT_LOG_FILE || \
+               grep -q "initialized the node" $INIT_LOG_FILE || \
+               grep -q "Node is running" $INIT_LOG_FILE; then
+              INIT_COMPLETED=true
+              echo "ChainSimulator initialization completed successfully"
+              break
+            fi
+          
+            # If there's a known fatal error, exit early
+            if grep -q "fatal error" $INIT_LOG_FILE || grep -q "panic:" $INIT_LOG_FILE; then
+              echo "Fatal error detected during initialization:"
+              grep -A 10 -E "fatal error|panic:" $INIT_LOG_FILE
+              break
+            fi
+          
+            # Print progress every 10 seconds
+            if [ $((RETRY_COUNT % 10)) -eq 0 ]; then
+              echo "Still waiting for initialization... ($RETRY_COUNT seconds elapsed)"
+              tail -5 $INIT_LOG_FILE
+            fi
+          
+            RETRY_COUNT=$((RETRY_COUNT+1))
+            sleep 1
+          done
+          
+          # Kill the initialization process - try graceful shutdown first
+          echo "Stopping initialization process (PID: $INIT_PROCESS_PID)..."
+          kill -TERM $INIT_PROCESS_PID 2>/dev/null || true
+          sleep 3
+          
+          # Check if process still exists and force kill if needed
+          if ps -p $INIT_PROCESS_PID > /dev/null 2>&1; then
+            echo "Process still running, forcing kill..."
+            kill -9 $INIT_PROCESS_PID 2>/dev/null || true
+            sleep 1
+          fi
+          
+          if [ "$INIT_COMPLETED" != "true" ]; then
+            echo "ChainSimulator initialization failed after $MAX_RETRIES seconds"
+            echo "Last 20 lines of log:"
+            tail -20 $INIT_LOG_FILE
+            exit 1
+          fi
+          
+          # Create a marker file to indicate successful initialization
+          touch /tmp/chain_simulator_initialized.lock
+          echo "Chain Simulator successfully initialized"
+          
+          echo "Initialization log stored at: $INIT_LOG_FILE"
+
       - name: Checkout mx-chain-testing-suite
         uses: actions/checkout@v4
         with:

api/errors/errors.go

@@ -28,6 +28,9 @@ var ErrGetValueForKey = errors.New("get value for key error")
 // ErrGetKeyValuePairs signals an error in getting the key-value pairs of a key for an account
 var ErrGetKeyValuePairs = errors.New("get key-value pairs error")
 
+// ErrIterateKeys signals an error in iterating over the keys of an account
+var ErrIterateKeys = errors.New("iterate keys error")
+
 // ErrGetESDTBalance signals an error in getting esdt balance for given address
 var ErrGetESDTBalance = errors.New("get esdt balance for account error")
 
@@ -43,6 +46,12 @@ var ErrGetESDTNFTData = errors.New("get esdt nft data for account error")
 // ErrEmptyAddress signals that an empty address was provided
 var ErrEmptyAddress = errors.New("address is empty")
 
+// ErrEmptyNumKeys signals that an empty numKeys was provided
+var ErrEmptyNumKeys = errors.New("numKeys is empty")
+
+// ErrEmptyCheckpointId signals that an empty checkpointId was provided
+var ErrEmptyCheckpointId = errors.New("checkpointId is empty")
+
 // ErrEmptyKey signals that an empty key was provided
 var ErrEmptyKey = errors.New("key is empty")
 

api/groups/addressGroup.go

@@ -32,6 +32,7 @@ const (
 	getRegisteredNFTsPath          = "/:address/registered-nfts"
 	getESDTNFTDataPath             = "/:address/nft/:tokenIdentifier/nonce/:nonce"
 	getGuardianData                = "/:address/guardian-data"
+	iterateKeysPath                = "/iterate-keys"
 	urlParamOnFinalBlock           = "onFinalBlock"
 	urlParamOnStartOfEpoch         = "onStartOfEpoch"
 	urlParamBlockNonce             = "blockNonce"
@@ -55,6 +56,7 @@ type addressFacadeHandler interface {
 	GetESDTsWithRole(address string, role string, options api.AccountQueryOptions) ([]string, api.BlockInfo, error)
 	GetAllESDTTokens(address string, options api.AccountQueryOptions) (map[string]*esdt.ESDigitalToken, api.BlockInfo, error)
 	GetKeyValuePairs(address string, options api.AccountQueryOptions) (map[string]string, api.BlockInfo, error)
+	IterateKeys(address string, numKeys uint, iteratorState [][]byte, options api.AccountQueryOptions) (map[string]string, [][]byte, api.BlockInfo, error)
 	GetGuardianData(address string, options api.AccountQueryOptions) (api.GuardianData, api.BlockInfo, error)
 	IsDataTrieMigrated(address string, options api.AccountQueryOptions) (bool, error)
 	IsInterfaceNil() bool
@@ -134,6 +136,11 @@ func NewAddressGroup(facade addressFacadeHandler) (*addressGroup, error) {
 			Method:  http.MethodGet,
 			Handler: ag.getKeyValuePairs,
 		},
+		{
+			Path:    iterateKeysPath,
+			Method:  http.MethodPost,
+			Handler: ag.iterateKeys,
+		},
 		{
 			Path:    getESDTBalancePath,
 			Method:  http.MethodGet,
@@ -327,7 +334,7 @@ func (ag *addressGroup) getGuardianData(c *gin.Context) {
 	shared.RespondWithSuccess(c, gin.H{"guardianData": guardianData, "blockInfo": blockInfo})
 }
 
-// addressGroup returns all the key-value pairs for the given address
+// getKeyValuePairs returns all the key-value pairs for the given address
 func (ag *addressGroup) getKeyValuePairs(c *gin.Context) {
 	addr, options, err := extractBaseParams(c)
 	if err != nil {
@@ -344,6 +351,47 @@ func (ag *addressGroup) getKeyValuePairs(c *gin.Context) {
 	shared.RespondWithSuccess(c, gin.H{"pairs": value, "blockInfo": blockInfo})
 }
 
+// IterateKeysRequest defines the request structure for iterating keys
+type IterateKeysRequest struct {
+	Address       string   `json:"address"`
+	NumKeys       uint     `json:"numKeys"`
+	IteratorState [][]byte `json:"iteratorState"`
+}
+
+// iterateKeys iterates keys for the given address
+func (ag *addressGroup) iterateKeys(c *gin.Context) {
+	var iterateKeysRequest = &IterateKeysRequest{}
+	err := c.ShouldBindJSON(&iterateKeysRequest)
+	if err != nil {
+		shared.RespondWithValidationError(c, errors.ErrValidation, err)
+		return
+	}
+
+	if len(iterateKeysRequest.Address) == 0 {
+		shared.RespondWithValidationError(c, errors.ErrValidation, errors.ErrEmptyAddress)
+		return
+	}
+
+	options, err := extractAccountQueryOptions(c)
+	if err != nil {
+		shared.RespondWithValidationError(c, errors.ErrIterateKeys, err)
+		return
+	}
+
+	value, newIteratorState, blockInfo, err := ag.getFacade().IterateKeys(
+		iterateKeysRequest.Address,
+		iterateKeysRequest.NumKeys,
+		iterateKeysRequest.IteratorState,
+		options,
+	)
+	if err != nil {
+		shared.RespondWithInternalError(c, errors.ErrIterateKeys, err)
+		return
+	}
+
+	shared.RespondWithSuccess(c, gin.H{"pairs": value, "newIteratorState": newIteratorState, "blockInfo": blockInfo})
+}
+
 // getESDTBalance returns the balance for the given address and esdt token
 func (ag *addressGroup) getESDTBalance(c *gin.Context) {
 	addr, tokenIdentifier, options, err := extractGetESDTBalanceParams(c)

api/groups/addressGroup_test.go

@@ -112,7 +112,7 @@ type esdtTokensCompleteResponseData struct {
 type esdtTokensCompleteResponse struct {
 	Data  esdtTokensCompleteResponseData `json:"data"`
 	Error string                         `json:"error"`
-	Code  string
+	Code  string                         `json:"code"`
 }
 
 type keyValuePairsResponseData struct {
@@ -122,7 +122,17 @@ type keyValuePairsResponseData struct {
 type keyValuePairsResponse struct {
 	Data  keyValuePairsResponseData `json:"data"`
 	Error string                    `json:"error"`
-	Code  string
+	Code  string                    `json:"code"`
+}
+
+type iterateKeysResponseData struct {
+	Pairs            map[string]string `json:"pairs"`
+	NewIteratorState [][]byte          `json:"newIteratorState"`
+}
+type iterateKeysResponse struct {
+	Data  iterateKeysResponseData `json:"data"`
+	Error string                  `json:"error"`
+	Code  string                  `json:"code"`
 }
 
 type esdtRolesResponseData struct {
@@ -132,7 +142,7 @@ type esdtRolesResponseData struct {
 type esdtRolesResponse struct {
 	Data  esdtRolesResponseData `json:"data"`
 	Error string                `json:"error"`
-	Code  string
+	Code  string                `json:"code"`
 }
 
 type usernameResponseData struct {
@@ -662,6 +672,106 @@ func TestAddressGroup_getKeyValuePairs(t *testing.T) {
 	})
 }
 
+func TestAddressGroup_iterateKeys(t *testing.T) {
+	t.Parallel()
+
+	t.Run("invalid body should error",
+		testErrorScenario("/address/iterate-keys", "POST", bytes.NewBuffer([]byte("invalid body")),
+			formatExpectedErr(apiErrors.ErrValidation, errors.New("invalid character 'i' looking for beginning of value"))))
+	t.Run("empty address should error", func(t *testing.T) {
+		t.Parallel()
+
+		body := &groups.IterateKeysRequest{
+			Address: "",
+		}
+		bodyBytes, _ := json.Marshal(body)
+		testAddressGroup(
+			t,
+			&mock.FacadeStub{},
+			"/address/iterate-keys",
+			"POST",
+			bytes.NewBuffer(bodyBytes),
+			http.StatusBadRequest,
+			formatExpectedErr(apiErrors.ErrValidation, apiErrors.ErrEmptyAddress),
+		)
+	})
+	t.Run("invalid query options should error", func(t *testing.T) {
+		t.Parallel()
+
+		body := &groups.IterateKeysRequest{
+			Address: "erd1",
+		}
+		bodyBytes, _ := json.Marshal(body)
+		testAddressGroup(
+			t,
+			&mock.FacadeStub{},
+			"/address/iterate-keys?blockNonce=not-uint64",
+			"POST",
+			bytes.NewBuffer(bodyBytes),
+			http.StatusBadRequest,
+			formatExpectedErr(apiErrors.ErrIterateKeys, apiErrors.ErrBadUrlParams),
+		)
+	})
+	t.Run("with node fail should err", func(t *testing.T) {
+		t.Parallel()
+
+		body := &groups.IterateKeysRequest{
+			Address: "erd1",
+		}
+		bodyBytes, _ := json.Marshal(body)
+		facade := &mock.FacadeStub{
+			IterateKeysCalled: func(address string, numKeys uint, iteratorState [][]byte, options api.AccountQueryOptions) (map[string]string, [][]byte, api.BlockInfo, error) {
+				return nil, nil, api.BlockInfo{}, expectedErr
+			},
+		}
+		testAddressGroup(
+			t,
+			facade,
+			"/address/iterate-keys",
+			"POST",
+			bytes.NewBuffer(bodyBytes),
+			http.StatusInternalServerError,
+			formatExpectedErr(apiErrors.ErrIterateKeys, expectedErr),
+		)
+	})
+	t.Run("should work", func(t *testing.T) {
+		t.Parallel()
+
+		pairs := map[string]string{
+			"k1": "v1",
+			"k2": "v2",
+		}
+
+		body := &groups.IterateKeysRequest{
+			Address:       "erd1",
+			NumKeys:       10,
+			IteratorState: [][]byte{[]byte("starting"), []byte("state")},
+		}
+		newIteratorState := [][]byte{[]byte("new"), []byte("state")}
+		bodyBytes, _ := json.Marshal(body)
+		facade := &mock.FacadeStub{
+			IterateKeysCalled: func(address string, numKeys uint, iteratorState [][]byte, options api.AccountQueryOptions) (map[string]string, [][]byte, api.BlockInfo, error) {
+				assert.Equal(t, body.Address, address)
+				assert.Equal(t, body.NumKeys, numKeys)
+				assert.Equal(t, body.IteratorState, iteratorState)
+				return pairs, newIteratorState, api.BlockInfo{}, nil
+			},
+		}
+
+		response := &iterateKeysResponse{}
+		loadAddressGroupResponse(
+			t,
+			facade,
+			"/address/iterate-keys",
+			"POST",
+			bytes.NewBuffer(bodyBytes),
+			response,
+		)
+		assert.Equal(t, pairs, response.Data.Pairs)
+		assert.Equal(t, newIteratorState, response.Data.NewIteratorState)
+	})
+}
+
 func TestAddressGroup_getESDTBalance(t *testing.T) {
 	t.Parallel()
 
@@ -1143,6 +1253,7 @@ func getAddressRoutesConfig() config.ApiRoutesConfig {
 					{Name: "/:address/username", Open: true},
 					{Name: "/:address/code-hash", Open: true},
 					{Name: "/:address/keys", Open: true},
+					{Name: "/iterate-keys", Open: true},
 					{Name: "/:address/key/:key", Open: true},
 					{Name: "/:address/esdt", Open: true},
 					{Name: "/:address/esdts/roles", Open: true},