Merge branch 'tomas/sorted-prefix-iter' (#458)

* tomas/sorted-prefix-iter:
  [ci skip] wasm checksums update
  changelog: add #458
  tests: extend prefix iter tests for reverse order
  add support for rev_iter_prefix in storage and VP and tx envs
  test/vm_host_env: check prefix iter order in tx and VP
  shared/storage/key: add support for int/uint keys that maintain order
  deps: replace hex with data-encoding

Author: tzemanovic

.changelog/v0.7.1/improvements/409-sorted-prefix-iter.md

@@ -0,0 +1,3 @@
+- Fix order of prefix iterator to be sorted by storage
+  keys and add support for a reverse order prefix iterator.
+  ([#409](https://github.com/anoma/namada/issues/409))

Cargo.lock

@@ -64,6 +64,7 @@ clap = {git = "https://github.com/clap-rs/clap/", tag = "v3.0.0-beta.2", default
 color-eyre = "0.5.10"
 config = "0.11.0"
 curl = "0.4.43"
+data-encoding = "2.3.2"
 derivative = "2.2.0"
 directories = "4.0.1"
 ed25519-consensus = "1.2.0"
@@ -73,7 +74,6 @@ eyre = "0.6.5"
 flate2 = "1.0.22"
 file-lock = "2.0.2"
 futures = "0.3"
-hex = "0.4.3"
 itertools = "0.10.1"
 jsonpath_lib = "0.3.0"
 libc = "0.2.97"

apps/Cargo.toml

@@ -11,6 +11,7 @@ use async_std::fs::{self};
 use async_std::path::PathBuf;
 use async_std::prelude::*;
 use borsh::BorshDeserialize;
+use data_encoding::HEXLOWER;
 use itertools::Itertools;
 use namada::ledger::governance::parameters::GovParams;
 use namada::ledger::governance::storage as gov_storage;
@@ -83,7 +84,7 @@ pub async fn query_raw_bytes(_ctx: Context, args: args::QueryRawBytes) {
         .unwrap();
     match response.code {
         Code::Ok => {
-            println!("{}", hex::encode(&response.value));
+            println!("{}", HEXLOWER.encode(&response.value));
         }
         Code::Err(err) => {
             eprintln!(

apps/src/lib/client/rpc.rs

@@ -26,8 +26,8 @@ pub mod genesis_config {
     use std::path::Path;
     use std::str::FromStr;
 
+    use data_encoding::HEXLOWER;
     use eyre::Context;
-    use hex;
     use namada::ledger::governance::parameters::GovParams;
     use namada::ledger::parameters::{EpochDuration, Parameters};
     use namada::ledger::pos::types::BasisPoints;
@@ -51,12 +51,12 @@ pub mod genesis_config {
 
     impl HexString {
         pub fn to_bytes(&self) -> Result<Vec<u8>, HexKeyError> {
-            let bytes = hex::decode(&self.0)?;
+            let bytes = HEXLOWER.decode(self.0.as_ref())?;
             Ok(bytes)
         }
 
         pub fn to_sha256_bytes(&self) -> Result<[u8; 32], HexKeyError> {
-            let bytes = hex::decode(&self.0)?;
+            let bytes = HEXLOWER.decode(self.0.as_ref())?;
             let slice = bytes.as_slice();
             let array: [u8; 32] = slice.try_into()?;
             Ok(array)
@@ -77,15 +77,15 @@ pub mod genesis_config {
     #[derive(Error, Debug)]
     pub enum HexKeyError {
         #[error("Invalid hex string: {0:?}")]
-        InvalidHexString(hex::FromHexError),
+        InvalidHexString(data_encoding::DecodeError),
         #[error("Invalid sha256 checksum: {0}")]
         InvalidSha256(TryFromSliceError),
         #[error("Invalid public key: {0}")]
         InvalidPublicKey(ParsePublicKeyError),
     }
 
-    impl From<hex::FromHexError> for HexKeyError {
-        fn from(err: hex::FromHexError) -> Self {
+    impl From<data_encoding::DecodeError> for HexKeyError {
+        fn from(err: data_encoding::DecodeError) -> Self {
             Self::InvalidHexString(err)
         }
     }

apps/src/lib/config/genesis.rs

@@ -21,6 +21,7 @@ pub struct Identity {
 // TODO this is needed because libp2p does not export ed255519 serde
 // feature maybe a MR for libp2p to export theses functions ?
 mod keypair_serde {
+    use data_encoding::HEXLOWER;
     use libp2p::identity::ed25519::Keypair;
     use serde::de::Error;
     use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -33,15 +34,16 @@ mod keypair_serde {
         S: Serializer,
     {
         let bytes = value.encode();
-        let string = hex::encode(&bytes[..]);
+        let string = HEXLOWER.encode(&bytes[..]);
         string.serialize(serializer)
     }
     pub fn deserialize<'d, D>(deserializer: D) -> Result<Keypair, D::Error>
     where
         D: Deserializer<'d>,
     {
         let string = String::deserialize(deserializer)?;
-        let mut bytes = hex::decode(&string).map_err(Error::custom)?;
+        let mut bytes =
+            HEXLOWER.decode(string.as_ref()).map_err(Error::custom)?;
         Keypair::decode(bytes.as_mut()).map_err(Error::custom)
     }
 }

apps/src/lib/node/gossip/p2p/identity.rs

@@ -802,24 +802,36 @@ impl<'iter> DBIter<'iter> for RocksDB {
         &'iter self,
         prefix: &Key,
     ) -> PersistentPrefixIterator<'iter> {
-        let db_prefix = "subspace/".to_owned();
-        let prefix = format!("{}{}", db_prefix, prefix);
+        iter_prefix(self, prefix, Direction::Forward)
+    }
 
-        let mut read_opts = ReadOptions::default();
-        // don't use the prefix bloom filter
-        read_opts.set_total_order_seek(true);
-        let mut upper_prefix = prefix.clone().into_bytes();
-        if let Some(last) = upper_prefix.pop() {
-            upper_prefix.push(last + 1);
-        }
-        read_opts.set_iterate_upper_bound(upper_prefix);
+    fn rev_iter_prefix(&'iter self, prefix: &Key) -> Self::PrefixIter {
+        iter_prefix(self, prefix, Direction::Reverse)
+    }
+}
 
-        let iter = self.0.iterator_opt(
-            IteratorMode::From(prefix.as_bytes(), Direction::Forward),
-            read_opts,
-        );
-        PersistentPrefixIterator(PrefixIterator::new(iter, db_prefix))
+fn iter_prefix<'iter>(
+    db: &'iter RocksDB,
+    prefix: &Key,
+    direction: Direction,
+) -> PersistentPrefixIterator<'iter> {
+    let db_prefix = "subspace/".to_owned();
+    let prefix = format!("{}{}", db_prefix, prefix);
+
+    let mut read_opts = ReadOptions::default();
+    // don't use the prefix bloom filter
+    read_opts.set_total_order_seek(true);
+    let mut upper_prefix = prefix.clone().into_bytes();
+    if let Some(last) = upper_prefix.pop() {
+        upper_prefix.push(last + 1);
     }
+    read_opts.set_iterate_upper_bound(upper_prefix);
+
+    let iter = db.0.iterator_opt(
+        IteratorMode::From(prefix.as_bytes(), direction),
+        read_opts,
+    );
+    PersistentPrefixIterator(PrefixIterator::new(iter, db_prefix))
 }
 
 #[derive(Debug)]

apps/src/lib/node/ledger/storage/rocksdb.rs

@@ -5,6 +5,7 @@ use std::rc::Rc;
 use std::str::FromStr;
 
 use borsh::{BorshDeserialize, BorshSerialize};
+use data_encoding::HEXLOWER;
 use namada::types::key::*;
 use orion::{aead, kdf};
 use serde::{Deserialize, Serialize};
@@ -108,15 +109,15 @@ pub struct EncryptedKeypair(Vec<u8>);
 
 impl Display for EncryptedKeypair {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", hex::encode(&self.0))
+        write!(f, "{}", HEXLOWER.encode(self.0.as_ref()))
     }
 }
 
 impl FromStr for EncryptedKeypair {
-    type Err = hex::FromHexError;
+    type Err = data_encoding::DecodeError;
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
-        hex::decode(s).map(Self)
+        HEXLOWER.decode(s.as_ref()).map(Self)
     }
 }
 

apps/src/lib/wallet/keys.rs

@@ -4,9 +4,9 @@ use std::collections::HashMap;
 use std::fs;
 use std::path::Path;
 
+use data_encoding::HEXLOWER;
 use eyre::{eyre, WrapErr};
 use futures::future::join_all;
-use hex;
 use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
 use thiserror::Error;
@@ -144,7 +144,7 @@ pub async fn pre_fetch_wasm(wasm_directory: impl AsRef<Path>) {
                 Ok(bytes) => {
                     let mut hasher = Sha256::new();
                     hasher.update(bytes);
-                    let result = hex::encode(hasher.finalize());
+                    let result = HEXLOWER.encode(&hasher.finalize());
                     let derived_name = format!(
                         "{}.{}.wasm",
                         &name.split('.').collect::<Vec<&str>>()[0],

apps/src/lib/wasm_loader/mod.rs

@@ -60,11 +60,11 @@ borsh = "0.9.0"
 chrono = {version = "0.4.22", default-features = false, features = ["clock", "std"]}
 # Using unreleased commit on top of version 0.5.0 that adds Sync to the CLruCache
 clru = {git = "https://github.com/marmeladema/clru-rs.git", rev = "71ca566"}
+data-encoding = "2.3.2"
 derivative = "2.2.0"
 ed25519-consensus = "1.2.0"
 ferveo = {optional = true, git = "https://github.com/anoma/ferveo"}
 ferveo-common = {git = "https://github.com/anoma/ferveo"}
-hex = "0.4.3"
 tpke = {package = "group-threshold-cryptography", optional = true, git = "https://github.com/anoma/ferveo"}
 # TODO using the same version of tendermint-rs as we do here.
 ibc = {git = "https://github.com/heliaxdev/ibc-rs", rev = "30b3495ac56c6c37c99bc69ef9f2e84c3309c6cc", default-features = false}