diff --git a/eqc/h3dex_eqc.erl b/eqc/h3dex_eqc.erl new file mode 100644 index 0000000000..707c0c0300 --- /dev/null +++ b/eqc/h3dex_eqc.erl @@ -0,0 +1,54 @@ +-module(h3dex_eqc). + +-include_lib("eqc/include/eqc.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-define(INDICES, lists:flatten([ h3:children(I, 5) || I <- h3:get_res0_indexes()])). + +-export([prop_h3dex_check/0]). + +prop_h3dex_check() -> + ?FORALL({Hex1, Hex2, Resolution}, ?SUCHTHAT({X1, X2, _}, {gen_h3(), gen_h3(), choose(6, 6)}, X1 /= 0 andalso X2 /= 0 andalso X1 /= X2), + begin + Children1 = h3:children(Hex1, Resolution) -- [0], + Children2 = h3:children(Hex2, Resolution) -- [0], + + End = h3_to_key(Hex1), + Start = find_lower_bound_hex(Hex1), + ?WHENFAIL(begin + io:format("Hex 1: ~p (~w -> ~w), Hex 2: ~p (~w -> ~w), Resolution ~p~n", [Hex1, find_lower_bound_hex(Hex1), h3_to_key(Hex1), Hex2, find_lower_bound_hex(Hex2), h3_to_key(Hex2), Resolution]) + end, + noshrink(conjunction( + [{all_children_in_range, eqc:equals([], lists:filter(fun(X) -> X < Start orelse X > End end, lists:sort(lists:map(fun h3_to_key/1, Children1)))) }, + {all_non_children_out_of_range, eqc:equals([], lists:filter(fun(E) -> X = h3_to_key(E), X > Start andalso X =< End end, Children2)) }, + {unparse_works, eqc:equals(Hex1, key_to_h3(h3_to_key(Hex1)))} + ] + ) + ) + ) + end). + +gen_h3() -> + elements(?INDICES). + +h3_to_key(H3) -> + %% both reserved fields must be 0 and Mode must be 1 for this to be a h3 cell + <<0:1/integer-unsigned-big, 1:4/integer-unsigned-big, 0:3/integer-unsigned-big, Resolution:4/integer-unsigned-big, BaseCell:7/integer-unsigned-big, Digits:45/integer-unsigned-big>> = <>, + %% store the resolution inverted (15 - Resolution) so it sorts later + <>. + +key_to_h3(Key) -> + <> = Key, + <> = <<0:1, 1:4/integer-unsigned-big, 0:3, (15 - InverseResolution):4/integer-unsigned-big, BaseCell:7/integer-unsigned-big, Digits:45/integer-unsigned-big>>, + H3. + +find_lower_bound_hex(Hex) -> + %% both reserved fields must be 0 and Mode must be 1 for this to be a h3 cell + <<0:1, 1:4/integer-unsigned-big, 0:3, Resolution:4/integer-unsigned-big, BaseCell:7/integer-unsigned-big, Digits/bitstring>> = <>, + ActualDigitCount = Resolution * 3, + %% pull out the actual digits used and dump the rest + <> = Digits, + Padding = 45 - ActualDigitCount, + %% store the resolution inverted (15 - 15) = 0 so it sorts earlier + %% pad the actual digits used with 0s on the end + <>. diff --git a/include/blockchain.hrl b/include/blockchain.hrl index 90f9aecc1b..3a17d7962f 100644 --- a/include/blockchain.hrl +++ b/include/blockchain.hrl @@ -25,4 +25,4 @@ % Misc -define(EVT_MGR, blockchain_event_mgr). --define(BC_UPGRADE_NAMES, [<<"gateway_v2">>, <<"hex_targets">>, <<"gateway_oui">>]). +-define(BC_UPGRADE_NAMES, [<<"gateway_v2">>, <<"hex_targets">>, <<"gateway_oui">>, <<"h3dex">>]). diff --git a/include/blockchain_vars.hrl b/include/blockchain_vars.hrl index b1eef14369..75714f7df4 100644 --- a/include/blockchain_vars.hrl +++ b/include/blockchain_vars.hrl @@ -370,3 +370,32 @@ %% POC challenge for X blocks would be considered stale for the purposes %% of a hotspot transfer. (We do not allow stale hotspots to be transferred.) -define(transfer_hotspot_stale_poc_blocks, transfer_hotspot_stale_poc_blocks). + +%% ------------------------------------------------------------------ +%% HIP 17 vars +%% +%% For every possible h3 resolution, we will define: +%% - number of siblings +%% - density_tgt +%% - density_max +%% +%% So hip17_res_0 value could be: 2, 10000, 10000 for example; +%% where num_siblings=2, density_tgt=10000, density_max=10000 +%% +%% We'd specify any of the below variables like so: <<"2,10000,10000">> +%% We expect the value of any of these variables to be in format: <<"int,int,int">> +-define(hip17_res_0, hip17_res_0). +-define(hip17_res_1, hip17_res_1). +-define(hip17_res_2, hip17_res_2). +-define(hip17_res_3, hip17_res_3). +-define(hip17_res_4, hip17_res_4). +-define(hip17_res_5, hip17_res_5). +-define(hip17_res_6, hip17_res_6). +-define(hip17_res_7, hip17_res_7). +-define(hip17_res_8, hip17_res_8). +-define(hip17_res_9, hip17_res_9). +-define(hip17_res_10, hip17_res_10). +-define(hip17_res_11, hip17_res_11). +-define(hip17_res_12, hip17_res_12). +-define(density_tgt_res, density_tgt_res). +-define(hip17_interactivity_blocks, hip17_interactivity_blocks). diff --git a/rebar.lock b/rebar.lock index 831e509eaa..931d0af8cb 100644 --- a/rebar.lock +++ b/rebar.lock @@ -37,7 +37,7 @@ {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},1}, {<<"h3">>, {git,"https://github.com/helium/erlang-h3.git", - {ref,"a92737698d45c97b7b9b6694513b48c29522a42e"}}, + {ref,"8541da45596549e36bdbf82dcb77f19c8608e9d4"}}, 0}, {<<"helium_proto">>, {git,"https://github.com/helium/proto.git", diff --git a/src/blockchain.erl b/src/blockchain.erl index 1a1aa7a904..2afabee034 100644 --- a/src/blockchain.erl +++ b/src/blockchain.erl @@ -63,7 +63,7 @@ -include("blockchain_vars.hrl"). -ifdef(TEST). --export([bootstrap_hexes/1, can_add_block/2, get_plausible_blocks/1]). +-export([bootstrap_hexes/1, can_add_block/2, get_plausible_blocks/1, bootstrap_h3dex/1]). %% export a macro so we can interpose block saving to test failure -define(save_block(Block, Chain), ?MODULE:save_block(Block, Chain)). -include_lib("eunit/include/eunit.hrl"). @@ -94,7 +94,8 @@ -define(BC_UPGRADE_FUNS, [fun upgrade_gateways_v2/1, fun bootstrap_hexes/1, - fun upgrade_gateways_oui/1]). + fun upgrade_gateways_oui/1, + fun bootstrap_h3dex/1]). -type blocks() :: #{blockchain_block:hash() => blockchain_block:block()}. -type blockchain() :: #blockchain{}. @@ -262,6 +263,18 @@ upgrade_gateways_oui_(Ledger) -> end, Gateways), ok. +-spec bootstrap_h3dex(blockchain_ledger_v1:ledger()) -> ok. +%% @doc Bootstrap the H3Dex for both the active and delayed ledgers +bootstrap_h3dex(Ledger) -> + ok = do_bootstrap_h3dex(Ledger), + Ledger1 = blockchain_ledger_v1:mode(delayed, Ledger), + Ledger2 = blockchain_ledger_v1:new_context(Ledger1), + ok = do_bootstrap_h3dex(Ledger2), + blockchain_ledger_v1:commit_context(Ledger2). + +do_bootstrap_h3dex(Ledger) -> + blockchain_ledger_v1:bootstrap_h3dex(Ledger). + %%-------------------------------------------------------------------- %% @doc %% @end diff --git a/src/blockchain_hex.erl b/src/blockchain_hex.erl new file mode 100644 index 0000000000..2fed6709af --- /dev/null +++ b/src/blockchain_hex.erl @@ -0,0 +1,372 @@ +-module(blockchain_hex). + +-export([var_map/1, + scale/2, scale/4, + destroy_memoization/0]). + +-ifdef(TEST). +-export([densities/3]). +-endif. + +-include("blockchain_vars.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(SCALE_MEMO_TBL, '__blockchain_hex_scale_memoization_tbl'). +-define(DENSITY_MEMO_TBL, '__blockchain_hex_density_memoization_tbl'). +-define(ETS_OPTS, [named_table, public]). + +-type density_map() :: #{h3:h3_index() => pos_integer()}. +-type densities() :: {UnclippedDensities :: density_map(), ClippedDensities :: density_map()}. +-type var_map() :: #{0..12 => map()}. +-type locations() :: #{h3:h3_index() => [libp2p_crypto:pubkey_bin(), ...]}. +-type h3_indices() :: [h3:h3_index()]. +-type tblnames() :: ?SCALE_MEMO_TBL|?DENSITY_MEMO_TBL. + +-export_type([var_map/0]). + +%%-------------------------------------------------------------------- +%% Public functions +%%-------------------------------------------------------------------- +-spec destroy_memoization() -> true. +%% @doc This call will destroy the memoization context used during a rewards +%% calculation. +destroy_memoization() -> + try ets:delete(?SCALE_MEMO_TBL) catch _:_ -> true end, + try ets:delete(?DENSITY_MEMO_TBL) catch _:_ -> true end. + +%% @doc This call is for blockchain_etl to use directly +-spec scale(Location :: h3:h3_index(), + Ledger :: blockchain_ledger_v1:ledger()) -> {error, any()} | {ok, float()}. +scale(Location, Ledger) -> + case var_map(Ledger) of + {error, _}=E -> E; + {ok, VarMap} -> + case get_target_res(Ledger) of + {error, _}=E -> E; + {ok, TargetRes} -> + S = scale(Location, VarMap, TargetRes, Ledger), + {ok, S} + end + end. + +-spec scale( + Location :: h3:h3_index(), + VarMap :: var_map(), + TargetRes :: 0..12, + Ledger :: blockchain_ledger_v1:ledger() +) -> float(). +%% @doc Given a hex location, return the rewards scaling factor. This call is +%% memoized. +scale(Location, VarMap, TargetRes, Ledger) -> + case lookup(Location, ?SCALE_MEMO_TBL) of + {ok, Scale} -> Scale; + not_found -> + memoize(?SCALE_MEMO_TBL, Location, + calculate_scale(Location, VarMap, TargetRes, Ledger)) + end. + +-spec var_map(Ledger :: blockchain_ledger_v1:ledger()) -> {error, any()} | {ok, var_map()}. +%% @doc This function returns a map of hex resolutions mapped to hotspot density targets and +%% maximums. These numbers are used during PoC witness and challenge rewards calculations. +var_map(Ledger) -> + ResolutionVars = [ + ?hip17_res_0, + ?hip17_res_1, + ?hip17_res_2, + ?hip17_res_3, + ?hip17_res_4, + ?hip17_res_5, + ?hip17_res_6, + ?hip17_res_7, + ?hip17_res_8, + ?hip17_res_9, + ?hip17_res_10, + ?hip17_res_11, + ?hip17_res_12 + ], + + {_I, Errors, M} = lists:foldl( + fun(A, {I, Errors, Acc}) -> + case get_density_var(A, Ledger) of + {error, _} = E -> + {I + 1, [{A, E} | Errors], Acc}; + {ok, [N, Tgt, Max]} -> + {I + 1, Errors, + maps:put( + I, + #{ + n => N, + tgt => Tgt, + max => Max + }, + Acc + )} + end + end, + {0, [], #{}}, + ResolutionVars + ), + + case Errors of + [] -> {ok, M}; + Errors -> {error, Errors} + end. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +-spec lookup(Key :: term(), + TblName :: tblnames()) -> + {ok, Result :: term()} | not_found. +lookup(Key, TblName) -> + try + case ets:lookup(TblName, Key) of + [{_Key, Res}] -> {ok, Res}; + [] -> not_found + end + catch + %% if the table doesn't exist yet, create it and return `not_found' + error:badarg -> + _ = maybe_start(TblName), + not_found + end. + +-spec maybe_start(TblName :: tblnames()) -> tblnames(). +maybe_start(TblName) -> + try + _TblName = ets:new(TblName, ?ETS_OPTS) + catch + error:badarg -> TblName + end. + +-spec memoize(TblName :: tblnames(), + Key :: term(), + Result :: term()) -> Result :: term(). +memoize(TblName, Key, Result) -> + true = ets:insert(TblName, {Key, Result}), + Result. + +-spec calculate_scale( + Location :: h3:h3_index(), + VarMap :: var_map(), + TargetRes :: 0..12, + Ledger :: blockchain_ledger_v1:ledger() ) -> float(). +calculate_scale(Location, VarMap, TargetRes, Ledger) -> + %% hip0017 states to go from R -> 0 and take a product of the clipped(parent)/unclipped(parent) + %% however, we specify the lower bound instead of going all the way down to 0 + + R = h3:get_resolution(Location), + + %% Calculate densities at the outermost hex + OuterMostParent = h3:parent(Location, TargetRes), + {UnclippedDensities, ClippedDensities} = densities(OuterMostParent, VarMap, Ledger), + + lists:foldl(fun(Res, Acc) -> + Parent = h3:parent(Location, Res), + Acc * (maps:get(Parent, ClippedDensities) / maps:get(Parent, UnclippedDensities)) + end, 1.0, lists:seq(R, TargetRes, -1)). + + +-spec densities( + H3Index :: h3:h3_index(), + VarMap :: var_map(), + Ledger :: blockchain_ledger_v1:ledger() +) -> densities(). +densities(H3Index, VarMap, Ledger) -> + case lookup(H3Index, ?DENSITY_MEMO_TBL) of + {ok, Densities} -> Densities; + not_found -> memoize(?DENSITY_MEMO_TBL, H3Index, + calculate_densities(H3Index, VarMap, Ledger)) + end. + +-spec calculate_densities( + H3Index :: h3:h3_index(), + VarMap :: var_map(), + Ledger :: blockchain_ledger_v1:ledger() +) -> densities(). +calculate_densities(H3Index, VarMap, Ledger) -> + InteractiveBlocks = case blockchain_ledger_v1:config(?hip17_interactivity_blocks, Ledger) of + {ok, V} -> V; + {error, not_found} -> 0 % XXX what should this value be? + end, + Locations = blockchain_ledger_v1:lookup_gateways_from_hex(h3:k_ring(H3Index, 2), Ledger), + + Interactive = case application:get_env(blockchain, hip17_test_mode, false) of + true -> + %% HIP17 test mode, no interactive filtering + Locations; + false -> + maps:map( + fun(_K, V) -> + filter_interactive_gws(V, InteractiveBlocks, Ledger) + end, Locations) + end, + + %% Calculate clipped and unclipped densities + densities(H3Index, VarMap, Interactive, Ledger). + +-spec densities( + H3Root :: h3:h3_index(), + VarMap :: var_map(), + Locations :: locations(), + Ledger :: blockchain_ledger_v1:ledger() +) -> densities(). +densities(H3Root, VarMap, Locations, Ledger) -> + case maps:size(Locations) of + 0 -> + {#{}, #{}}; + _ -> + UpperBoundRes = lists:max([h3:get_resolution(H3) || H3 <- maps:keys(Locations)]), + LowerBoundRes = h3:get_resolution(H3Root), + + [Head | Tail] = lists:seq(UpperBoundRes, LowerBoundRes, -1), + + %% find parent hexes to all hotspots at highest resolution in chain variables + {ParentHexes, InitialDensities} = + maps:fold( + fun(Hex, GWs, {HAcc, MAcc}) -> + ParentHex = h3:parent(Hex, Head), + case maps:find(ParentHex, MAcc) of + error -> + {[ParentHex | HAcc], maps:put(ParentHex, length(GWs), MAcc)}; + {ok, OldCount} -> + {HAcc, maps:put(ParentHex, OldCount + length(GWs), MAcc)} + end + end, + {[], #{}}, + Locations + ), + + build_densities( + H3Root, + Ledger, + VarMap, + ParentHexes, + {InitialDensities, InitialDensities}, + Tail + ) + end. + +-spec build_densities( + h3:h3_index(), + blockchain_ledger_v1:ledger(), + var_map(), + h3_indices(), + densities(), + [0..15] +) -> densities(). +build_densities(_H3Root, _Ledger, _VarMap, _ParentHexes, {UAcc, Acc}, []) -> + {UAcc, Acc}; +build_densities(H3Root, Ledger, VarMap, ChildHexes, {UAcc, Acc}, [Res | Tail]) -> + UD = unclipped_densities(ChildHexes, Res, Acc), + UM0 = maps:merge(UAcc, UD), + M0 = maps:merge(Acc, UD), + + OccupiedHexesThisRes = maps:keys(UD), + + DensityTarget = maps:get(tgt, maps:get(Res, VarMap)), + + M1 = lists:foldl( + fun(ThisResHex, Acc3) -> + OccupiedCount = occupied_count(DensityTarget, ThisResHex, UD), + Limit = limit(Res, VarMap, OccupiedCount), + maps:put(ThisResHex, min(Limit, maps:get(ThisResHex, M0)), Acc3) + end, + M0, + OccupiedHexesThisRes + ), + + build_densities(H3Root, Ledger, VarMap, OccupiedHexesThisRes, {UM0, M1}, Tail). + +-spec filter_interactive_gws( GWs :: [libp2p_crypto:pubkey_bin(), ...], + InteractiveBlocks :: pos_integer(), + Ledger :: blockchain_ledger_v1:ledger()) -> + [libp2p_crypto:pubkey_bin(), ...]. +%% @doc This function filters a list of gateway addresses which are considered +%% "interactive" for the purposes of HIP17 based on the last block when it +%% responded to a POC challenge compared to the current chain height. +filter_interactive_gws(GWs, InteractiveBlocks, Ledger) -> + {ok, CurrentHeight} = blockchain_ledger_v1:current_height(Ledger), + lists:filter(fun(GWAddr) -> + case blockchain_ledger_v1:find_gateway_info(GWAddr, Ledger) of + {ok, GWInfo} -> + case blockchain_ledger_gateway_v2:last_poc_challenge(GWInfo) of + undefined -> false; + LastChallenge -> + (CurrentHeight - LastChallenge) =< InteractiveBlocks + end; + {error, not_found} -> false + end + end, GWs). + +-spec limit( + Res :: 0..12, + VarMap :: var_map(), + OccupiedCount :: non_neg_integer() +) -> non_neg_integer(). +limit(Res, VarMap, OccupiedCount) -> + VarAtRes = maps:get(Res, VarMap), + DensityMax = maps:get(max, VarAtRes), + DensityTgt = maps:get(tgt, VarAtRes), + N = maps:get(n, VarAtRes), + Max = max((OccupiedCount - N), 1), + min(DensityMax, DensityTgt * Max). + +-spec occupied_count( + DensityTarget :: 0..12, + ThisResHex :: h3:h3_index(), + DensityMap :: density_map() +) -> non_neg_integer(). +occupied_count(DensityTarget, ThisResHex, DensityMap) -> + H3Neighbors = h3:k_ring(ThisResHex, 1), + + lists:foldl( + fun(Neighbor, Acc) -> + case maps:get(Neighbor, DensityMap, 0) >= DensityTarget of + false -> Acc; + true -> Acc + 1 + end + end, + 0, + H3Neighbors + ). + +-spec unclipped_densities(h3_indices(), 0..12, density_map()) -> density_map(). +unclipped_densities(ChildToParents, Res, Acc) -> + lists:foldl( + fun(ChildHex, Acc2) -> + ThisParentHex = h3:parent(ChildHex, Res), + maps:update_with( + ThisParentHex, + fun(V) -> V + maps:get(ChildHex, Acc, 0) end, + maps:get(ChildHex, Acc, 0), + Acc2 + ) + end, + #{}, + ChildToParents + ). + +-spec get_density_var( + Var :: atom(), + Ledger :: blockchain_ledger_v1:ledger() +) -> {error, any()} | {ok, [pos_integer()]}. +get_density_var(Var, Ledger) -> + case blockchain:config(Var, Ledger) of + {error, _} = E -> + E; + {ok, Bin} -> + [N, Tgt, Max] = [ + list_to_integer(I) + || I <- string:tokens(binary:bin_to_list(Bin), ",") + ], + {ok, [N, Tgt, Max]} + end. + +-spec get_target_res(Ledger :: blockchain_ledger_v1:ledger()) -> {error, any()} | {ok, non_neg_integer()}. +get_target_res(Ledger) -> + case blockchain:config(?density_tgt_res, Ledger) of + {error, _}=E -> E; + {ok, V} -> {ok, V} + end. + diff --git a/src/ledger/v1/blockchain_ledger_snapshot_v1.erl b/src/ledger/v1/blockchain_ledger_snapshot_v1.erl index 52104a7219..bd175aaf9b 100644 --- a/src/ledger/v1/blockchain_ledger_snapshot_v1.erl +++ b/src/ledger/v1/blockchain_ledger_snapshot_v1.erl @@ -31,7 +31,8 @@ snapshot(Ledger0, Blocks) -> Parent = self(), Ref = make_ref(), - {_Pid, MonitorRef} = spawn_opt(fun ThisFun() -> + {_Pid, MonitorRef} = + spawn_opt(fun ThisFun() -> Ledger = blockchain_ledger_v1:mode(delayed, Ledger0), {ok, CurrHeight} = blockchain_ledger_v1:current_height(Ledger), %% this should not leak a huge amount of atoms @@ -109,6 +110,7 @@ generate_snapshot(Ledger0, Blocks) -> {ok, OUICounter} = blockchain_ledger_v1:get_oui_counter(Ledger), Hexes = blockchain_ledger_v1:snapshot_hexes(Ledger), + H3dex = blockchain_ledger_v1:snapshot_h3dex(Ledger), StateChannels = blockchain_ledger_v1:snapshot_state_channels(Ledger), @@ -116,45 +118,47 @@ generate_snapshot(Ledger0, Blocks) -> {ok, OraclePriceList} = blockchain_ledger_v1:current_oracle_price_list(Ledger), Snapshot = - #blockchain_snapshot_v4{ - current_height = CurrHeight, - transaction_fee = 0, - consensus_members = ConsensusMembers, + #{ + version => v5, + current_height => CurrHeight, + transaction_fee => 0, + consensus_members => ConsensusMembers, - election_height = ElectionHeight, - election_epoch = ElectionEpoch, + election_height => ElectionHeight, + election_epoch => ElectionEpoch, - delayed_vars = DelayedVars, - threshold_txns = ThresholdTxns, + delayed_vars => DelayedVars, + threshold_txns => ThresholdTxns, - master_key = MasterKey, - multi_keys = MultiKeys, - vars_nonce = VarsNonce, - vars = Vars, + master_key => MasterKey, + multi_keys => MultiKeys, + vars_nonce => VarsNonce, + vars => Vars, - gateways = Gateways, - pocs = PoCs, + gateways => Gateways, + pocs => PoCs, - accounts = Accounts, - dc_accounts = DCAccounts, + accounts => Accounts, + dc_accounts => DCAccounts, - security_accounts = SecurityAccounts, + security_accounts => SecurityAccounts, - htlcs = HTLCs, + htlcs => HTLCs, - ouis = OUIs, - subnets = Subnets, - oui_counter = OUICounter, + ouis => OUIs, + subnets => Subnets, + oui_counter => OUICounter, - hexes = Hexes, + hexes => Hexes, + h3dex => H3dex, - state_channels = StateChannels, + state_channels => StateChannels, - blocks = Blocks, + blocks => Blocks, - oracle_price = OraclePrice, - oracle_price_list = OraclePriceList - }, + oracle_price => OraclePrice, + oracle_price_list => OraclePriceList + }, {ok, Snapshot} catch C:E:S -> @@ -170,9 +174,9 @@ serialize(Snapshot, BlocksP) -> Blocks = lists:map(fun(B) when is_tuple(B) -> blockchain_block:serialize(B); (B) -> B - end, Snapshot#blockchain_snapshot_v4.blocks), - Snapshot#blockchain_snapshot_v4{blocks = Blocks}; - noblocks -> Snapshot#blockchain_snapshot_v4{blocks = []} + end, maps:get(blocks, Snapshot, [])), + lists:sort(maps:to_list(Snapshot#{blocks => Blocks})); + noblocks -> Snapshot#{blocks => []} end, Bin = term_to_binary(Snapshot1, [{compressed, 9}]), BinSz = byte_size(Bin), @@ -215,12 +219,24 @@ serialize_v3(Snapshot, noblocks) -> BinSz:32/little-unsigned-integer, Bin/binary>>, {ok, Snap}. +serialize_v4(Snapshot, noblocks) -> + %% NOTE: serialize_v4 only gets called with noblocks + Snapshot1 = Snapshot#blockchain_snapshot_v4{blocks = []}, + Bin = term_to_binary(Snapshot1, [{compressed, 9}]), + BinSz = byte_size(Bin), + + %% do some simple framing with version, size, & snap + Snap = <<4, %% version + BinSz:32/little-unsigned-integer, Bin/binary>>, + {ok, Snap}. + + deserialize(<<1, %%SHASz:16/little-unsigned-integer, SHA:SHASz/binary, BinSz:32/little-unsigned-integer, BinSnap:BinSz/binary>>) -> try binary_to_term(BinSnap) of OldSnapshot -> - Snapshot = v3_to_v4(v2_to_v3(v1_to_v2(OldSnapshot))), + Snapshot = v4_to_v5(v3_to_v4(v2_to_v3(v1_to_v2(OldSnapshot)))), {ok, Snapshot} catch _:_ -> {error, bad_snapshot_binary} @@ -230,7 +246,7 @@ deserialize(<<2, BinSz:32/little-unsigned-integer, BinSnap:BinSz/binary>>) -> try binary_to_term(BinSnap) of OldSnapshot -> - Snapshot = v3_to_v4(v2_to_v3(OldSnapshot)), + Snapshot = v4_to_v5(v3_to_v4(v2_to_v3(OldSnapshot))), {ok, Snapshot} catch _:_ -> {error, bad_snapshot_binary} @@ -240,7 +256,7 @@ deserialize(<<3, BinSz:32/little-unsigned-integer, BinSnap:BinSz/binary>>) -> try binary_to_term(BinSnap) of OldSnapshot -> - Snapshot = v3_to_v4(OldSnapshot), + Snapshot = v4_to_v5(v3_to_v4(OldSnapshot)), {ok, Snapshot} catch _:_ -> {error, bad_snapshot_binary} @@ -249,7 +265,17 @@ deserialize(<<4, %%SHASz:16/little-unsigned-integer, SHA:SHASz/binary, BinSz:32/little-unsigned-integer, BinSnap:BinSz/binary>>) -> try binary_to_term(BinSnap) of - #blockchain_snapshot_v4{} = Snapshot -> + OldSnapshot -> + Snapshot = v4_to_v5(OldSnapshot), + {ok, Snapshot} + catch _:_ -> + {error, bad_snapshot_binary} + end; +deserialize(<<5, + %%SHASz:16/little-unsigned-integer, SHA:SHASz/binary, + BinSz:32/little-unsigned-integer, BinSnap:BinSz/binary>>) -> + try maps:from_list(binary_to_term(BinSnap)) of + #{version := v5} = Snapshot -> {ok, Snapshot} catch _:_ -> {error, bad_snapshot_binary} @@ -257,50 +283,51 @@ deserialize(<<4, %% sha will be stored externally import(Chain, SHA, - #blockchain_snapshot_v4{ - current_height = CurrHeight, - transaction_fee = _TxnFee, - consensus_members = ConsensusMembers, + #{ + current_height := CurrHeight, + transaction_fee := _TxnFee, + consensus_members := ConsensusMembers, - election_height = ElectionHeight, - election_epoch = ElectionEpoch, + election_height := ElectionHeight, + election_epoch := ElectionEpoch, - delayed_vars = DelayedVars, - threshold_txns = ThresholdTxns, + delayed_vars := DelayedVars, + threshold_txns := ThresholdTxns, - master_key = MasterKey, - multi_keys = MultiKeys, - vars_nonce = VarsNonce, - vars = Vars, + master_key := MasterKey, + multi_keys := MultiKeys, + vars_nonce := VarsNonce, + vars := Vars, - gateways = Gateways, - pocs = PoCs, + gateways := Gateways, + pocs := PoCs, - accounts = Accounts, - dc_accounts = DCAccounts, + accounts := Accounts, + dc_accounts := DCAccounts, - security_accounts = SecurityAccounts, + security_accounts := SecurityAccounts, - htlcs = HTLCs, + htlcs := HTLCs, - ouis = OUIs, - subnets = Subnets, - oui_counter = OUICounter, + ouis := OUIs, + subnets := Subnets, + oui_counter := OUICounter, - hexes = Hexes, + hexes := Hexes, - state_channels = StateChannels, + state_channels := StateChannels, - blocks = Blocks, + blocks := Blocks, - oracle_price = OraclePrice, - oracle_price_list = OraclePriceList + oracle_price := OraclePrice, + oracle_price_list := OraclePriceList } = Snapshot) -> Dir = blockchain:dir(Chain), case hash(Snapshot) == SHA orelse - hash_v3(v4_to_v3(Snapshot)) == SHA orelse - hash_v2(v3_to_v2(v4_to_v3(Snapshot))) == SHA orelse - hash_v1(v2_to_v1(v3_to_v2(v4_to_v3(Snapshot)))) == SHA of + hash_v4(v5_to_v4(Snapshot)) == SHA orelse + hash_v3(v4_to_v3(v5_to_v4(Snapshot))) == SHA orelse + hash_v2(v3_to_v2(v4_to_v3(v5_to_v4(Snapshot)))) == SHA orelse + hash_v1(v2_to_v1(v3_to_v2(v4_to_v3(v5_to_v4(Snapshot))))) == SHA of true -> CLedger = blockchain:ledger(Chain), Ledger0 = @@ -439,10 +466,10 @@ get_blocks(Chain) -> end || N <- lists:seq(max(?min_height, LoadBlockStart), Height)]. -height(#blockchain_snapshot_v4{current_height = Height}) -> +height(#{current_height := Height}) -> Height. -hash(#blockchain_snapshot_v4{} = Snap) -> +hash(#{version := v5} = Snap) -> {ok, BinSnap} = serialize(Snap, noblocks), crypto:hash(sha256, BinSnap). @@ -458,6 +485,10 @@ hash_v3(#blockchain_snapshot_v3{} = Snap) -> {ok, BinSnap} = serialize_v3(Snap, noblocks), crypto:hash(sha256, BinSnap). +hash_v4(#blockchain_snapshot_v4{} = Snap) -> + {ok, BinSnap} = serialize_v4(Snap, noblocks), + crypto:hash(sha256, BinSnap). + v1_to_v2(#blockchain_snapshot_v1{ previous_snapshot_hash = <<>>, leading_hash = <<>>, @@ -784,6 +815,87 @@ v3_to_v4(#blockchain_snapshot_v3{ oracle_price_list = OraclePriceList }. +v4_to_v5(#blockchain_snapshot_v4{ + current_height = CurrHeight, + transaction_fee = _TxnFee, + consensus_members = ConsensusMembers, + + election_height = ElectionHeight, + election_epoch = ElectionEpoch, + + delayed_vars = DelayedVars, + threshold_txns = ThresholdTxns, + + master_key = MasterKey, + multi_keys = MultiKeys, + vars_nonce = VarsNonce, + vars = Vars, + + gateways = Gateways, + pocs = PoCs, + + accounts = Accounts, + dc_accounts = DCAccounts, + + security_accounts = SecurityAccounts, + + htlcs = HTLCs, + + ouis = OUIs, + subnets = Subnets, + oui_counter = OUICounter, + + hexes = Hexes, + + state_channels = StateChannels, + + blocks = Blocks, + + oracle_price = OraclePrice, + oracle_price_list = OraclePriceList + }) -> + #{ + version => v5, + current_height => CurrHeight, + transaction_fee => 0, + consensus_members => ConsensusMembers, + + election_height => ElectionHeight, + election_epoch => ElectionEpoch, + + delayed_vars => DelayedVars, + threshold_txns => ThresholdTxns, + + master_key => MasterKey, + multi_keys => MultiKeys, + vars_nonce => VarsNonce, + vars => Vars, + + gateways => Gateways, + pocs => PoCs, + + accounts => Accounts, + dc_accounts => DCAccounts, + + security_accounts => SecurityAccounts, + + htlcs => HTLCs, + + ouis => OUIs, + subnets => Subnets, + oui_counter => OUICounter, + + hexes => Hexes, + h3dex => [], + + state_channels => StateChannels, + + blocks => Blocks, + + oracle_price => OraclePrice, + oracle_price_list => OraclePriceList + }. + reserialize(Fun, Values) -> lists:map(fun({K, V}) -> @@ -962,6 +1074,85 @@ v4_to_v3(#blockchain_snapshot_v4{ oracle_price_list = OraclePriceList }. +v5_to_v4(#{ + version := v5, + current_height := CurrHeight, + consensus_members := ConsensusMembers, + + election_height := ElectionHeight, + election_epoch := ElectionEpoch, + + delayed_vars := DelayedVars, + threshold_txns := ThresholdTxns, + + master_key := MasterKey, + multi_keys := MultiKeys, + vars_nonce := VarsNonce, + vars := Vars, + + gateways := Gateways, + pocs := PoCs, + + accounts := Accounts, + dc_accounts := DCAccounts, + + security_accounts := SecurityAccounts, + + htlcs := HTLCs, + + ouis := OUIs, + subnets := Subnets, + oui_counter := OUICounter, + + hexes := Hexes, + h3dex := _H3dex, + + state_channels := StateChannels, + + blocks := Blocks, + + oracle_price := OraclePrice, + oracle_price_list := OraclePriceList}) -> + #blockchain_snapshot_v4{ + current_height = CurrHeight, + consensus_members = ConsensusMembers, + + transaction_fee = 0, + + election_height = ElectionHeight, + election_epoch = ElectionEpoch, + + delayed_vars = DelayedVars, + threshold_txns = ThresholdTxns, + + master_key = MasterKey, + multi_keys = MultiKeys, + vars_nonce = VarsNonce, + vars = Vars, + + gateways = Gateways, + pocs = PoCs, + + accounts = Accounts, + dc_accounts = DCAccounts, + + security_accounts = SecurityAccounts, + + htlcs = HTLCs, + + ouis = OUIs, + subnets = Subnets, + oui_counter = OUICounter, + + hexes = Hexes, + + state_channels = StateChannels, + + blocks = Blocks, + + oracle_price = OraclePrice, + oracle_price_list = OraclePriceList + }. deserialize(Fun, Values) -> lists:map(fun({K, V}) -> @@ -978,10 +1169,15 @@ deserialize_pocs(Values) -> Values). diff(A, B) -> - Fields = record_info(fields, blockchain_snapshot_v4), - [_ | AL] = tuple_to_list(A), - [_ | BL] = tuple_to_list(B), - Comp = lists:zip3(Fields, AL, BL), + Fields = lists:sort(maps:keys(A)), + Comp = lists:foldl( + fun(Field, Acc) -> + AV = maps:get(Field, A), + BV = maps:get(Field, B), + [{Field, AV, BV} | Acc] + end, + [], + Fields), lists:foldl( fun({Field, AI, BI}, Acc) -> case AI == BI of diff --git a/src/ledger/v1/blockchain_ledger_v1.erl b/src/ledger/v1/blockchain_ledger_v1.erl index 4e9e81d988..a438604955 100644 --- a/src/ledger/v1/blockchain_ledger_v1.erl +++ b/src/ledger/v1/blockchain_ledger_v1.erl @@ -120,6 +120,12 @@ clean_all_hexes/1, + bootstrap_h3dex/1, + get_h3dex/1, + lookup_gateways_from_hex/2, + add_gw_to_hex/3, + remove_gw_from_hex/3, + %% snapshot save/restore stuff snapshot_vars/1, @@ -142,6 +148,8 @@ load_state_channels/2, snapshot_hexes/1, load_hexes/2, + snapshot_h3dex/1, + load_h3dex/2, snapshot_delayed_vars/1, load_delayed_vars/2, snapshot_threshold_txns/1, @@ -207,6 +215,7 @@ routing :: rocksdb:cf_handle(), subnets :: rocksdb:cf_handle(), state_channels :: rocksdb:cf_handle(), + h3dex :: rocksdb:cf_handle(), cache :: undefined | ets:tid(), gateway_cache :: undefined | ets:tid() }). @@ -244,15 +253,16 @@ -type state_channel_map() :: #{blockchain_state_channel_v1:id() => blockchain_ledger_state_channel_v1:state_channel() | blockchain_ledger_state_channel_v2:state_channel_v2()}. - +-type h3dex() :: #{h3:h3_index() => [libp2p_crypto:pubkey_bin()]}. %% these keys are gateway addresses -export_type([ledger/0]). -spec new(file:filename_all()) -> ledger(). new(Dir) -> {ok, DB, CFs} = open_db(Dir), [DefaultCF, AGwsCF, EntriesCF, DCEntriesCF, HTLCsCF, PoCsCF, SecuritiesCF, RoutingCF, - SubnetsCF, SCsCF, DelayedDefaultCF, DelayedAGwsCF, DelayedEntriesCF, DelayedDCEntriesCF, - DelayedHTLCsCF, DelayedPoCsCF, DelayedSecuritiesCF, DelayedRoutingCF, DelayedSubnetsCF, DelayedSCsCF] = CFs, + SubnetsCF, SCsCF, H3DexCF, DelayedDefaultCF, DelayedAGwsCF, DelayedEntriesCF, + DelayedDCEntriesCF, DelayedHTLCsCF, DelayedPoCsCF, DelayedSecuritiesCF, + DelayedRoutingCF, DelayedSubnetsCF, DelayedSCsCF, DelayedH3DexCF] = CFs, #ledger_v1{ dir=Dir, db=DB, @@ -268,7 +278,8 @@ new(Dir) -> securities=SecuritiesCF, routing=RoutingCF, subnets=SubnetsCF, - state_channels=SCsCF + state_channels=SCsCF, + h3dex=H3DexCF }, delayed= #sub_ledger_v1{ default=DelayedDefaultCF, @@ -280,7 +291,8 @@ new(Dir) -> securities=DelayedSecuritiesCF, routing=DelayedRoutingCF, subnets=DelayedSubnetsCF, - state_channels=DelayedSCsCF + state_channels=DelayedSCsCF, + h3dex=DelayedH3DexCF } }. @@ -2795,6 +2807,10 @@ state_channels_cf(#ledger_v1{mode=active, active=#sub_ledger_v1{state_channels=S state_channels_cf(#ledger_v1{mode=delayed, delayed=#sub_ledger_v1{state_channels=SCsCF}}) -> SCsCF. +-spec h3dex_cf(ledger()) -> rocksdb:cf_handle(). +h3dex_cf(#ledger_v1{mode=active, active=#sub_ledger_v1{h3dex=H3DexCF}}) -> H3DexCF; +h3dex_cf(#ledger_v1{mode=delayed, delayed=#sub_ledger_v1{h3dex=H3DexCF}}) -> H3DexCF. + -spec cache_put(ledger(), rocksdb:cf_handle(), binary(), binary()) -> ok. cache_put(Ledger, CF, Key, Value) -> {Cache, _GwCache} = context_cache(Ledger), @@ -2950,9 +2966,12 @@ open_db(Dir) -> CFOpts = GlobalOpts, - DefaultCFs = ["default", "active_gateways", "entries", "dc_entries", "htlcs", "pocs", "securities", "routing", "subnets", "state_channels", - "delayed_default", "delayed_active_gateways", "delayed_entries", "delayed_dc_entries", "delayed_htlcs", - "delayed_pocs", "delayed_securities", "delayed_routing", "delayed_subnets","delayed_state_channels"], + DefaultCFs = ["default", "active_gateways", "entries", "dc_entries", "htlcs", + "pocs", "securities", "routing", "subnets", "state_channels", "h3dex", + "delayed_default", "delayed_active_gateways", "delayed_entries", + "delayed_dc_entries", "delayed_htlcs", "delayed_pocs", + "delayed_securities", "delayed_routing", "delayed_subnets", + "delayed_state_channels", "delayed_h3dex"], ExistingCFs = case rocksdb:list_column_families(DBDir, DBOptions) of {ok, CFs0} -> @@ -3090,6 +3109,127 @@ clean_all_hexes(Ledger) -> _ -> ok end. +-spec bootstrap_h3dex(ledger()) -> ok. +bootstrap_h3dex(Ledger) -> + AGwsCF = active_gateways_cf(Ledger), + H3Dex = cache_fold( + Ledger, + AGwsCF, + fun({GwAddr, Binary}, Acc) -> + Gw = blockchain_ledger_gateway_v2:deserialize(Binary), + case blockchain_ledger_gateway_v2:location(Gw) of + undefined -> + Acc; + Location -> + maps:update_with(Location, fun(V) -> [GwAddr | V] end, [GwAddr], Acc) + end + end, + #{}), + set_h3dex(H3Dex, Ledger). + +-spec set_h3dex(h3dex(), ledger()) -> ok. +set_h3dex(H3Dex, Ledger) -> + H3CF = h3dex_cf(Ledger), + _ = maps:map(fun(Loc, Gateways) -> + BinLoc = h3_to_key(Loc), + BinGWs = term_to_binary(Gateways, [compressed]), + cache_put(Ledger, H3CF, BinLoc, BinGWs) + end, H3Dex), + ok. + +-spec get_h3dex(ledger()) -> h3dex(). +get_h3dex(Ledger) -> + H3CF = h3dex_cf(Ledger), + Res = cache_fold(Ledger, H3CF, + fun({Key, GWs}, Acc) -> + maps:put(key_to_h3(Key), binary_to_term(GWs), Acc) + end, #{}, []), + Res. + +-spec lookup_gateways_from_hex(Hex :: [non_neg_integer()] | non_neg_integer(), + Ledger :: ledger()) -> Results :: h3dex(). +%% @doc Given a hex find candidate gateways in the span to the next adjacent +%% hex. N.B. May return an empty map. +lookup_gateways_from_hex(Hexes, Ledger) when is_list(Hexes) -> + lists:foldl(fun(Hex, Acc) -> + maps:merge(Acc, lookup_gateways_from_hex(Hex, Ledger)) + end, #{}, Hexes); +lookup_gateways_from_hex(Hex, Ledger) when is_integer(Hex) -> + H3CF = h3dex_cf(Ledger), + cache_fold(Ledger, H3CF, + fun({Key, GWs}, Acc) -> + maps:put(key_to_h3(Key), binary_to_term(GWs), Acc) + end, #{}, [ + {start, {seek, find_lower_bound_hex(Hex)}}, + {iterate_upper_bound, increment_bin(h3_to_key(Hex))} + ] + ). + +-spec find_lower_bound_hex(Hex :: non_neg_integer()) -> binary(). +%% @doc Let's find the nearest set of k neighbors for this hex at the +%% same resolution and return the "lowest" one. Since these numbers +%% are actually packed binaries, we will destructure them to sort better +%% lexically. +find_lower_bound_hex(Hex) -> + %% both reserved fields must be 0 and Mode must be 1 for this to be a h3 cell + <<0:1, 1:4/integer-unsigned-big, 0:3, Resolution:4/integer-unsigned-big, BaseCell:7/integer-unsigned-big, Digits/bitstring>> = <>, + ActualDigitCount = Resolution * 3, + %% pull out the actual digits used and dump the rest + <> = Digits, + Padding = 45 - ActualDigitCount, + %% store the resolution inverted (15 - 15) = 0 so it sorts earlier + %% pad the actual digits used with 0s on the end + <>. + +h3_to_key(H3) -> + %% both reserved fields must be 0 and Mode must be 1 for this to be a h3 cell + <<0:1/integer-unsigned-big, 1:4/integer-unsigned-big, 0:3/integer-unsigned-big, Resolution:4/integer-unsigned-big, BaseCell:7/integer-unsigned-big, Digits:45/integer-unsigned-big>> = <>, + %% store the resolution inverted (15 - Resolution) so it sorts later + <>. + +key_to_h3(Key) -> + <> = Key, + <> = <<0:1, 1:4/integer-unsigned-big, 0:3, (15 - InverseResolution):4/integer-unsigned-big, BaseCell:7/integer-unsigned-big, Digits:45/integer-unsigned-big>>, + H3. + + +-spec add_gw_to_hex(Hex :: non_neg_integer(), + GWAddr :: libp2p_crypto:pubkey_bin(), + Ledger :: ledger()) -> ok | {error, any()}. +%% @doc During an assert, this function will add a gateway address to a hex +add_gw_to_hex(Hex, GWAddr, Ledger) -> + H3CF = h3dex_cf(Ledger), + BinHex = h3_to_key(Hex), + case cache_get(Ledger, H3CF, BinHex, []) of + not_found -> + cache_put(Ledger, H3CF, BinHex, term_to_binary([GWAddr], [compressed])); + {ok, BinGws} -> + GWs = binary_to_term(BinGws), + cache_put(Ledger, H3CF, BinHex, term_to_binary([GWAddr | GWs], [compressed])); + Error -> Error + end. + +-spec remove_gw_from_hex(Hex :: non_neg_integer(), + GWAddr :: libp2p_crypto:pubkey_bin(), + Ledger :: ledger()) -> ok | {error, any()}. +%% @doc During an assert, if a gateway already had an asserted location +%% (and has been reasserted), this function will remove a gateway +%% address from a hex +remove_gw_from_hex(Hex, GWAddr, Ledger) -> + H3CF = h3dex_cf(Ledger), + BinHex = h3_to_key(Hex), + case cache_get(Ledger, H3CF, BinHex, []) of + not_found -> ok; + {ok, BinGws} -> + case lists:delete(GWAddr, binary_to_term(BinGws)) of + [] -> + cache_delete(Ledger, H3CF, BinHex); + NewGWs -> + cache_put(Ledger, H3CF, BinHex, term_to_binary(NewGWs, [compressed])) + end; + Error -> Error + end. + batch_from_cache(ETS) -> {ok, Batch} = rocksdb:batch(), ets:foldl(fun({{CF, Key}, ?CACHE_TOMBSTONE}, Acc) -> @@ -3441,6 +3581,20 @@ load_hexes(Hexes0, Ledger) -> ok end. +snapshot_h3dex(Ledger) -> + H3CF = h3dex_cf(Ledger), + lists:sort( + maps:to_list( + cache_fold( + Ledger, H3CF, + fun({Loc, GWs}, Acc) -> + maps:put(<>, binary_to_term(GWs), Acc) + end, #{}, + []))). + +load_h3dex(H3DexList, Ledger) -> + set_h3dex(maps:from_list(H3DexList), Ledger). + -spec get_sc_mod( Entry :: blockchain_ledger_state_channel_v1:state_channel() | blockchain_ledger_state_channel_v2:state_channel_v2(), Ledger :: ledger() ) -> blockchain_ledger_state_channel_v1 diff --git a/src/transactions/v1/blockchain_txn_assert_location_v1.erl b/src/transactions/v1/blockchain_txn_assert_location_v1.erl index b74ae74284..48b826bd3d 100644 --- a/src/transactions/v1/blockchain_txn_assert_location_v1.erl +++ b/src/transactions/v1/blockchain_txn_assert_location_v1.erl @@ -396,10 +396,13 @@ absorb(Txn, Chain) -> %% moved within the hex, no need to update ok; _ when OldHex == undefined -> - blockchain_ledger_v1:add_to_hex(Hex, Gateway, Ledger); + blockchain_ledger_v1:add_to_hex(Hex, Gateway, Ledger), + blockchain_ledger_v1:add_gw_to_hex(Hex, Gateway, Ledger); _ -> blockchain_ledger_v1:remove_from_hex(OldHex, Gateway, Ledger), - blockchain_ledger_v1:add_to_hex(Hex, Gateway, Ledger) + blockchain_ledger_v1:remove_gw_from_hex(OldHex, Gateway, Ledger), + blockchain_ledger_v1:add_to_hex(Hex, Gateway, Ledger), + blockchain_ledger_v1:add_gw_to_hex(Hex, Gateway, Ledger) end, diff --git a/src/transactions/v1/blockchain_txn_poc_receipts_v1.erl b/src/transactions/v1/blockchain_txn_poc_receipts_v1.erl index c3a5f400fb..1ffcdab354 100644 --- a/src/transactions/v1/blockchain_txn_poc_receipts_v1.erl +++ b/src/transactions/v1/blockchain_txn_poc_receipts_v1.erl @@ -1269,7 +1269,6 @@ get_channels(Txn, Chain) -> PathLength = length(Path), OnionKeyHash = ?MODULE:onion_key_hash(Txn), - {ok, Head} = blockchain:head_block(Chain), BlockHash = case blockchain:config(?poc_version, blockchain:ledger(Chain)) of {ok, POCVer} when POCVer >= 10 -> @@ -1282,6 +1281,7 @@ get_channels(Txn, Chain) -> blockchain_txn_poc_request_v1:onion_key_hash(T) == OnionKeyHash end, + {ok, Head} = blockchain:head_block(Chain), blockchain:fold_chain(fun(Block, undefined) -> case blockchain_utils:find_txn(Block, RequestFilter) of [_T] -> diff --git a/src/transactions/v1/blockchain_txn_rewards_v1.erl b/src/transactions/v1/blockchain_txn_rewards_v1.erl index 5ae1b6fcae..3c1fa50003 100644 --- a/src/transactions/v1/blockchain_txn_rewards_v1.erl +++ b/src/transactions/v1/blockchain_txn_rewards_v1.erl @@ -194,6 +194,10 @@ calculate_rewards(Start, End, Chain) -> [ConsensusRewards, SecuritiesRewards, POCChallengersRewards, POCChallengeesRewards, POCWitnessesRewards, DCRewards] ), + %% we are only keeping hex density calculations memoized for a single + %% rewards transaction calculation, then we discard that work and avoid + %% cache invalidation issues. + true = blockchain_hex:destroy_memoization(), {ok, Result} end. @@ -234,10 +238,13 @@ get_rewards_for_epoch(Start, End, Chain, Vars, Ledger, DCRewards) -> get_rewards_for_epoch(Start, End, _Chain, Vars0, _Ledger, ChallengerRewards, ChallengeeRewards, WitnessRewards, DCRewards) when Start == End+1 -> {DCRemainder, NewDCRewards} = normalize_dc_rewards(DCRewards, Vars0), Vars = maps:put(dc_remainder, DCRemainder, Vars0), + + NormalizedWitnessRewards = normalize_witness_rewards(WitnessRewards, Vars), + %% apply the DC remainder, if any to the other PoC categories pro rata {ok, normalize_challenger_rewards(ChallengerRewards, Vars), normalize_challengee_rewards(ChallengeeRewards, Vars), - normalize_witness_rewards(WitnessRewards, Vars), + NormalizedWitnessRewards, NewDCRewards}; get_rewards_for_epoch(Current, End, Chain, Vars, Ledger, ChallengerRewards, ChallengeeRewards, WitnessRewards, DCRewards) -> case blockchain:get_block(Current, Chain) of @@ -250,11 +257,43 @@ get_rewards_for_epoch(Current, End, Chain, Vars, Ledger, ChallengerRewards, Chal true -> {error, already_existing_rewards}; false -> - get_rewards_for_epoch(Current+1, End, Chain, Vars, Ledger, - poc_challengers_rewards(Transactions, Vars, ChallengerRewards), - poc_challengees_rewards(Transactions, Vars, Chain, Ledger, ChallengeeRewards), - poc_witnesses_rewards(Transactions, Vars, Chain, Ledger, WitnessRewards), - dc_rewards(Transactions, End, Vars, Ledger, DCRewards)) + case blockchain_hex:var_map(Ledger) of + {error, _} -> + %% do the old thing + get_rewards_for_epoch(Current+1, End, Chain, Vars, Ledger, + poc_challengers_rewards(Transactions, Vars, + ChallengerRewards), + poc_challengees_rewards(Transactions, Vars, + Chain, Ledger, + ChallengeeRewards, #{}), + poc_witnesses_rewards(Transactions, Vars, + Chain, Ledger, + WitnessRewards, #{}), + dc_rewards(Transactions, End, Vars, + Ledger, DCRewards)); + {ok, VarMap} -> + WR = poc_witnesses_rewards(Transactions, Vars, + Chain, Ledger, + WitnessRewards, VarMap), + + C0R = poc_challengers_rewards(Transactions, Vars, + ChallengerRewards), + + DCR = dc_rewards(Transactions, End, Vars, + Ledger, DCRewards), + + CR = poc_challengees_rewards(Transactions, Vars, + Chain, Ledger, + ChallengeeRewards, + VarMap), + + %% do the new thing + get_rewards_for_epoch(Current+1, End, Chain, Vars, Ledger, + C0R, + CR, + WR, + DCR) + end end end. @@ -305,6 +344,11 @@ get_reward_vars(Start, End, Ledger) -> _ -> undefined end, + DensityTgtRes = case blockchain:config(?density_tgt_res, Ledger) of + {ok, D} -> D; + _ -> undefined + end, + EpochReward = calculate_epoch_reward(Start, End, Ledger), #{ monthly_reward => MonthlyReward, @@ -322,7 +366,8 @@ get_reward_vars(Start, End, Ledger) -> poc_version => POCVersion, reward_version => RewardVersion, witness_redundancy => WitnessRedundancy, - poc_reward_decay_rate => DecayRate + poc_reward_decay_rate => DecayRate, + density_tgt_res => DensityTgtRes }. -spec calculate_epoch_reward(pos_integer(), pos_integer(), blockchain_ledger_v1:ledger()) -> float(). @@ -456,12 +501,15 @@ normalize_challenger_rewards(ChallengerRewards, #{epoch_reward := EpochReward, Vars :: map(), Chain :: blockchain:blockchain(), Ledger :: blockchain_ledger_v1:ledger(), - map()) -> #{{gateway, libp2p_crypto:pubkey_bin()} => non_neg_integer()}. + ExistingRewards :: map(), + VarMap :: blockchain_hex:var_map()) -> + #{{gateway, libp2p_crypto:pubkey_bin()} => non_neg_integer()}. poc_challengees_rewards(Transactions, Vars, Chain, Ledger, - ExistingRewards) -> + ExistingRewards, + VarMap) -> lists:foldl( fun(Txn, Acc0) -> case blockchain_txn:type(Txn) == blockchain_txn_poc_receipts_v1 of @@ -469,7 +517,7 @@ poc_challengees_rewards(Transactions, Acc0; true -> Path = blockchain_txn_poc_receipts_v1:path(Txn), - poc_challengees_rewards_(Vars, Path, Path, Txn, Chain, Ledger, true, Acc0) + poc_challengees_rewards_(Vars, Path, Path, Txn, Chain, Ledger, true, VarMap, Acc0) end end, ExistingRewards, @@ -481,7 +529,6 @@ normalize_challengee_rewards(ChallengeeRewards, #{epoch_reward := EpochReward, TotalChallenged = lists:sum(maps:values(ChallengeeRewards)), ShareOfDCRemainder = share_of_dc_rewards(poc_challengees_percent, Vars), ChallengeesReward = (EpochReward * PocChallengeesPercent) + ShareOfDCRemainder, - lager:info("TotalChallenged: ~p", [TotalChallenged]), maps:fold( fun(Challengee, Challenged, Acc) -> PercentofReward = (Challenged*100/TotalChallenged)/100, @@ -493,7 +540,7 @@ normalize_challengee_rewards(ChallengeeRewards, #{epoch_reward := EpochReward, ChallengeeRewards ). -poc_challengees_rewards_(_Vars, [], _StaticPath, _Txn, _Chain, _Ledger, _, Acc) -> +poc_challengees_rewards_(_Vars, [], _StaticPath, _Txn, _Chain, _Ledger, _, _, Acc) -> Acc; poc_challengees_rewards_(#{poc_version := Version}=Vars, [Elem|Path], @@ -502,12 +549,20 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, Chain, Ledger, IsFirst, + VarMap, Acc0) when Version >= 2 -> WitnessRedundancy = maps:get(witness_redundancy, Vars, undefined), DecayRate = maps:get(poc_reward_decay_rate, Vars, undefined), + DensityTgtRes = maps:get(density_tgt_res, Vars, undefined), %% check if there were any legitimate witnesses Witnesses = legit_witnesses(Txn, Chain, Ledger, Elem, StaticPath, Version), Challengee = blockchain_poc_path_element_v1:challengee(Elem), + ChallengeeLoc = case blockchain_ledger_v1:find_gateway_info(Challengee, Ledger) of + {ok, ChallengeeGw} -> + blockchain_ledger_gateway_v2:location(ChallengeeGw); + _ -> + undefined + end, I = maps:get(Challengee, Acc0, 0), case blockchain_poc_path_element_v1:receipt(Elem) of undefined -> @@ -524,7 +579,12 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+1, Acc0); {ok, ToAdd} -> - maps:put(Challengee, I+ToAdd, Acc0) + TxScale = maybe_calc_tx_scale(Challengee, + DensityTgtRes, + ChallengeeLoc, + VarMap, + Ledger), + maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; true when is_integer(Version), Version > 4, IsFirst == false -> %% while we don't have a receipt for this node, we do know @@ -537,12 +597,17 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+2, Acc0); {ok, ToAdd} -> - maps:put(Challengee, I+ToAdd, Acc0) + TxScale = maybe_calc_tx_scale(Challengee, + DensityTgtRes, + ChallengeeLoc, + VarMap, + Ledger), + maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; _ -> Acc0 end, - poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, Acc1); + poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, VarMap, Acc1); Receipt -> case blockchain_poc_receipt_v1:origin(Receipt) of radio -> @@ -559,7 +624,12 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+3, Acc0); {ok, ToAdd} -> - maps:put(Challengee, I+ToAdd, Acc0) + TxScale = maybe_calc_tx_scale(Challengee, + DensityTgtRes, + ChallengeeLoc, + VarMap, + Ledger), + maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; false when is_integer(Version), Version > 4 -> %% this challengee rx'd and sent a receipt @@ -568,18 +638,18 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+2, Acc0); {ok, ToAdd} -> - maps:put(Challengee, I+ToAdd, Acc0) + TxScale = maybe_calc_tx_scale(Challengee, + DensityTgtRes, + ChallengeeLoc, + VarMap, + Ledger), + maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; _ -> - case poc_challengee_reward_unit(WitnessRedundancy, DecayRate, Witnesses) of - {error, _} -> - %% Old behavior - maps:put(Challengee, I+1, Acc0); - {ok, ToAdd} -> - maps:put(Challengee, I+ToAdd, Acc0) - end + %% Old behavior + maps:put(Challengee, I+1, Acc0) end, - poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, Acc1); + poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, VarMap, Acc1); p2p -> %% if there are legitimate witnesses or the path continues %% the challengee did their job @@ -597,29 +667,28 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% Old behavior maps:put(Challengee, I+2, Acc0); {ok, ToAdd} -> - maps:put(Challengee, I+ToAdd, Acc0) + TxScale = maybe_calc_tx_scale(Challengee, + DensityTgtRes, + ChallengeeLoc, + VarMap, + Ledger), + maps:put(Challengee, I+(ToAdd * TxScale), Acc0) end; true -> - case poc_challengee_reward_unit(WitnessRedundancy, DecayRate, Witnesses) of - {error, _} -> - %% Old behavior - maps:put(Challengee, I+1, Acc0); - {ok, ToAdd} -> - maps:put(Challengee, I+ToAdd, Acc0) - end + maps:put(Challengee, I+1, Acc0) end, - poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, Acc1) + poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, VarMap, Acc1) end end; -poc_challengees_rewards_(Vars, [Elem|Path], StaticPath, Txn, Chain, Ledger, _IsFirst, Acc0) -> +poc_challengees_rewards_(Vars, [Elem|Path], StaticPath, Txn, Chain, Ledger, _IsFirst, VarMap, Acc0) -> case blockchain_poc_path_element_v1:receipt(Elem) of undefined -> - poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, Acc0); + poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, VarMap, Acc0); _Receipt -> Challengee = blockchain_poc_path_element_v1:challengee(Elem), I = maps:get(Challengee, Acc0, 0), Acc1 = maps:put(Challengee, I+1, Acc0), - poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, Acc1) + poc_challengees_rewards_(Vars, Path, StaticPath, Txn, Chain, Ledger, false, VarMap, Acc1) end. -spec poc_challengee_reward_unit(WitnessRedundancy :: undefined | pos_integer(), @@ -632,22 +701,29 @@ poc_challengee_reward_unit(WitnessRedundancy, DecayRate, Witnesses) -> {N, R} -> W = length(Witnesses), Unit = poc_reward_tx_unit(R, W, N), - lager:info("poc_challengee_reward_unit: ~p", [Unit]), - {ok, Unit} + {ok, normalize_reward_unit(Unit)} end. +-spec normalize_reward_unit(Unit :: float()) -> float(). +normalize_reward_unit(Unit) when Unit > 1.0 -> 1.0; +normalize_reward_unit(Unit) -> Unit. + -spec poc_witnesses_rewards(Transactions :: blockchain_txn:txns(), Vars :: map(), Chain :: blockchain:blockchain(), Ledger :: blockchain_ledger_v1:ledger(), - WitnessRewards :: map()) -> #{{gateway, libp2p_crypto:pubkey_bin()} => non_neg_integer()}. + WitnessRewards :: map(), + VarMap :: blockchain_hex:var_map()) -> + #{{gateway, libp2p_crypto:pubkey_bin()} => non_neg_integer()}. poc_witnesses_rewards(Transactions, #{poc_version := POCVersion}=Vars, Chain, Ledger, - WitnessRewards) -> + WitnessRewards, + VarMap) -> WitnessRedundancy = maps:get(witness_redundancy, Vars, undefined), DecayRate = maps:get(poc_reward_decay_rate, Vars, undefined), + DensityTgtRes = maps:get(density_tgt_res, Vars, undefined), lists:foldl( fun(Txn, Acc0) -> case blockchain_txn:type(Txn) == blockchain_txn_poc_receipts_v1 of @@ -662,7 +738,7 @@ poc_witnesses_rewards(Transactions, Path = blockchain_txn_poc_receipts_v1:path(Txn), %% Do the new thing for witness filtering - lists:foldl( + Res = lists:foldl( fun(Elem, Acc1) -> ElemPos = blockchain_utils:index_of(Elem, Path), WitnessChannel = lists:nth(ElemPos, Channels), @@ -680,25 +756,50 @@ poc_witnesses_rewards(Transactions, {_, undefined} -> 1; {N, R} -> W = length(ValidWitnesses), - U = poc_reward_rx_unit(R, W, N), - lager:info("poc_reward_rx_unit: ~p", [U]), + U = poc_witness_reward_unit(R, W, N), U end, - lists:foldl( - fun(WitnessRecord, Map) -> - Witness = blockchain_poc_witness_v1:gateway(WitnessRecord), - I = maps:get(Witness, Map, 0), - maps:put(Witness, I+ToAdd, Map) - end, - Acc1, - ValidWitnesses - ) + case DensityTgtRes of + undefined -> + %% old (HIP15) + lists:foldl( + fun(WitnessRecord, Acc2) -> + Witness = blockchain_poc_witness_v1:gateway(WitnessRecord), + I = maps:get(Witness, Acc2, 0), + maps:put(Witness, I+ToAdd, Acc2) + end, + Acc1, + ValidWitnesses + ); + D -> + %% new (HIP17) + lists:foldl( + fun(WitnessRecord, Acc2) -> + Challengee = blockchain_poc_path_element_v1:challengee(Elem), + %% This must always be {ok, ...} + {ok, ChallengeeGw} = blockchain_ledger_v1:find_gateway_info(Challengee, Ledger), + %% Challengee must have a location + ChallengeeLoc = blockchain_ledger_gateway_v2:location(ChallengeeGw), + Witness = blockchain_poc_witness_v1:gateway(WitnessRecord), + %% The witnesses get scaled by the value of their transmitters + RxScale = blockchain_hex:scale(ChallengeeLoc, + VarMap, + D, + Ledger), + I = maps:get(Witness, Acc2, 0), + maps:put(Witness, I+(ToAdd*RxScale), Acc2) + end, + Acc1, + ValidWitnesses + ) + end end end, Acc0, Path - ) + ), + Res catch What:Why:ST -> lager:error("failed to calculate poc_witnesses_rewards, error ~p:~p:~p", [What, Why, ST]), Acc0 @@ -910,21 +1011,19 @@ normalize_dc_rewards(DCRewards0, #{epoch_reward := EpochReward, -spec poc_reward_tx_unit(R :: float(), W :: pos_integer(), N :: pos_integer()) -> float(). -poc_reward_tx_unit(R, W, N) when W =< N -> - lager:info("nonorm, R: ~p, W: ~p, N: ~p, poc_reward_tx_unit: ~p", [R, W, N, W / N]), +poc_reward_tx_unit(_R, W, N) when W =< N -> blockchain_utils:normalize_float(W / N); poc_reward_tx_unit(R, W, N) -> NoNorm = 1 + (1 - math:pow(R, (W - N))), - lager:info("nonorm, R: ~p, W: ~p, N: ~p, poc_reward_tx_unit: ~p", [R, W, N, NoNorm]), blockchain_utils:normalize_float(NoNorm). --spec poc_reward_rx_unit(R :: float(), - W :: pos_integer(), - N :: pos_integer()) -> float(). -poc_reward_rx_unit(_R, W, N) when W =< N -> - 1; -poc_reward_rx_unit(R, W, N) -> - blockchain_utils:normalize_float((N - (1 - math:pow(R, (W - N))))/W). +-spec poc_witness_reward_unit(R :: float(), + W :: pos_integer(), + N :: pos_integer()) -> float(). +poc_witness_reward_unit(_R, W, N) when W =< N -> + 1.0; +poc_witness_reward_unit(R, W, N) -> + normalize_reward_unit(blockchain_utils:normalize_float((N - (1 - math:pow(R, (W - N))))/W)). legit_witnesses(Txn, Chain, Ledger, Elem, StaticPath, Version) -> case Version of @@ -935,8 +1034,8 @@ legit_witnesses(Txn, Chain, Ledger, Elem, StaticPath, Version) -> ElemPos = blockchain_utils:index_of(Elem, StaticPath), WitnessChannel = lists:nth(ElemPos, Channels), ValidWitnesses = blockchain_txn_poc_receipts_v1:valid_witnesses(Elem, WitnessChannel, Ledger), - lager:debug("ValidWitnesses: ~p", - [[blockchain_utils:addr2name(blockchain_poc_witness_v1:gateway(W)) || W <- ValidWitnesses]]), + %% lager:info("ValidWitnesses: ~p", + %% [[blockchain_utils:addr2name(blockchain_poc_witness_v1:gateway(W)) || W <- ValidWitnesses]]), ValidWitnesses catch What:Why:ST -> lager:error("failed to calculate poc_challengees_rewards, error ~p:~p:~p", [What, Why, ST]), @@ -948,6 +1047,21 @@ legit_witnesses(Txn, Chain, Ledger, Elem, StaticPath, Version) -> blockchain_poc_path_element_v1:witnesses(Elem) end. +maybe_calc_tx_scale(_Challengee, + DensityTgtRes, + ChallengeeLoc, + VarMap, + Ledger) -> + case {DensityTgtRes, ChallengeeLoc} of + {undefined, _} -> 1.0; + {_, undefined} -> 1.0; + {D, Loc} -> + TxScale = blockchain_hex:scale(Loc, VarMap, D, Ledger), + %% lager:info("Challengee: ~p, TxScale: ~p", + %% [blockchain_utils:addr2name(Challengee), TxScale]), + TxScale + end. + %%-------------------------------------------------------------------- %% @doc %% @end @@ -969,6 +1083,7 @@ share_of_dc_rewards(_Key, #{dc_remainder := 0}) -> share_of_dc_rewards(Key, Vars=#{dc_remainder := DCRemainder}) -> erlang:round(DCRemainder * ((maps:get(Key, Vars) / (maps:get(poc_challengers_percent, Vars) + maps:get(poc_challengees_percent, Vars) + maps:get(poc_witnesses_percent, Vars))))). + %% ------------------------------------------------------------------ %% EUNIT Tests %% ------------------------------------------------------------------ @@ -1140,7 +1255,8 @@ poc_challengees_rewards_1_test() -> {gateway, poc_challengees, <<"a">>} => 117, {gateway, poc_challengees, <<"b">>} => 233 }, - ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}), Vars)), + ?assertEqual(Rewards, normalize_challengee_rewards( + poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, #{}), Vars)), test_utils:cleanup_tmp_dir(BaseDir). poc_challengees_rewards_2_test() -> @@ -1199,7 +1315,8 @@ poc_challengees_rewards_2_test() -> {gateway, poc_challengees, <<"a">>} => 117, {gateway, poc_challengees, <<"b">>} => 233 }, - ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}), Vars)), + ?assertEqual(Rewards, normalize_challengee_rewards( + poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, {}), Vars)), test_utils:cleanup_tmp_dir(BaseDir). poc_challengees_rewards_3_test() -> @@ -1267,7 +1384,8 @@ poc_challengees_rewards_3_test() -> %% c gets 2 shares {gateway, poc_challengees, <<"c">>} => 44 }, - ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}), Vars)), + ?assertEqual(Rewards, normalize_challengee_rewards( + poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, {}), Vars)), test_utils:cleanup_tmp_dir(BaseDir). poc_witnesses_rewards_test() -> @@ -1323,7 +1441,8 @@ poc_witnesses_rewards_test() -> Rewards = #{{gateway,poc_witnesses,<<"a">>} => 25, {gateway,poc_witnesses,<<"b">>} => 25}, - ?assertEqual(Rewards, normalize_witness_rewards(poc_witnesses_rewards(Txns, EpochVars, Chain, Ledger, #{}), EpochVars)), + ?assertEqual(Rewards, normalize_witness_rewards( + poc_witnesses_rewards(Txns, EpochVars, Chain, Ledger, #{}, #{}), EpochVars)), test_utils:cleanup_tmp_dir(BaseDir). old_poc_challengers_rewards_test() -> @@ -1379,7 +1498,8 @@ old_poc_challengees_rewards_version_1_test() -> {gateway, poc_challengees, <<"1">>} => 175, {gateway, poc_challengees, <<"2">>} => 175 }, - ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}), Vars)), + ?assertEqual(Rewards, normalize_challengee_rewards( + poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, #{}), Vars)), test_utils:cleanup_tmp_dir(BaseDir). old_poc_challengees_rewards_version_2_test() -> @@ -1425,7 +1545,8 @@ old_poc_challengees_rewards_version_2_test() -> {gateway, poc_challengees, <<"1">>} => 175, {gateway, poc_challengees, <<"2">>} => 175 }, - ?assertEqual(Rewards, normalize_challengee_rewards(poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}), Vars)), + ?assertEqual(Rewards, normalize_challengee_rewards( + poc_challengees_rewards(Txns, Vars, Chain, Ledger, #{}, #{}), Vars)), test_utils:cleanup_tmp_dir(BaseDir). old_poc_witnesses_rewards_test() -> @@ -1459,7 +1580,8 @@ old_poc_witnesses_rewards_test() -> {gateway, poc_witnesses, <<"1">>} => 25, {gateway, poc_witnesses, <<"2">>} => 25 }, - ?assertEqual(Rewards, normalize_witness_rewards(poc_witnesses_rewards(Txns, EpochVars, Chain, Ledger, #{}), EpochVars)), + ?assertEqual(Rewards, normalize_witness_rewards( + poc_witnesses_rewards(Txns, EpochVars, Chain, Ledger, #{}, #{}), EpochVars)), test_utils:cleanup_tmp_dir(BaseDir). dc_rewards_test() -> @@ -1656,8 +1778,10 @@ dc_rewards_v3_spillover_test() -> %% compute the original rewards with no spillover ChallengerRewards = normalize_challenger_rewards(poc_challengers_rewards(AllTxns, Vars, #{}), Vars), - ChallengeeRewards = normalize_challengee_rewards(poc_challengees_rewards(AllTxns, Vars, Chain, Ledger, #{}), Vars), - WitnessRewards = normalize_witness_rewards(poc_witnesses_rewards(AllTxns, Vars, Chain, Ledger, #{}), Vars), + ChallengeeRewards = normalize_challengee_rewards( + poc_challengees_rewards(AllTxns, Vars, Chain, Ledger, #{}, #{}), Vars), + WitnessRewards = normalize_witness_rewards( + poc_witnesses_rewards(AllTxns, Vars, Chain, Ledger, #{}, #{}), Vars), ChallengersAward = trunc(maps:get(epoch_reward, Vars) * maps:get(poc_challengers_percent, Vars)), ?assertEqual(#{{gateway,poc_challengers,<<"X">>} => ChallengersAward}, ChallengerRewards), %% entire 15% allocation @@ -1673,8 +1797,10 @@ dc_rewards_v3_spillover_test() -> %% apply the DC remainder, if any to the other PoC categories pro rata SpilloverChallengerRewards = normalize_challenger_rewards(poc_challengers_rewards(AllTxns, Vars, #{}), NewVars), - SpilloverChallengeeRewards = normalize_challengee_rewards(poc_challengees_rewards(AllTxns, Vars, Chain, Ledger, #{}), NewVars), - SpilloverWitnessRewards = normalize_witness_rewards(poc_witnesses_rewards(AllTxns, Vars, Chain, Ledger, #{}), NewVars), + SpilloverChallengeeRewards = normalize_challengee_rewards( + poc_challengees_rewards(AllTxns, Vars, Chain, Ledger, #{}, #{}), NewVars), + SpilloverWitnessRewards = normalize_witness_rewards( + poc_witnesses_rewards(AllTxns, Vars, Chain, Ledger, #{}, #{}), NewVars), ChallengerSpilloverAward = erlang:round(DCRemainder * ((maps:get(poc_challengers_percent, Vars) / (maps:get(poc_challengees_percent, Vars) + maps:get(poc_witnesses_percent, Vars) + diff --git a/src/transactions/v1/blockchain_txn_vars_v1.erl b/src/transactions/v1/blockchain_txn_vars_v1.erl index 8500ba8905..4e636e9b18 100644 --- a/src/transactions/v1/blockchain_txn_vars_v1.erl +++ b/src/transactions/v1/blockchain_txn_vars_v1.erl @@ -1071,10 +1071,101 @@ validate_var(?use_multi_keys, Value) -> validate_var(?transfer_hotspot_stale_poc_blocks, Value) -> validate_int(Value, "transfer_hotspot_stale_poc_blocks", 1, 50000, false); +%% HIP 17 vars +validate_var(?hip17_res_0, Value) -> + validate_hip17_vars(Value, "hip17_res_0"); +validate_var(?hip17_res_1, Value) -> + validate_hip17_vars(Value, "hip17_res_1"); +validate_var(?hip17_res_2, Value) -> + validate_hip17_vars(Value, "hip17_res_2"); +validate_var(?hip17_res_3, Value) -> + validate_hip17_vars(Value, "hip17_res_3"); +validate_var(?hip17_res_4, Value) -> + validate_hip17_vars(Value, "hip17_res_4"); +validate_var(?hip17_res_5, Value) -> + validate_hip17_vars(Value, "hip17_res_5"); +validate_var(?hip17_res_6, Value) -> + validate_hip17_vars(Value, "hip17_res_6"); +validate_var(?hip17_res_7, Value) -> + validate_hip17_vars(Value, "hip17_res_7"); +validate_var(?hip17_res_8, Value) -> + validate_hip17_vars(Value, "hip17_res_8"); +validate_var(?hip17_res_9, Value) -> + validate_hip17_vars(Value, "hip17_res_9"); +validate_var(?hip17_res_10, Value) -> + validate_hip17_vars(Value, "hip17_res_10"); +validate_var(?hip17_res_11, Value) -> + validate_hip17_vars(Value, "hip17_res_11"); +validate_var(?hip17_res_12, Value) -> + validate_hip17_vars(Value, "hip17_res_12"); +validate_var(?density_tgt_res, Value) -> + validate_int(Value, "density_tgt_res", 1, 15, false); +validate_var(?hip17_interactivity_blocks, Value) -> + validate_int(Value, "hip17_interactivity_blocks", 1, 5000, false); + validate_var(Var, Value) -> %% something we don't understand, crash invalid_var(Var, Value). +validate_hip17_vars(Value, Var) when is_binary(Value) -> + case get_density_var(Value) of + {error, _}=E0 -> + lager:error("unable to get density var, reason: ~p", [E0]), + throw({error, {invalid_density_var, Var, Value}}); + {ok, Res} -> + case length(Res) == 3 of + false -> + throw({error, {invalid_size, Var, Value}}); + true -> + [Siblings, DensityTgt, DensityMax] = Res, + CheckSiblings = validate_int_min_max(Siblings, "siblings", 1, 1000), + CheckDensityTgt = validate_int_min_max(DensityTgt, "density_tgt", 1, 200000), + CheckDensityMax = validate_int_min_max(DensityMax, "density_max", 1, 200000), + + case CheckSiblings of + {error, _}=E1 -> + lager:error("invalid_siblings, reason: ~p", [E1]), + throw({error, {invalid_siblings, Var, Value}}); + ok -> + case CheckDensityTgt of + {error, _}=E2 -> + lager:error("invalid_density_tgt, reason: ~p", [E2]), + throw({error, {invalid_density_tgt, Var, Value}}); + ok -> + case CheckDensityMax of + {error, _}=E3 -> + lager:error("invalid_density_max, reason: ~p", [E3]), + throw({error, {invalid_density_max, Var, Value}}); + ok -> + ok + end + end + end + end + end; +validate_hip17_vars(Value, Var) -> + throw({error, {invalid_format, Var, Value}}). + +validate_int_min_max(Value, Name, Min, Max) -> + case Value >= Min andalso Value =< Max of + false -> {error, {list_to_atom(Name ++ "_out_of_range"), Value}}; + _ -> ok + end. + +get_density_var(Value) -> + try + Res = [list_to_integer(I) || I <- string:tokens(binary:bin_to_list(Value), ",")], + {ok, Res} + catch + What:Why:Stack -> + lager:error("Unable to get_density_var, What: ~p, Why: ~p, Stack: ~p", [ + What, + Why, + Stack + ]), + {error, {unable_to_get_density_var, Value}} + end. + -ifdef(TEST). invalid_var(Var, Value) -> case lists:member(Var, ?exceptions) of % test only diff --git a/test/blockchain_ct_utils.erl b/test/blockchain_ct_utils.erl index 0fbead0cbe..7d4a74ab58 100644 --- a/test/blockchain_ct_utils.erl +++ b/test/blockchain_ct_utils.erl @@ -24,7 +24,9 @@ create_vars/0, create_vars/1, raw_vars/1, init_base_dir_config/3, - join_packet/3 + join_packet/3, + ledger/2, + destroy_ledger/0 ]). pmap(F, L) -> @@ -431,3 +433,73 @@ reverse_bin(Bin) -> reverse_bin(Bin, <<>>). reverse_bin(<<>>, Acc) -> Acc; reverse_bin(<>, Acc) -> reverse_bin(Rest, <>). + +ledger(ExtraVars, S3URL) -> + %% Ledger at height: 481929 + %% ActiveGateway Count: 8000 + {ok, Dir} = file:get_cwd(), + %% Ensure priv dir exists + PrivDir = filename:join([Dir, "priv"]), + ok = filelib:ensure_dir(PrivDir ++ "/"), + %% Path to static ledger tar + LedgerTar = filename:join([PrivDir, "ledger.tar.gz"]), + %% Extract ledger tar if required + ok = extract_ledger_tar(PrivDir, LedgerTar, S3URL), + %% Get the ledger + Ledger = blockchain_ledger_v1:new(PrivDir), + %% Get current ledger vars + LedgerVars = ledger_vars(Ledger), + %% Ensure the ledger has the vars we're testing against + Ledger1 = blockchain_ledger_v1:new_context(Ledger), + blockchain_ledger_v1:vars(maps:merge(LedgerVars, ExtraVars), [], Ledger1), + %% If the hexes aren't on the ledger add them + blockchain:bootstrap_hexes(Ledger1), + blockchain_ledger_v1:commit_context(Ledger1), + Ledger. + +extract_ledger_tar(PrivDir, LedgerTar, S3URL) -> + case filelib:is_file(LedgerTar) of + true -> + %% if we have already unpacked it, no need to do it again + LedgerDB = filename:join([PrivDir, "ledger.db"]), + case filelib:is_dir(LedgerDB) of + true -> + ok; + false -> + %% ledger tar file present, extract + erl_tar:extract(LedgerTar, [compressed, {cwd, PrivDir}]) + end; + false -> + %% ledger tar file not found, download & extract + ok = ssl:start(), + {ok, {{_, 200, "OK"}, _, Body}} = httpc:request(S3URL), + ok = file:write_file(filename:join([PrivDir, "ledger.tar.gz"]), Body), + erl_tar:extract(LedgerTar, [compressed, {cwd, PrivDir}]) + end. + +ledger_vars(Ledger) -> + blockchain_utils:vars_binary_keys_to_atoms(maps:from_list(blockchain_ledger_v1:snapshot_vars(Ledger))). + +destroy_ledger() -> + {ok, Dir} = file:get_cwd(), + %% Ensure priv dir exists + PrivDir = filename:join([Dir, "priv"]), + ok = filelib:ensure_dir(PrivDir ++ "/"), + LedgerTar = filename:join([PrivDir, "ledger.tar.gz"]), + LedgerDB = filename:join([PrivDir, "ledger.db"]), + + case filelib:is_file(LedgerTar) of + true -> + %% we found a ledger tarball, remove it + file:delete(LedgerTar); + false -> + ok + end, + case filelib:is_dir(LedgerDB) of + true -> + %% we found a ledger.db, remove it + file:del_dir(LedgerDB); + false -> + %% ledger.db dir not found, don't do anything + ok + end. diff --git a/test/blockchain_hex_SUITE.erl b/test/blockchain_hex_SUITE.erl new file mode 100644 index 0000000000..68cddb6de8 --- /dev/null +++ b/test/blockchain_hex_SUITE.erl @@ -0,0 +1,410 @@ +-module(blockchain_hex_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-include("blockchain_vars.hrl"). + +-export([ + all/0, + init_per_testcase/2, + end_per_testcase/2, + init_per_suite/1, + end_per_suite/1 +]). + +-export([ + full_known_values_test/1, + known_values_test/1, + known_differences_test/1, + scale_test/1, + h3dex_test/1, + export_scale_test/1 +]). + +%% Values taken from python model +-define(KNOWN, [ + {631210968849144319, 1}, + {631179325713598463, 2}, + {631196205757572607, 1}, + {631243921328349695, 1}, + {631243921527814655, 3}, + {631211399470291455, 1}, + {577234808489377791, 2319}, + {577023702256844799, 449}, + {577692205326532607, 1039}, + {577340361605644287, 6}, + {576812596024311807, 54}, + {577621836582354943, 5}, + {577164439745200127, 1459}, + {579768083279773695, 2}, + {577305177233555455, 5}, + {576601489791778815, 9}, + {576636674163867647, 56}, + {576777411652222975, 3}, + {577481099093999615, 43}, + {578114417791598591, 1}, + {577269992861466623, 1}, + {577586652210266111, 24}, + {577762574070710271, 381}, + {577199624117288959, 1817}, + {577058886628933631, 1}, + {578325524024131583, 1}, + {577832942814887935, 26}, + {576953333512667135, 2}, + {579838452023951359, 1}, + {579451423930974207, 1}, + {576918149140578303, 600}, + {599663374669709311, 3}, + {599238598109167615, 1}, + {599653418935517183, 4}, + {600256126327455743, 2}, + {600176343014965247, 1}, + {599718760420474879, 2}, + {599736317173039103, 1}, + {599721183855771647, 2}, + {599685859897245695, 1} +]). + +%% There are 1952 differences when we calculate clipped vs unclipped densities +-define(KNOWN_CLIP_VS_UNCLIPPED, 1952). + +%% Some known differences between clipped vs unclipped densities +-define(KNOWN_DIFFERENCES, #{ + 617712130192310271 => {1, 2}, + 617761313717485567 => {1, 2}, + 617733151106007039 => {2, 3}, + 617684909104562175 => {2, 3}, + 617700174986739711 => {1, 2}, + 617700552987901951 => {2, 4}, + 617700552891170815 => {2, 4}, + 617733123580887039 => {1, 2}, + 617733151091589119 => {1, 3}, + 617743888390553599 => {1, 3}, + 617733269885550591 => {1, 2} +}). + +%% Value taken from hip17 +-define(KNOWN_RES_FOR_SCALING, "8828361563fffff"). + +all() -> + [ + known_values_test, + known_differences_test, + scale_test, + h3dex_test + %% export_scale_test + ]. + +%%-------------------------------------------------------------------- +%% TEST SUITE SETUP +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + LedgerURL = "https://blockchain-core.s3-us-west-1.amazonaws.com/ledger-586724.tar.gz", + Ledger = blockchain_ct_utils:ledger(hip17_vars(), LedgerURL), + + ok = application:set_env(blockchain, hip17_test_mode, true), + + Ledger1 = blockchain_ledger_v1:new_context(Ledger), + blockchain:bootstrap_h3dex(Ledger1), + blockchain_ledger_v1:commit_context(Ledger1), + blockchain_ledger_v1:compact(Ledger), + + %% Check that the pinned ledger is at the height we expect it to be + {ok, 586724} = blockchain_ledger_v1:current_height(Ledger), + + %% Check that the vars are correct, one is enough... + {ok, VarMap} = blockchain_hex:var_map(Ledger), + ct:pal("var_map: ~p", [VarMap]), + #{n := 1, tgt := 250, max := 800} = maps:get(4, VarMap), + + [ + {ledger, Ledger}, + {var_map, VarMap} + | Config + ]. + +%%-------------------------------------------------------------------- +%% TEST CASE SETUP +%%-------------------------------------------------------------------- +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% TEST CASE TEARDOWN +%%-------------------------------------------------------------------- +end_per_testcase(_, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% TEST SUITE TEARDOWN +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + %% destroy the downloaded ledger + blockchain_ct_utils:destroy_ledger(), + ok. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%% XXX: Remove this test when done cross-checking with python +full_known_values_test(Config) -> + Ledger = ?config(ledger, Config), + + {ok, [List]} = file:consult("/tmp/tracker.erl"), + + %% assert some known values calculated from the python model (thanks @para1!) + true = lists:all( + fun({Hex, Res, UnclippedValue, _Limit, ClippedValue}) -> + case h3:get_resolution(Hex) of + 0 -> + true; + _ -> + {ok, VarMap} = blockchain_hex:var_map(Ledger), + + {UnclippedDensities, ClippedDensities} = blockchain_hex:densities( + Hex, + VarMap, + Ledger + ), + + Dex = blockchain_ledger_v1:lookup_gateways_from_hex(Hex, Ledger), + Spots = [blockchain_utils:addr2name(I) || I <- lists:flatten(maps:values(Dex))], + NumSpots = length(Spots), + + ct:pal( + "Hex: ~p, Res: ~p, PythonUnclippedDensity: ~p, CalculatedUnclippedDensity: ~p, PythonClippedDensity: ~p, CalculatedClippedDensity: ~p, NumSpots: ~p, Spots: ~p", + [ + Hex, + Res, + UnclippedValue, + maps:get(Hex, UnclippedDensities), + ClippedValue, + maps:get(Hex, ClippedDensities), + NumSpots, + Spots + ] + ), + + UnclippedValue == maps:get(Hex, UnclippedDensities) andalso + ClippedValue == maps:get(Hex, ClippedDensities) + end + end, + List + ), + + ok. + +known_values_test(Config) -> + Ledger = ?config(ledger, Config), + + %% assert some known values calculated from the python model (thanks @para1!) + true = lists:all( + fun({Hex, Density}) -> + case h3:get_resolution(Hex) of + 0 -> + true; + _ -> + {ok, VarMap} = blockchain_hex:var_map(Ledger), + {_, ClippedDensities} = blockchain_hex:densities(Hex, VarMap, Ledger), + + ct:pal("~p ~p", [ + Density, + maps:get(Hex, ClippedDensities) + ]), + + Density == maps:get(Hex, ClippedDensities) + end + end, + ?KNOWN + ), + + ok. + +known_differences_test(Config) -> + Ledger = ?config(ledger, Config), + + true = lists:all( + fun({Hex, {Clipped, Unclipped}}) -> + {ok, VarMap} = blockchain_hex:var_map(Ledger), + {UnclippedDensities, ClippedDensities} = blockchain_hex:densities(Hex, VarMap, Ledger), + GotUnclipped = maps:get(Hex, UnclippedDensities), + GotClipped = maps:get(Hex, ClippedDensities), + ct:pal( + "Hex: ~p, Clipped: ~p, GotClipped: ~p, Unclipped: ~p, GotUnclipped: ~p", + [Hex, Clipped, GotClipped, Unclipped, GotUnclipped] + ), + Unclipped == GotUnclipped andalso Clipped == GotClipped + end, + maps:to_list(?KNOWN_DIFFERENCES) + ), + + ok. + +scale_test(Config) -> + Ledger = ?config(ledger, Config), + VarMap = ?config(var_map, Config), + TargetResolutions = lists:seq(3, 10), + KnownHex = h3:from_string("8c2836152804dff"), + + Result = lists:foldl( + fun(TargetRes, Acc) -> + Scale = blockchain_hex:scale(KnownHex, VarMap, TargetRes, Ledger), + ct:pal("TargetRes: ~p, Scale: ~p", [TargetRes, Scale]), + blockchain_hex:destroy_memoization(), + maps:put(TargetRes, Scale, Acc) + end, + #{}, + TargetResolutions + ), + + ct:pal("Result: ~p", [Result]), + + "0.18" = io_lib:format("~.2f", [maps:get(8, Result)]), + + ok. + +h3dex_test(Config) -> + Ledger = ?config(ledger, Config), + + %% A known hotspot hex at res=12, there's only one here + Hex = 631236347406370303, + HexPubkeyBin = + <<0, 161, 86, 254, 148, 82, 27, 153, 2, 52, 158, 118, 1, 178, 133, 150, 238, 135, 228, 40, + 114, 253, 149, 194, 89, 170, 68, 170, 122, 230, 130, 196, 139>>, + + Gateways = blockchain_ledger_v1:lookup_gateways_from_hex(Hex, Ledger), + + GotPubkeyBin = hd(maps:get(Hex, Gateways)), + + ?assertEqual(1, map_size(Gateways)), + ?assertEqual(GotPubkeyBin, HexPubkeyBin), + + ct:pal("Hex: ~p", [Hex]), + ct:pal("Gateways: ~p", [Gateways]), + + ok. + +export_scale_test(Config) -> + Ledger = ?config(ledger, Config), + VarMap = ?config(var_map, Config), + + %% A list of possible density target resolution we'd output the scales at + DensityTargetResolutions = lists:seq(3, 10), + + %% Only do this for gateways with known locations + GatewaysWithLocs = gateways_with_locs(Ledger), + + ok = export_scale_data( + Ledger, + VarMap, + DensityTargetResolutions, + GatewaysWithLocs + ), + + ok. + +%%-------------------------------------------------------------------- +%% CHAIN VARIABLES +%%-------------------------------------------------------------------- + +hip17_vars() -> + #{ + hip17_res_0 => <<"2,100000,100000">>, + hip17_res_1 => <<"2,100000,100000">>, + hip17_res_2 => <<"2,100000,100000">>, + hip17_res_3 => <<"2,100000,100000">>, + hip17_res_4 => <<"1,250,800">>, + hip17_res_5 => <<"1,100,400">>, + hip17_res_6 => <<"1,25,100">>, + hip17_res_7 => <<"2,5,20">>, + hip17_res_8 => <<"2,1,4">>, + hip17_res_9 => <<"2,1,2">>, + hip17_res_10 => <<"2,1,1">>, + hip17_res_11 => <<"2,100000,100000">>, + hip17_res_12 => <<"2,100000,100000">>, + density_tgt_res => 8, + hip17_interactivity_blocks => 1200 * 3 + }. + +%%-------------------------------------------------------------------- +%% INTERNAL FUNCTIONS +%%-------------------------------------------------------------------- + +gateways_with_locs(Ledger) -> + AG = blockchain_ledger_v1:active_gateways(Ledger), + + maps:fold( + fun(Addr, GW, Acc) -> + case blockchain_ledger_gateway_v2:location(GW) of + undefined -> Acc; + Loc -> [{blockchain_utils:addr2name(Addr), Loc} | Acc] + end + end, + [], + AG + ). + +export_scale_data(Ledger, VarMap, DensityTargetResolutions, GatewaysWithLocs) -> + %% Calculate scale at each density target res for eventual comparison + lists:foreach( + fun(TargetRes) -> + %% Export scale data for every single gateway to a gps file + Scales = lists:foldl( + fun({GwName, Loc}, Acc) -> + Scale = blockchain_hex:scale( + Loc, + VarMap, + TargetRes, + Ledger + ), + [{GwName, Loc, Scale} | Acc] + end, + [], + GatewaysWithLocs + ), + + Fname = "/tmp/scale_" ++ integer_to_list(TargetRes), + ok = export_gps_file(Fname, Scales) + end, + DensityTargetResolutions + ). + +export_gps_file(Fname, Scales) -> + Header = ["name,latitude,longitude,h3,color,desc"], + + Data = lists:foldl( + fun({Name, H3, ScaleVal}, Acc) -> + {Lat, Long} = h3:to_geo(H3), + ToAppend = + Name ++ + "," ++ + io_lib:format("~.20f", [Lat]) ++ + "," ++ + io_lib:format("~.20f", [Long]) ++ + "," ++ + integer_to_list(H3) ++ + "," ++ + color(ScaleVal) ++ + "," ++ + io_lib:format("scale: ~p", [ScaleVal]), + [ToAppend | Acc] + end, + [], + Scales + ), + + TotalData = Header ++ Data, + + LineSep = io_lib:nl(), + Print = [string:join(TotalData, LineSep), LineSep], + file:write_file(Fname, Print), + ok. + +color(1.0) -> "green"; +color(V) when V =< 0.1 -> "red"; +color(V) when V =< 0.3 -> "orange"; +color(V) when V =< 0.5 -> "yellow"; +color(V) when V =< 0.8 -> "cyan"; +color(_) -> "blue". diff --git a/test/blockchain_reward_hip17_SUITE.erl b/test/blockchain_reward_hip17_SUITE.erl new file mode 100644 index 0000000000..ea3ba2b85c --- /dev/null +++ b/test/blockchain_reward_hip17_SUITE.erl @@ -0,0 +1,1159 @@ +%% +%% This is primarily a static test +%% +%% For reference: +%% -------------------------------------------------------------------------------------------------- +%% fake_name real_name letter location scale +%% -------------------------------------------------------------------------------------------------- +%% tall-azure-whale striped-umber-copperhead e 631210990515645439 0.25 +%% basic-seaweed-walrun fresh-honey-lizard b 631210990515609087 0.25 +%% brief-saffron-cricket round-fiery-platypus k 631210990516667903 0.5 +%% mini-eggshell-panther quick-admiral-gecko j 631210990528935935 0.2 +%% silly-azure-chicken curly-ivory-alligator d 631210990528385535 0.2 +%% daring-watermelon-sloth skinny-amber-condor c 631210990528546815 0.2 +%% keen-ultraviolet-guppy fantastic-blue-lemur i 631210990529462783 0.2 +%% sharp-green-swift crazy-silver-platypus a 631210990529337343 0.2 +%% bitter-saffron-ferret icy-paisley-lizard g 631210990524024831 0.33 +%% creamy-porcelain-poodled able-mocha-rhino h 631210990524753919 0.33 +%% blurry-raising-wolf puny-bubblegum-shell f 631210990525267455 0.33 +%% +%% Essentially we will check: +%% +%% - test_between_vars: +%% - Do the rewards follow: no_var > hip15_var >= hip17_var. +%% +%% - test_between_challengees: +%% - Paths tested: [e, h, j], [k, h, j], [g, h, j] +%% +%% - For path: [e, h, j] +%% - e: 0.25, h: 0.33, ignore j +%% - e's witnesses [a, b, c, f] should get scaled using 0.25 +%% - h's witnesses [i] should get scaled using 0.33 +%% +%% - For path: [k, h, j] +%% - k: 0.5, h: 0.33, ignore j +%% - k's witnesses [a, b, c, f] should get scaled using 0.5 +%% - h's witnesses [i] should get scaled using 0.33 +%% +%% - For path: [g, h, j] +%% - g: 0.33, h: 0.33, ignore j +%% - g's witnesses [a, b, c, f] should get scaled using 0.33 +%% - h's witnesses [i] should get scaled using 0.33 +%% + +-module(blockchain_reward_hip17_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-include("blockchain_vars.hrl"). + +-export([ + all/0, + groups/0, + init_per_group/2, + end_per_group/2, + init_per_suite/1, + end_per_suite/1, + init_per_testcase/2, + end_per_testcase/2 +]). + +-export([ + no_var_test/1, + only_hip15_vars_test/1, + hip15_17_vars_test/1, + comparison_test/1, + path_ehj/1, + path_khj/1, + path_ghj/1, + compare_challengee_test/1 +]). + +all_var_tests() -> + [ + no_var_test, + only_hip15_vars_test, + hip15_17_vars_test, + comparison_test + ]. + +all_challengee_tests() -> + [ + path_ehj, + path_khj, + path_ghj, + compare_challengee_test + ]. + +all() -> + [{group, test_between_vars}, {group, test_between_challengees}]. + +groups() -> + [{test_between_vars, + [], + all_var_tests() + }, + {test_between_challengees, + [], + all_challengee_tests() + }]. + +%%-------------------------------------------------------------------- +%% TEST GROUP SETUP +%%-------------------------------------------------------------------- +init_per_group(_, Config) -> + {ok, StorePid} = blockchain_test_reward_store:start(), + [{store, StorePid} | Config]. + +%%-------------------------------------------------------------------- +%% TEST GROUP TEARDOWN +%%-------------------------------------------------------------------- +end_per_group(_, _Config) -> + ok = blockchain_test_reward_store:stop(), + ok. + +%%-------------------------------------------------------------------- +%% TEST SUITE SETUP +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + + GenesisKeys = static_keys(), + GenesisMembers = [{PubkeyBin,{Pubkey, PrivKey, libp2p_crypto:mk_sig_fun(PrivKey)}} || {PubkeyBin, Pubkey, PrivKey} <- GenesisKeys], + Locations = known_locations(), + + [{genesis_members, GenesisMembers}, + {locations, Locations} | Config]. + +%%-------------------------------------------------------------------- +%% TEST SUITE TEARDOWN +%%-------------------------------------------------------------------- + +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% TEST CASE SETUP +%%-------------------------------------------------------------------- + +init_per_testcase(TestCase, Config) -> + Config0 = blockchain_ct_utils:init_base_dir_config(?MODULE, TestCase, Config), + Balance = 5000, + {ok, Sup, {PrivKey, PubKey}, Opts} = test_utils:init(?config(base_dir, Config0)), + ok = application:set_env(blockchain, hip17_test_mode, true), + application:ensure_all_started(lager), + + BothHip15And17Vars = maps:merge(hip15_vars(), hip17_vars()), + + ExtraVars = + case TestCase of + only_hip15_vars_test -> + hip15_vars(); + hip15_17_vars_test -> + BothHip15And17Vars; + path_ehj -> + BothHip15And17Vars; + path_ghj -> + BothHip15And17Vars; + path_khj -> + BothHip15And17Vars; + _ -> + #{} + end, + + GenesisMembers = ?config(genesis_members, Config), + Locations = ?config(locations, Config), + + {ok, _GenesisBlock, ConsensusMembers, Keys} = + test_utils:init_chain_with_fixed_locations(Balance, GenesisMembers, Locations, ExtraVars), + + Chain = blockchain_worker:blockchain(), + Swarm = blockchain_swarm:swarm(), + N = length(ConsensusMembers), + + % Check ledger to make sure everyone has the right balance + Ledger = blockchain:ledger(Chain), + + %% Add hexes to the ledger + LedgerC = blockchain_ledger_v1:new_context(Ledger), + ok = blockchain:bootstrap_hexes(LedgerC), + ok = blockchain_ledger_v1:bootstrap_h3dex(LedgerC), + ok = blockchain_ledger_v1:commit_context(LedgerC), + ok = blockchain_ledger_v1:compact(Ledger), + + Entries = blockchain_ledger_v1:entries(Ledger), + _ = lists:foreach( + fun(Entry) -> + Balance = blockchain_ledger_entry_v1:balance(Entry), + 0 = blockchain_ledger_entry_v1:nonce(Entry) + end, + maps:values(Entries) + ), + + meck:new(blockchain_txn_rewards_v1, [passthrough]), + meck:new(blockchain_txn_poc_receipts_v1, [passthrough]), + meck:new(blockchain_txn_poc_request_v1, [passthrough]), + + case TestCase of + hip15_17_vars_test -> + lists:foreach(fun(Loc) -> + Scale = blockchain_hex:scale(Loc, Ledger), + ct:pal("Loc: ~p, Scale: ~p", [Loc, Scale]) + end, + known_locations()); + _ -> + ok + end, + + [ + {balance, Balance}, + {sup, Sup}, + {pubkey, PubKey}, + {privkey, PrivKey}, + {opts, Opts}, + {chain, Chain}, + {swarm, Swarm}, + {n, N}, + {consensus_members, ConsensusMembers}, + {tc_name, TestCase}, + Keys + | Config0 + ]. + +%%-------------------------------------------------------------------- +%% TEST CASE TEARDOWN +%%-------------------------------------------------------------------- + +end_per_testcase(_TestCase, Config) -> + meck:unload(blockchain_txn_rewards_v1), + meck:unload(blockchain_txn_poc_request_v1), + meck:unload(blockchain_txn_poc_receipts_v1), + meck:unload(), + Sup = ?config(sup, Config), + BaseDir = ?config(base_dir, Config), + % Make sure blockchain saved on file = in memory + case erlang:is_process_alive(Sup) of + true -> + true = erlang:exit(Sup, normal), + ok = test_utils:wait_until(fun() -> false =:= erlang:is_process_alive(Sup) end); + false -> + ok + end, + test_utils:cleanup_tmp_dir(BaseDir), + ok. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +no_var_test(Config) -> + Witnesses = [b, c, e, f, g], + run_vars_test(Witnesses, Config). + +only_hip15_vars_test(Config) -> + %% - We have gateways: [a, b, c, d, e, f, g, h, i, j, k] + %% - We'll make a poc receipt txn by hand, without any validation + %% - We'll also consider that all witnesses are legit (legit_witnesses) + %% - The poc transaction will have path like so: a -> d -> h + %% - For a -> d; [b, c, e, f, g] will all be witnesses, their rewards should get scaled + %% - For d -> h; 0 witnesses + Witnesses = [b, c, e, f, g], + run_vars_test(Witnesses, Config). + +hip15_17_vars_test(Config) -> + %% - We have gateways: [a, b, c, d, e, f, g, h, i, j, k] + %% - We'll make a poc receipt txn by hand, without any validation + %% - We'll also consider that all witnesses are legit (legit_witnesses) + %% - The poc transaction will have path like so: a -> d -> h + %% - For a -> d; [b, c, e, f, g] will all be witnesses, their rewards should get scaled + %% - For d -> h; 0 witnesses + Witnesses = [b, c, e, f, g], + run_vars_test(Witnesses, Config). + +comparison_test(_Config) -> + %% Aggregate rewards from no_var_test + NoVarChallengeeRewards = blockchain_test_reward_store:fetch(no_var_test_challengee_rewards), + NoVarWitnessRewards = blockchain_test_reward_store:fetch(no_var_test_witness_rewards), + + TotalNoVarChallengeeRewards = lists:sum(maps:values(NoVarChallengeeRewards)), + TotalNoVarWitnessRewards = lists:sum(maps:values(NoVarWitnessRewards)), + + FractionalNoVarChallengeeRewards = maps:map(fun(_, V) -> V / TotalNoVarChallengeeRewards end, NoVarChallengeeRewards), + FractionalNoVarWitnessRewards = maps:map(fun(_, V) -> V / TotalNoVarWitnessRewards end, NoVarWitnessRewards), + + %% Aggregate rewards from hip15 test + Hip15ChallengeeRewards = blockchain_test_reward_store:fetch(only_hip15_vars_test_challengee_rewards), + Hip15WitnessRewards = blockchain_test_reward_store:fetch(only_hip15_vars_test_witness_rewards), + + TotalHip15ChallengeeRewards = lists:sum(maps:values(Hip15ChallengeeRewards)), + TotalHip15WitnessRewards = lists:sum(maps:values(Hip15WitnessRewards)), + + FractionalHip15ChallengeeRewards = maps:map(fun(_, V) -> V / TotalHip15ChallengeeRewards end, Hip15ChallengeeRewards), + FractionalHip15WitnessRewards = maps:map(fun(_, V) -> V / TotalHip15WitnessRewards end, Hip15WitnessRewards), + + %% Aggregate rewards from hip17 test + Hip17ChallengeeRewards = blockchain_test_reward_store:fetch(hip15_17_vars_test_challengee_rewards), + Hip17WitnessRewards = blockchain_test_reward_store:fetch(hip15_17_vars_test_witness_rewards), + + TotalHip17ChallengeeRewards = lists:sum(maps:values(Hip17ChallengeeRewards)), + TotalHip17WitnessRewards = lists:sum(maps:values(Hip17WitnessRewards)), + + FractionalHip17ChallengeeRewards = maps:map(fun(_, V) -> V / TotalHip17ChallengeeRewards end, Hip17ChallengeeRewards), + FractionalHip17WitnessRewards = maps:map(fun(_, V) -> V / TotalHip17WitnessRewards end, Hip17WitnessRewards), + + ct:pal("NoVarChallengeeRewards: ~p", [NoVarChallengeeRewards]), + ct:pal("FractionalNoVarChallengeeRewards: ~p", [FractionalNoVarChallengeeRewards]), + + ct:pal("NoVarWitnessRewards: ~p", [NoVarWitnessRewards]), + ct:pal("FractionalNoVarWitnessRewards: ~p", [FractionalNoVarWitnessRewards]), + + ct:pal("Hip15ChallengeeRewards: ~p", [Hip15ChallengeeRewards]), + ct:pal("FractionalHip15ChallengeeRewards: ~p", [FractionalHip15ChallengeeRewards]), + + ct:pal("Hip15WitnessRewards: ~p", [Hip15WitnessRewards]), + ct:pal("FractionalHip15WitnessRewards: ~p", [FractionalHip15WitnessRewards]), + + ct:pal("Hip17ChallengeeRewards: ~p", [Hip17ChallengeeRewards]), + ct:pal("FractionalHip17ChallengeeRewards: ~p", [FractionalHip17ChallengeeRewards]), + + ct:pal("Hip17WitnessRewards: ~p", [Hip17WitnessRewards]), + ct:pal("FractionalHip17WitnessRewards: ~p", [FractionalHip17WitnessRewards]), + + + %% challengee rewards for no_vars >= only_hip15_vars >= hip15_17_vars + Challengees = maps:keys(NoVarChallengeeRewards), + + true = lists:all(fun(Challengee) -> + NoVarReward = maps:get(Challengee, NoVarChallengeeRewards), + Hip15Reward = maps:get(Challengee, Hip15ChallengeeRewards), + Hip17Reward = maps:get(Challengee, Hip17ChallengeeRewards), + (NoVarReward >= Hip15Reward) andalso (Hip15Reward >= Hip17Reward) + end, Challengees), + + ok. + + +path_ehj(Config) -> + run_challengees_test(d, e, h, j, [a, b, c, f], [i], Config). + +path_khj(Config) -> + run_challengees_test(d, k, h, j, [a, b, c, f], [i], Config). + +path_ghj(Config) -> + run_challengees_test(d, g, h, j, [a, b, c, f], [i], Config). + +compare_challengee_test(_Config) -> + ChallengeeE = maps:get(e, blockchain_test_reward_store:fetch(path_ehj_challengee_rewards)), + ChallengeeG = maps:get(g, blockchain_test_reward_store:fetch(path_ghj_challengee_rewards)), + ChallengeeK = maps:get(k, blockchain_test_reward_store:fetch(path_khj_challengee_rewards)), + + ct:pal("ChallengeeE: ~p, ChallengeeG: ~p, ChallengeeK: ~p", + [ChallengeeE, ChallengeeG, ChallengeeK]), + + ChallengeeChecks = ((ChallengeeK > ChallengeeG) andalso (ChallengeeK > ChallengeeE) andalso (ChallengeeG > ChallengeeE)), + + true = ChallengeeChecks, + + %% for path e, h, j + %% i gets higher scaled reward + WitnessA_PathEHJ = maps:get(a, blockchain_test_reward_store:fetch(path_ehj_witness_rewards)), + WitnessB_PathEHJ = maps:get(b, blockchain_test_reward_store:fetch(path_ehj_witness_rewards)), + WitnessC_PathEHJ = maps:get(c, blockchain_test_reward_store:fetch(path_ehj_witness_rewards)), + WitnessF_PathEHJ = maps:get(f, blockchain_test_reward_store:fetch(path_ehj_witness_rewards)), + WitnessI_PathEHJ = maps:get(i, blockchain_test_reward_store:fetch(path_ehj_witness_rewards)), + + true = (WitnessA_PathEHJ == WitnessB_PathEHJ) andalso + (WitnessC_PathEHJ == WitnessF_PathEHJ) andalso + (WitnessA_PathEHJ == WitnessC_PathEHJ) andalso + (WitnessI_PathEHJ > WitnessA_PathEHJ), + + %% for path k, h, j + %% i gets lower scaled reward + WitnessA_PathKHJ = maps:get(a, blockchain_test_reward_store:fetch(path_khj_witness_rewards)), + WitnessB_PathKHJ = maps:get(b, blockchain_test_reward_store:fetch(path_khj_witness_rewards)), + WitnessC_PathKHJ = maps:get(c, blockchain_test_reward_store:fetch(path_khj_witness_rewards)), + WitnessF_PathKHJ = maps:get(f, blockchain_test_reward_store:fetch(path_khj_witness_rewards)), + WitnessI_PathKHJ = maps:get(i, blockchain_test_reward_store:fetch(path_khj_witness_rewards)), + + true = (WitnessA_PathKHJ == WitnessB_PathKHJ) andalso + (WitnessC_PathKHJ == WitnessF_PathKHJ) andalso + (WitnessA_PathKHJ == WitnessC_PathKHJ) andalso + (WitnessI_PathKHJ < WitnessA_PathKHJ), + + %% for path g, h, j + %% i gets equal scaled reward + WitnessA_PathGHJ = maps:get(a, blockchain_test_reward_store:fetch(path_ghj_witness_rewards)), + WitnessB_PathGHJ = maps:get(b, blockchain_test_reward_store:fetch(path_ghj_witness_rewards)), + WitnessC_PathGHJ = maps:get(c, blockchain_test_reward_store:fetch(path_ghj_witness_rewards)), + WitnessF_PathGHJ = maps:get(f, blockchain_test_reward_store:fetch(path_ghj_witness_rewards)), + WitnessI_PathGHJ = maps:get(i, blockchain_test_reward_store:fetch(path_ghj_witness_rewards)), + + true = (WitnessA_PathGHJ == WitnessB_PathGHJ) andalso + (WitnessC_PathGHJ == WitnessF_PathGHJ) andalso + (WitnessA_PathGHJ == WitnessC_PathGHJ) andalso + (WitnessI_PathGHJ == WitnessA_PathGHJ), + + ok. + +%%-------------------------------------------------------------------- +%% HELPER +%%-------------------------------------------------------------------- +run_challengees_test(Constructor, + Elem1, + Elem2, + Elem3, + Layer1Witnesses, + Layer2Witnesses, + Config) -> + GenesisMembers = ?config(genesis_members, Config), + ConsensusMembers = ?config(consensus_members, Config), + Chain = ?config(chain, Config), + TCName = ?config(tc_name, Config), + #{ gateway_names := GatewayNameMap, gateway_letter_to_addr := GatewayLetterToAddrMap } = cross_check_maps(Config), + + Challenger = maps:get(Constructor, GatewayLetterToAddrMap), + {_, {_, _, ChallengerSigFun}} = lists:keyfind(Challenger, 1, GenesisMembers), + + GwElem1 = maps:get(Elem1, GatewayLetterToAddrMap), + GwElem2 = maps:get(Elem2, GatewayLetterToAddrMap), + GwElem3 = maps:get(Elem3, GatewayLetterToAddrMap), + + Rx1 = blockchain_poc_receipt_v1:new( + GwElem1, + 1000, + 10, + <<"first_rx">>, + p2p + ), + Rx2 = blockchain_poc_receipt_v1:new( + GwElem2, + 1000, + 10, + <<"second_rx">>, + radio + ), + Rx3 = blockchain_poc_receipt_v1:new( + GwElem3, + 1000, + 10, + <<"third_rx">>, + radio + ), + + ConstructedWitnesses1 = lists:foldl( + fun(W, Acc) -> + WitnessGw = maps:get(W, GatewayLetterToAddrMap), + Witness = blockchain_poc_witness_v1:new( + WitnessGw, + 1001, + 10, + crypto:strong_rand_bytes(32), + 9, + 915, + 10, + "data_rate" + ), + [Witness | Acc] + end, + [], + Layer1Witnesses + ), + + ConstructedWitnesses2 = lists:foldl( + fun(W, Acc) -> + WitnessGw = maps:get(W, GatewayLetterToAddrMap), + Witness = blockchain_poc_witness_v1:new( + WitnessGw, + 1001, + 10, + crypto:strong_rand_bytes(32), + 9, + 915, + 10, + "data_rate" + ), + [Witness | Acc] + end, + [], + Layer2Witnesses + ), + + %% Construct poc receipt txn + P1 = blockchain_poc_path_element_v1:new(GwElem1, Rx1, ConstructedWitnesses1), + P2 = blockchain_poc_path_element_v1:new(GwElem2, Rx2, ConstructedWitnesses2), + P3 = blockchain_poc_path_element_v1:new(GwElem3, Rx3, []), + + ct:pal("P1: ~p", [P1]), + ct:pal("P2: ~p", [P2]), + ct:pal("P3: ~p", [P3]), + + %% We'll consider all the witnesses to be "good quality" for the sake of testing + meck:expect(blockchain_txn_poc_request_v1, is_valid, fun(_, _) -> + ok + end), + meck:expect(blockchain_txn_poc_receipts_v1, is_valid, fun(_, _) -> ok end), + meck:expect(blockchain_txn_poc_receipts_v1, absorb, fun(_, _) -> ok end), + meck:expect(blockchain_txn_poc_receipts_v1, get_channels, fun(_, _) -> + {ok, lists:seq(1, 11)} + end), + meck:expect(blockchain_txn_poc_receipts_v1, good_quality_witnesses, + fun + (E, _) when E == P1 -> + ConstructedWitnesses1; + (E, _) when E == P2 -> + ConstructedWitnesses2; + (_, _) -> + [] + end), + meck:expect( + blockchain_txn_rewards_v1, + legit_witnesses, + fun + (_, _, _, E, _, _) when E == P1 -> + ConstructedWitnesses1; + (_, _, _, E, _, _) when E == P2 -> + ConstructedWitnesses2; + (_, _, _, _, _, _) -> + [] + end + ), + meck:expect(blockchain_txn_poc_receipts_v1, valid_witnesses, + fun + (E, _, _) when E == P1 -> + ConstructedWitnesses1; + (E, _, _) when E == P2 -> + ConstructedWitnesses2; + (_, _, _) -> + [] + end), + + + Secret = crypto:strong_rand_bytes(32), + OnionKeyHash = crypto:strong_rand_bytes(32), + BlockHash = crypto:strong_rand_bytes(32), + + RTxn0 = blockchain_txn_poc_request_v1:new(Challenger, + Secret, + OnionKeyHash, + BlockHash, + 10), + RTxn = blockchain_txn_poc_request_v1:sign(RTxn0, ChallengerSigFun), + ct:pal("RTxn: ~p", [RTxn]), + %% Construct a block for the poc request txn + {ok, Block2} = test_utils:create_block(ConsensusMembers, [RTxn], #{}, false), + ct:pal("Block2: ~p", [Block2]), + _ = blockchain_gossip_handler:add_block(Block2, Chain, self(), blockchain_swarm:swarm()), + ?assertEqual({ok, 2}, blockchain:height(Chain)), + + Txn0 = blockchain_txn_poc_receipts_v1:new( + Challenger, + Secret, + OnionKeyHash, + BlockHash, + [P1, P2, P3] + ), + Txn = blockchain_txn_poc_receipts_v1:sign(Txn0, ChallengerSigFun), + ct:pal("Txn: ~p", [Txn]), + + %% Construct a block for the poc receipt txn WITHOUT validation + {ok, Block3} = test_utils:create_block(ConsensusMembers, [Txn], #{}, false), + ct:pal("Block3: ~p", [Block3]), + _ = blockchain_gossip_handler:add_block(Block3, Chain, self(), blockchain_swarm:swarm()), + ?assertEqual({ok, 3}, blockchain:height(Chain)), + + %% Empty block + {ok, Block4} = test_utils:create_block(ConsensusMembers, []), + ct:pal("Block4: ~p", [Block4]), + _ = blockchain_gossip_handler:add_block(Block4, Chain, self(), blockchain_swarm:swarm()), + ?assertEqual({ok, 4}, blockchain:height(Chain)), + + %% Calculate rewards by hand + Start = 1, + End = 3, + {ok, Rewards} = blockchain_txn_rewards_v1:calculate_rewards(Start, End, Chain), + + ChallengeesRewards = lists:filter( + fun(R) -> + blockchain_txn_reward_v1:type(R) == poc_challengees + end, + Rewards + ), + + WitnessRewards = lists:filter( + fun(R) -> + blockchain_txn_reward_v1:type(R) == poc_witnesses + end, + Rewards + ), + + ChallengeesRewardsMap = + lists:foldl( + fun(R, Acc) -> + maps:put( + maps:get( + blockchain_utils:addr2name(blockchain_txn_reward_v1:gateway(R)), + GatewayNameMap + ), + blockchain_txn_reward_v1:amount(R), + Acc + ) + end, + #{}, + ChallengeesRewards + ), + + WitnessRewardsMap = + lists:foldl( + fun(R, Acc) -> + maps:put( + maps:get( + blockchain_utils:addr2name(blockchain_txn_reward_v1:gateway(R)), + GatewayNameMap + ), + blockchain_txn_reward_v1:amount(R), + Acc + ) + end, + #{}, + WitnessRewards + ), + + ct:pal("ChallengeesRewardsMap: ~p", [ChallengeesRewardsMap]), + ct:pal("WitnessRewardsMap: ~p", [WitnessRewardsMap]), + + ok = blockchain_test_reward_store:insert( + list_to_atom(atom_to_list(TCName) ++ "_witness_rewards"), + WitnessRewardsMap + ), + ok = blockchain_test_reward_store:insert( + list_to_atom(atom_to_list(TCName) ++ "_challengee_rewards"), + ChallengeesRewardsMap + ), + + ok. + + +run_vars_test(Witnesses, Config) -> + GenesisMembers = ?config(genesis_members, Config), + ConsensusMembers = ?config(consensus_members, Config), + Chain = ?config(chain, Config), + TCName = ?config(tc_name, Config), + #{ gateway_names := GatewayNameMap, gateway_letter_to_addr := GatewayLetterToAddrMap } = cross_check_maps(Config), + + Challenger = maps:get(k, GatewayLetterToAddrMap), + {_, {_, _, ChallengerSigFun}} = lists:keyfind(Challenger, 1, GenesisMembers), + + GwA = maps:get(a, GatewayLetterToAddrMap), + GwD = maps:get(d, GatewayLetterToAddrMap), + GwI = maps:get(i, GatewayLetterToAddrMap), + + Rx1 = blockchain_poc_receipt_v1:new( + GwA, + 1000, + 10, + <<"first_rx">>, + p2p + ), + Rx2 = blockchain_poc_receipt_v1:new( + GwD, + 1000, + 10, + <<"second_rx">>, + radio + ), + Rx3 = blockchain_poc_receipt_v1:new( + GwI, + 1000, + 10, + <<"third_rx">>, + radio + ), + + ConstructedWitnesses = lists:foldl( + fun(W, Acc) -> + WitnessGw = maps:get(W, GatewayLetterToAddrMap), + Witness = blockchain_poc_witness_v1:new( + WitnessGw, + 1001, + 10, + crypto:strong_rand_bytes(32), + 9.8, + 915.2, + 10, + "data_rate" + ), + [Witness | Acc] + end, + [], + Witnesses + ), + + %% We'll consider all the witnesses to be "good quality" for the sake of testing + meck:expect( + blockchain_txn_rewards_v1, + legit_witnesses, + fun(_, _, _, _, _, _) -> + ConstructedWitnesses + end + ), + meck:expect(blockchain_txn_poc_request_v1, is_valid, fun(_, _) -> + ok + end), + meck:expect(blockchain_txn_poc_receipts_v1, is_valid, fun(_, _) -> ok end), + meck:expect(blockchain_txn_poc_receipts_v1, absorb, fun(_, _) -> ok end), + meck:expect(blockchain_txn_poc_receipts_v1, valid_witnesses, fun(_, _, _) -> + ConstructedWitnesses + end), + meck:expect(blockchain_txn_poc_receipts_v1, good_quality_witnesses, fun(_, _) -> + ConstructedWitnesses + end), + meck:expect(blockchain_txn_poc_receipts_v1, get_channels, fun(_, _) -> + {ok, lists:seq(1, 11)} + end), + %% meck:expect(blockchain_hex, destroy_memoization, fun() -> + %% true + %% end), + + Secret = crypto:strong_rand_bytes(32), + OnionKeyHash = crypto:strong_rand_bytes(32), + BlockHash = crypto:strong_rand_bytes(32), + + RTxn0 = blockchain_txn_poc_request_v1:new(Challenger, + Secret, + OnionKeyHash, + BlockHash, + 10), + RTxn = blockchain_txn_poc_request_v1:sign(RTxn0, ChallengerSigFun), + ct:pal("RTxn: ~p", [RTxn]), + %% Construct a block for the poc request txn + {ok, Block2} = test_utils:create_block(ConsensusMembers, [RTxn], #{}, false), + ct:pal("Block2: ~p", [Block2]), + _ = blockchain_gossip_handler:add_block(Block2, Chain, self(), blockchain_swarm:swarm()), + ?assertEqual({ok, 2}, blockchain:height(Chain)), + + %% Construct poc receipt txn + P1 = blockchain_poc_path_element_v1:new(GwA, Rx1, []), + P2 = blockchain_poc_path_element_v1:new(GwD, Rx2, ConstructedWitnesses), + P3 = blockchain_poc_path_element_v1:new(GwI, Rx3, []), + + ct:pal("P1: ~p", [P1]), + ct:pal("P2: ~p", [P2]), + ct:pal("P3: ~p", [P3]), + + Txn0 = blockchain_txn_poc_receipts_v1:new( + Challenger, + Secret, + OnionKeyHash, + BlockHash, + [P1, P2, P3] + ), + Txn = blockchain_txn_poc_receipts_v1:sign(Txn0, ChallengerSigFun), + ct:pal("Txn: ~p", [Txn]), + + %% Construct a block for the poc receipt txn WITHOUT validation + {ok, Block3} = test_utils:create_block(ConsensusMembers, [Txn], #{}, false), + ct:pal("Block3: ~p", [Block3]), + _ = blockchain_gossip_handler:add_block(Block3, Chain, self(), blockchain_swarm:swarm()), + ?assertEqual({ok, 3}, blockchain:height(Chain)), + + %% Empty block + {ok, Block4} = test_utils:create_block(ConsensusMembers, []), + ct:pal("Block4: ~p", [Block4]), + _ = blockchain_gossip_handler:add_block(Block4, Chain, self(), blockchain_swarm:swarm()), + ?assertEqual({ok, 4}, blockchain:height(Chain)), + + %% Calculate rewards by hand + Start = 1, + End = 3, + {ok, Rewards} = blockchain_txn_rewards_v1:calculate_rewards(Start, End, Chain), + + ChallengeesRewards = lists:filter( + fun(R) -> + blockchain_txn_reward_v1:type(R) == poc_challengees + end, + Rewards + ), + + WitnessRewards = lists:filter( + fun(R) -> + blockchain_txn_reward_v1:type(R) == poc_witnesses + end, + Rewards + ), + + ChallengeesRewardsMap = + lists:foldl( + fun(R, Acc) -> + maps:put( + maps:get( + blockchain_utils:addr2name(blockchain_txn_reward_v1:gateway(R)), + GatewayNameMap + ), + blockchain_txn_reward_v1:amount(R), + Acc + ) + end, + #{}, + ChallengeesRewards + ), + + WitnessRewardsMap = + lists:foldl( + fun(R, Acc) -> + maps:put( + maps:get( + blockchain_utils:addr2name(blockchain_txn_reward_v1:gateway(R)), + GatewayNameMap + ), + blockchain_txn_reward_v1:amount(R), + Acc + ) + end, + #{}, + WitnessRewards + ), + + ct:pal("ChallengeesRewardsMap: ~p", [ChallengeesRewardsMap]), + ct:pal("WitnessRewardsMap: ~p", [WitnessRewardsMap]), + + ok = blockchain_test_reward_store:insert( + list_to_atom(atom_to_list(TCName) ++ "_witness_rewards"), + WitnessRewardsMap + ), + ok = blockchain_test_reward_store:insert( + list_to_atom(atom_to_list(TCName) ++ "_challengee_rewards"), + ChallengeesRewardsMap + ), + + ok. + +cross_check_maps(Config) -> + Chain = ?config(chain, Config), + Ledger = blockchain:ledger(Chain), + AG = blockchain_ledger_v1:active_gateways(Ledger), + GatewayAddrs = lists:sort(maps:keys(AG)), + AllGws = [a, b, c, d, e, f, g, h, i, j, k], + GatewayNameMap = lists:foldl( + fun({Letter, A}, Acc) -> + maps:put(blockchain_utils:addr2name(A), Letter, Acc) + end, + #{}, + lists:zip(AllGws, GatewayAddrs) + ), + + %% For crosscheck + GatewayLocMap = lists:foldl( + fun(A, Acc) -> + {ok, Gw} = blockchain_ledger_v1:find_gateway_info(A, Ledger), + GwLoc = blockchain_ledger_gateway_v2:location(Gw), + maps:put(blockchain_utils:addr2name(A), GwLoc, Acc) + end, + #{}, + GatewayAddrs + ), + + %% For crosscheck + GatewayLetterToAddrMap = lists:foldl( + fun({Letter, A}, Acc) -> + maps:put(Letter, A, Acc) + end, + #{}, + lists:zip(AllGws, GatewayAddrs) + ), + + %% For crosscheck + GatewayLetterLocMap = lists:foldl( + fun({Letter, A}, Acc) -> + {ok, Gw} = blockchain_ledger_v1:find_gateway_info(A, Ledger), + GwLoc = blockchain_ledger_gateway_v2:location(Gw), + maps:put(Letter, GwLoc, Acc) + end, + #{}, + lists:zip(AllGws, GatewayAddrs) + ), + + #{ + gateway_names => GatewayNameMap, + gateway_locs => GatewayLocMap, + gateway_letter_to_addr => GatewayLetterToAddrMap, + gateway_letter_to_loc => GatewayLetterLocMap + }. + + +hip15_vars() -> + #{ + %% configured on chain + ?poc_version => 10, + ?reward_version => 5, + %% new hip15 vars for testing + ?poc_reward_decay_rate => 0.8, + ?witness_redundancy => 4 + }. + +hip17_vars() -> + #{ + ?hip17_res_0 => <<"2,100000,100000">>, + ?hip17_res_1 => <<"2,100000,100000">>, + ?hip17_res_2 => <<"2,100000,100000">>, + ?hip17_res_3 => <<"2,100000,100000">>, + ?hip17_res_4 => <<"1,250,800">>, + ?hip17_res_5 => <<"1,100,400">>, + ?hip17_res_6 => <<"1,25,100">>, + ?hip17_res_7 => <<"2,5,20">>, + ?hip17_res_8 => <<"2,1,4">>, + ?hip17_res_9 => <<"2,1,2">>, + ?hip17_res_10 => <<"2,1,1">>, + ?hip17_res_11 => <<"2,100000,100000">>, + ?hip17_res_12 => <<"2,100000,100000">>, + ?density_tgt_res => 8 + }. + +known_locations() -> + [631210990515645439, + 631210990515609087, + 631210990516667903, + 631210990528935935, + 631210990528385535, + 631210990528546815, + 631210990529462783, + 631210990529337343, + 631210990524024831, + 631210990524753919, + 631210990525267455 + ]. + +static_keys() -> + %% These static keys are only generated to ensure we get consistent results + %% between successive test runs + [{<<0,135,35,145,156,56,241,143,56,203,33,59,137,227,51,220,158,75,110, + 92,79,47,197,189,61,2,166,40,158,85,94,187,210>>, + {ecc_compact,{{'ECPoint',<<4,135,35,145,156,56,241,143,56,203,33,59, + 137,227,51,220,158,75,110,92,79,47,197,189, + 61,2,166,40,158,85,94,187,210,23,124,197, + 126,82,136,156,224,77,29,244,7,181,54,27, + 193,238,247,20,220,223,82,172,29,184,166, + 244,80,180,158,234,200>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<156,174,164,201,194,229,247,253,190,55, + 192,6,178,54,77,8,107,207,119,165,225, + 56,57,77,56,93,18,7,204,3,87,19>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,135,35,145,156,56,241,143,56,203, + 33,59,137,227,51,220,158,75,110,92, + 79,47,197,189,61,2,166,40,158,85,94, + 187,210,23,124,197,126,82,136,156, + 224,77,29,244,7,181,54,27,193,238, + 247,20,220,223,82,172,29,184,166,244, + 80,180,158,234,200>>}}}, + {<<0,20,71,124,234,252,184,6,227,161,188,190,47,137,191,118,55,90,107, + 76,18,110,125,250,200,219,154,35,120,32,13,85,162>>, + {ecc_compact,{{'ECPoint',<<4,20,71,124,234,252,184,6,227,161,188,190, + 47,137,191,118,55,90,107,76,18,110,125,250, + 200,219,154,35,120,32,13,85,162,82,177,64, + 62,209,191,108,219,71,95,159,45,165,110,57, + 225,131,208,229,15,227,239,68,150,156,254, + 184,111,119,196,72,157>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<74,195,136,8,5,64,160,74,239,70,204,88, + 28,125,218,23,158,45,29,211,216,145,16, + 44,78,66,232,65,60,96,37,195>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,20,71,124,234,252,184,6,227,161, + 188,190,47,137,191,118,55,90,107,76, + 18,110,125,250,200,219,154,35,120,32, + 13,85,162,82,177,64,62,209,191,108, + 219,71,95,159,45,165,110,57,225,131, + 208,229,15,227,239,68,150,156,254, + 184,111,119,196,72,157>>}}}, + {<<0,217,189,89,225,39,191,180,16,213,28,126,134,2,140,86,174,237,57, + 197,104,123,176,138,216,163,253,140,2,4,159,237,17>>, + {ecc_compact,{{'ECPoint',<<4,217,189,89,225,39,191,180,16,213,28,126, + 134,2,140,86,174,237,57,197,104,123,176, + 138,216,163,253,140,2,4,159,237,17,100,96, + 191,118,251,152,42,105,120,182,220,31,13, + 120,76,56,254,170,50,153,63,47,84,160,68, + 36,156,45,187,209,160,81>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<225,210,133,73,145,176,53,145,226,86,23, + 195,148,179,16,42,71,247,197,165,158, + 144,151,227,103,187,209,110,2,16,100,99>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,217,189,89,225,39,191,180,16,213, + 28,126,134,2,140,86,174,237,57,197, + 104,123,176,138,216,163,253,140,2,4, + 159,237,17,100,96,191,118,251,152,42, + 105,120,182,220,31,13,120,76,56,254, + 170,50,153,63,47,84,160,68,36,156,45, + 187,209,160,81>>}}}, + {<<0,215,157,27,147,66,217,10,140,181,194,91,108,130,23,111,221,203,186, + 194,157,244,168,53,45,184,162,228,141,214,155,106,122>>, + {ecc_compact,{{'ECPoint',<<4,215,157,27,147,66,217,10,140,181,194,91, + 108,130,23,111,221,203,186,194,157,244,168, + 53,45,184,162,228,141,214,155,106,122,38, + 162,55,138,90,19,132,142,109,40,39,237,77, + 117,34,14,160,114,41,62,104,54,56,240,115, + 124,9,53,189,251,42,56>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<77,233,24,124,91,70,54,186,28,49,82,177, + 176,200,8,68,211,204,31,128,74,142,24, + 118,112,207,51,86,10,74,155,139>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,215,157,27,147,66,217,10,140,181, + 194,91,108,130,23,111,221,203,186, + 194,157,244,168,53,45,184,162,228, + 141,214,155,106,122,38,162,55,138,90, + 19,132,142,109,40,39,237,77,117,34, + 14,160,114,41,62,104,54,56,240,115, + 124,9,53,189,251,42,56>>}}}, + {<<0,85,63,44,227,26,250,122,155,247,250,201,91,215,217,210,181,152,209, + 90,103,54,116,57,145,2,191,107,135,227,150,188,139>>, + {ecc_compact,{{'ECPoint',<<4,85,63,44,227,26,250,122,155,247,250,201, + 91,215,217,210,181,152,209,90,103,54,116, + 57,145,2,191,107,135,227,150,188,139,106, + 164,125,200,121,31,25,3,125,231,89,189,212, + 151,43,237,167,194,1,145,180,132,128,149, + 213,142,55,113,43,57,129,192>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<241,178,164,75,59,239,80,187,86,100,0, + 137,105,108,64,161,36,76,103,226,66,241, + 42,114,119,131,125,203,205,2,213,21>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,85,63,44,227,26,250,122,155,247, + 250,201,91,215,217,210,181,152,209, + 90,103,54,116,57,145,2,191,107,135, + 227,150,188,139,106,164,125,200,121, + 31,25,3,125,231,89,189,212,151,43, + 237,167,194,1,145,180,132,128,149, + 213,142,55,113,43,57,129,192>>}}}, + {<<0,33,183,166,74,151,68,17,14,58,106,91,30,171,149,116,42,54,136,187, + 6,135,149,78,44,132,144,224,168,180,185,5,210>>, + {ecc_compact,{{'ECPoint',<<4,33,183,166,74,151,68,17,14,58,106,91,30, + 171,149,116,42,54,136,187,6,135,149,78,44, + 132,144,224,168,180,185,5,210,8,205,10,58, + 37,17,206,158,32,200,182,231,53,43,66,110, + 7,107,125,127,244,91,98,235,213,107,130, + 177,229,189,26,225>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<35,37,85,183,224,18,148,11,77,133,138, + 152,248,222,56,7,8,70,251,212,124,223, + 107,122,184,18,15,60,254,173,172,18>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,33,183,166,74,151,68,17,14,58,106, + 91,30,171,149,116,42,54,136,187,6, + 135,149,78,44,132,144,224,168,180, + 185,5,210,8,205,10,58,37,17,206,158, + 32,200,182,231,53,43,66,110,7,107, + 125,127,244,91,98,235,213,107,130, + 177,229,189,26,225>>}}}, + {<<0,197,206,105,167,43,204,77,56,215,206,79,130,83,194,243,95,100,232, + 161,135,166,145,34,142,103,155,65,147,209,189,13,145>>, + {ecc_compact,{{'ECPoint',<<4,197,206,105,167,43,204,77,56,215,206,79, + 130,83,194,243,95,100,232,161,135,166,145, + 34,142,103,155,65,147,209,189,13,145,120,0, + 190,129,210,122,118,155,125,166,201,50,78, + 61,217,80,236,99,106,75,181,30,27,230,173, + 173,102,84,25,102,28,126>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<248,151,148,106,71,235,30,171,175,38,61, + 208,228,196,194,195,249,95,180,188,95, + 132,216,225,68,184,114,177,226,242,21,60>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,197,206,105,167,43,204,77,56,215, + 206,79,130,83,194,243,95,100,232,161, + 135,166,145,34,142,103,155,65,147, + 209,189,13,145,120,0,190,129,210,122, + 118,155,125,166,201,50,78,61,217,80, + 236,99,106,75,181,30,27,230,173,173, + 102,84,25,102,28,126>>}}}, + {<<0,7,94,141,107,189,125,163,87,153,196,200,77,40,78,50,238,22,1,154, + 70,45,135,148,16,46,120,188,198,164,147,250,255>>, + {ecc_compact,{{'ECPoint',<<4,7,94,141,107,189,125,163,87,153,196,200, + 77,40,78,50,238,22,1,154,70,45,135,148,16, + 46,120,188,198,164,147,250,255,19,149,195, + 194,244,103,75,60,21,25,210,102,160,221, + 218,228,176,202,38,39,78,110,184,59,52,196, + 95,173,105,168,140,210>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<208,185,203,219,249,155,80,235,230,243, + 229,64,55,110,230,34,135,106,11,22,26, + 202,149,11,135,154,242,158,9,77,136,242>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,7,94,141,107,189,125,163,87,153, + 196,200,77,40,78,50,238,22,1,154,70, + 45,135,148,16,46,120,188,198,164,147, + 250,255,19,149,195,194,244,103,75,60, + 21,25,210,102,160,221,218,228,176, + 202,38,39,78,110,184,59,52,196,95, + 173,105,168,140,210>>}}}, + {<<0,161,224,241,247,215,177,248,246,170,70,111,93,20,168,111,142,225, + 183,129,50,237,242,215,38,34,0,224,216,228,118,55,26>>, + {ecc_compact,{{'ECPoint',<<4,161,224,241,247,215,177,248,246,170,70, + 111,93,20,168,111,142,225,183,129,50,237, + 242,215,38,34,0,224,216,228,118,55,26,68, + 26,209,120,63,198,91,107,11,223,80,59,88, + 34,239,206,159,182,46,177,249,154,53,8,38, + 195,129,102,176,32,85,201>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<180,123,47,194,133,184,161,103,6,218, + 189,247,36,157,70,102,114,5,199,223,38, + 24,244,74,248,111,229,69,30,232,234,205>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,161,224,241,247,215,177,248,246, + 170,70,111,93,20,168,111,142,225,183, + 129,50,237,242,215,38,34,0,224,216, + 228,118,55,26,68,26,209,120,63,198, + 91,107,11,223,80,59,88,34,239,206, + 159,182,46,177,249,154,53,8,38,195, + 129,102,176,32,85,201>>}}}, + {<<0,181,132,129,89,193,104,228,73,203,137,46,161,153,156,56,205,253, + 243,206,109,218,93,13,9,77,222,143,147,148,135,39,15>>, + {ecc_compact,{{'ECPoint',<<4,181,132,129,89,193,104,228,73,203,137,46, + 161,153,156,56,205,253,243,206,109,218,93, + 13,9,77,222,143,147,148,135,39,15,122,11, + 108,114,34,18,8,83,94,141,159,31,164,88, + 209,127,192,66,45,132,137,86,40,57,122,188, + 185,86,99,180,161,224>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<155,29,3,116,69,125,244,121,73,22,179, + 49,210,8,187,245,179,70,6,58,3,60,12, + 136,25,186,144,133,58,236,232,160>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,181,132,129,89,193,104,228,73,203, + 137,46,161,153,156,56,205,253,243, + 206,109,218,93,13,9,77,222,143,147, + 148,135,39,15,122,11,108,114,34,18,8, + 83,94,141,159,31,164,88,209,127,192, + 66,45,132,137,86,40,57,122,188,185, + 86,99,180,161,224>>}}}, + {<<0,156,114,72,115,65,211,12,113,160,134,127,252,250,62,10,149,32,182, + 30,19,158,41,162,182,224,15,48,57,27,13,50,200>>, + {ecc_compact,{{'ECPoint',<<4,156,114,72,115,65,211,12,113,160,134,127, + 252,250,62,10,149,32,182,30,19,158,41,162, + 182,224,15,48,57,27,13,50,200,108,73,193, + 233,44,47,102,122,37,188,76,253,248,32,143, + 166,59,54,47,46,239,151,157,211,72,75,185, + 81,203,125,20,50>>}, + {namedCurve,{1,2,840,10045,3,1,7}}}}, + {ecc_compact,{'ECPrivateKey',1, + <<32,249,3,92,124,248,0,146,225,224,17,2, + 87,54,7,126,165,185,28,110,196,141,58, + 35,250,244,162,224,158,40,0,28>>, + {namedCurve,{1,2,840,10045,3,1,7}}, + <<4,156,114,72,115,65,211,12,113,160, + 134,127,252,250,62,10,149,32,182,30, + 19,158,41,162,182,224,15,48,57,27, + 13,50,200,108,73,193,233,44,47,102, + 122,37,188,76,253,248,32,143,166,59, + 54,47,46,239,151,157,211,72,75,185, + 81,203,125,20,50>>}}}]. diff --git a/test/blockchain_reward_perf_SUITE.erl b/test/blockchain_reward_perf_SUITE.erl new file mode 100644 index 0000000000..e1d7f87831 --- /dev/null +++ b/test/blockchain_reward_perf_SUITE.erl @@ -0,0 +1,157 @@ +%%%------------------------------------------------------------------- +%%% @author Evan Vigil-McClanahan +%%% @copyright (C) 2020, Evan Vigil-McClanahan +%%% @doc +%%% +%%% @end +%%% Created : 1 Dec 2020 by Evan Vigil-McClanahan +%%%------------------------------------------------------------------- +-module(blockchain_reward_perf_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include("blockchain_vars.hrl"). + + +-export([ + suite/0, + all/0, + init_per_suite/1, + end_per_suite/1, + init_per_testcase/2, + end_per_testcase/2 + ]). + +-export([ + reward_perf_test/1, + hip15_vars/0, + hip17_vars/0 + ]). + +%%-------------------------------------------------------------------- +%% @spec suite() -> Info +%% Info = [tuple()] +%% @end +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{seconds,200}}]. + +%%-------------------------------------------------------------------- +%% @spec init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% @end +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% @spec end_per_suite(Config0) -> term() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%% @end +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + {ok, _} = application:ensure_all_started(lager), + + {ok, Dir} = file:get_cwd(), + PrivDir = filename:join([Dir, "priv"]), + NewDir = PrivDir ++ "/ledger/", + ok = filelib:ensure_dir(NewDir), + + os:cmd("wget https://blockchain-core.s3-us-west-1.amazonaws.com/snap-591841"), + + Filename = Dir ++ "/snap-591841", + + {ok, BinSnap} = file:read_file(Filename), + + {ok, Snapshot} = blockchain_ledger_snapshot_v1:deserialize(BinSnap), + SHA = blockchain_ledger_snapshot_v1:hash(Snapshot), + + {ok, _GWCache} = blockchain_gateway_cache:start_link(), + {ok, _Pid} = blockchain_score_cache:start_link(), + + {ok, BinGen} = file:read_file("../../../../test/genesis"), + GenesisBlock = blockchain_block:deserialize(BinGen), + {ok, Chain} = blockchain:new(NewDir, GenesisBlock, blessed_snapshot, undefined), + + {ok, Ledger1} = blockchain_ledger_snapshot_v1:import(Chain, SHA, Snapshot), + {ok, Height} = blockchain_ledger_v1:current_height(Ledger1), + + ct:pal("loaded ledger at height ~p", [Height]), + + [{chain, Chain} | Config]. + +end_per_testcase(_TestCase, _Config) -> + blockchain_score_cache:stop(), + ok. + +all() -> + []. + +reward_perf_test(Config) -> + Chain = ?config(chain, Config), + Ledger = blockchain:ledger(Chain), + + {ok, Height} = blockchain_ledger_v1:current_height(Ledger), + + {Time, _} = + timer:tc( + fun() -> + {ok, _} = blockchain_txn_rewards_v1:calculate_rewards(Height - 15, Height, Chain) + end), + ct:pal("basic calc took: ~p ms", [Time div 1000]), + + %% {Time2, _} = + %% timer:tc( + %% fun() -> + %% blockchain_txn_rewards_v1:calculate_rewards(Height - 20, Height, Chain) + %% end), + %% ct:pal("basic calc 2 took: ~p ms", [Time2 div 1000]), + + Vars = maps:merge(blockchain_reward_perf_SUITE:hip15_vars(), blockchain_reward_perf_SUITE:hip17_vars()), + Ledger1 = blockchain_ledger_v1:new_context(Ledger), + ok = blockchain_ledger_v1:vars(Vars, [], Ledger1), + blockchain:bootstrap_h3dex(Ledger1), + blockchain_ledger_v1:commit_context(Ledger1), + + {Time3, _} = + timer:tc( + fun() -> + blockchain_txn_rewards_v1:calculate_rewards(Height - 15, Height, Chain) + end), + ct:pal("hip 17 calc took: ~p ms", [Time3 div 1000]), + + error(print), + ok. + + +hip15_vars() -> + #{ + %% configured on chain + ?poc_version => 10, + ?reward_version => 5, + %% new hip15 vars for testing + ?poc_reward_decay_rate => 0.8, + ?witness_redundancy => 4 + }. + +hip17_vars() -> + #{ + ?hip17_res_0 => <<"2,100000,100000">>, + ?hip17_res_1 => <<"2,100000,100000">>, + ?hip17_res_2 => <<"2,100000,100000">>, + ?hip17_res_3 => <<"2,100000,100000">>, + ?hip17_res_4 => <<"1,250,800">>, + ?hip17_res_5 => <<"1,100,400">>, + ?hip17_res_6 => <<"1,25,100">>, + ?hip17_res_7 => <<"2,5,20">>, + ?hip17_res_8 => <<"2,1,4">>, + ?hip17_res_9 => <<"2,1,2">>, + ?hip17_res_10 => <<"2,1,1">>, + ?hip17_res_11 => <<"2,100000,100000">>, + ?hip17_res_12 => <<"2,100000,100000">>, + ?density_tgt_res => 8 + }. diff --git a/test/blockchain_snapshot_SUITE.erl b/test/blockchain_snapshot_SUITE.erl index 7186a77f8c..61535e4620 100644 --- a/test/blockchain_snapshot_SUITE.erl +++ b/test/blockchain_snapshot_SUITE.erl @@ -71,6 +71,9 @@ basic_test(_Config) -> {ok, Snapshot1} = blockchain_ledger_snapshot_v1:snapshot(Ledger1, []), ?assertEqual([], blockchain_ledger_snapshot_v1:diff(Snapshot, Snapshot1)), + + ?assertEqual(blockchain_ledger_snapshot_v1:hash(Snapshot), + blockchain_ledger_snapshot_v1:hash(Snapshot1)), ok. diff --git a/test/test_utils.erl b/test/test_utils.erl index c521a51e83..02f2c58bea 100644 --- a/test/test_utils.erl +++ b/test/test_utils.erl @@ -7,6 +7,8 @@ -export([ init/1, init/2, init_chain/2, init_chain/3, init_chain/4, + init_chain_with_fixed_locations/4, + generate_plain_keys/2, generate_keys/1, generate_keys/2, wait_until/1, wait_until/3, create_block/2, create_block/3, create_block/4, @@ -41,20 +43,23 @@ init_chain(Balance, Keys) -> init_chain(Balance, Keys, true, #{}). -init_chain(Balance, {PrivKey, PubKey}, InConsensus, ExtraVars) -> - % Generate fake blockchains (just the keys) - GenesisMembers = case InConsensus of - true -> - RandomKeys = test_utils:generate_keys(10), - Address = blockchain_swarm:pubkey_bin(), - [ - {Address, {PubKey, PrivKey, libp2p_crypto:mk_sig_fun(PrivKey)}} - ] ++ RandomKeys; - false -> - test_utils:generate_keys(11) - end, +init_chain(Balance, {_PrivKey, _PubKey}=Keys, InConsensus, ExtraVars) -> + GenesisMembers = init_genesis_members(Keys, InConsensus), init_chain(Balance, GenesisMembers, ExtraVars). +init_genesis_members({PrivKey, PubKey}, InConsensus) -> + % Generate fake blockchains (just the keys) + case InConsensus of + true -> + RandomKeys = test_utils:generate_keys(10), + Address = blockchain_swarm:pubkey_bin(), + [ + {Address, {PubKey, PrivKey, libp2p_crypto:mk_sig_fun(PrivKey)}} + ] ++ RandomKeys; + false -> + test_utils:generate_keys(11) + end. + init_chain(Balance, Keys, InConsensus) when is_tuple(Keys), is_boolean(InConsensus) -> init_chain(Balance, Keys, InConsensus, #{}); init_chain(Balance, GenesisMembers, ExtraVars) when is_list(GenesisMembers), is_map(ExtraVars) -> @@ -103,6 +108,57 @@ init_chain(Balance, GenesisMembers, ExtraVars) when is_list(GenesisMembers), is_ ?assertEqual({ok, 1}, blockchain:height(Chain)), {ok, GenesisMembers, GenesisBlock, ConsensusMembers, Keys}. +init_chain_with_fixed_locations(Balance, GenesisMembers, Locations, ExtraVars) when is_list(Locations), + is_list(GenesisMembers), + is_map(ExtraVars) -> + % Create genesis block + {InitialVars, Keys} = blockchain_ct_utils:create_vars(ExtraVars), + + GenPaymentTxs = [blockchain_txn_coinbase_v1:new(Addr, Balance) + || {Addr, _} <- GenesisMembers], + + GenSecPaymentTxs = [blockchain_txn_security_coinbase_v1:new(Addr, Balance) + || {Addr, _} <- GenesisMembers], + + Addresses = [Addr || {Addr, _} <- GenesisMembers], + + InitialGatewayTxn = [blockchain_txn_gen_gateway_v1:new(Addr, Addr, Loc, 0) + || {Addr, Loc} <- lists:zip(Addresses, Locations)], + + ConsensusMembers = lists:sublist(GenesisMembers, 7), + GenConsensusGroupTx = blockchain_txn_consensus_group_v1:new( + [Addr || {Addr, _} <- ConsensusMembers], <<"proof">>, 1, 0), + Txs = InitialVars ++ + GenPaymentTxs ++ + GenSecPaymentTxs ++ + InitialGatewayTxn ++ + [GenConsensusGroupTx], + lager:info("initial transactions: ~p", [Txs]), + + GenesisBlock = blockchain_block:new_genesis_block(Txs), + ok = blockchain_worker:integrate_genesis_block(GenesisBlock), + + Chain = blockchain_worker:blockchain(), + {ok, HeadBlock} = blockchain:head_block(Chain), + ok = test_utils:wait_until(fun() ->{ok, 1} =:= blockchain:height(Chain) end), + + ?assertEqual(blockchain_block:hash_block(GenesisBlock), blockchain_block:hash_block(HeadBlock)), + ?assertEqual({ok, GenesisBlock}, blockchain:head_block(Chain)), + ?assertEqual({ok, blockchain_block:hash_block(GenesisBlock)}, blockchain:genesis_hash(Chain)), + ?assertEqual({ok, GenesisBlock}, blockchain:genesis_block(Chain)), + ?assertEqual({ok, 1}, blockchain:height(Chain)), + {ok, GenesisBlock, ConsensusMembers, Keys}. + +generate_plain_keys(N, Type) -> + lists:foldl( + fun(_, Acc) -> + #{public := PubKey, secret := PrivKey} = libp2p_crypto:generate_keys(Type), + [{libp2p_crypto:pubkey_to_bin(PubKey), PubKey, PrivKey}|Acc] + end + ,[] + ,lists:seq(1, N) + ). + generate_keys(N) -> generate_keys(N, ecc_compact).