diff --git a/examples/tz1boot1pK9h2BVGXdyvfQSv8kd1LQM6H889.yaml b/examples/tz1boot1pK9h2BVGXdyvfQSv8kd1LQM6H889.yaml index 932b3fe2..0e94981b 100644 --- a/examples/tz1boot1pK9h2BVGXdyvfQSv8kd1LQM6H889.yaml +++ b/examples/tz1boot1pK9h2BVGXdyvfQSv8kd1LQM6H889.yaml @@ -7,11 +7,13 @@ owners_map : {tz1boot1pK9h2BVGXdyvfQSv8kd1LQM6H889: 0.3, KT1KLQbYFtFZ5mAEnfEMZaW specials_map : {KT19g4JHTd3QYuYcpKFFiwViEzQ5n6ovYx1V: 7.5} supporters_set : {KT1SenDhs9rL9fpZV3cLRXFovEBafmGqkki8} min_delegation_amt : 1000 +reactivate_zeroed : False delegator_pays_xfer_fee : True +delegator_pays_ra_fee : True rules_map: KT1MMhmTkUoHez4u58XMZL7NkpU9FWY4QLn3: KT1MMhmTkUoHez4u58XMZL7NkpU9FWY4QLn0 KT1D33n8zp1bqBkViiQtLLPLEGRW9xcqihY3: KT1MMhmTkUoHez4u58XMZL7NkpU9FWY4QLn0 KT1Ao8UXNJ9Dz71Wx3m8yzYNdnNQp2peqtM0: TOE KT1VyxJWhe9oz3v4qwTp2U6Rb17ocHGpJmW0: TOB KT19cJWfbDNXT4azVbgTBvtLMeqweuHH8W20: TOF - mindelegation: TOB \ No newline at end of file + mindelegation: TOB diff --git a/src/calc/calculate_phase0.py b/src/calc/calculate_phase0.py index 4a45d0bf..b0c517c6 100644 --- a/src/calc/calculate_phase0.py +++ b/src/calc/calculate_phase0.py @@ -33,14 +33,19 @@ def calculate(self, reward_logs=None, total_reward_amount=None): reward_logs = [] delegate_staking_balance = self.reward_provider_model.delegate_staking_balance delegators_balance_dict = self.reward_provider_model.delegator_balance_dict + # calculate how rewards will be distributed # ratio is stake/total staking balance # total of ratios must be 1 - for address, balance in delegators_balance_dict.items(): - total_delegator_balance += balance - ratio = balance / delegate_staking_balance - reward_item = RewardLog(address=address, type=reward_log.TYPE_DELEGATOR, balance=balance) + for address, delegator_info in delegators_balance_dict.items(): + + staking_balance = delegator_info["staking_balance"] + current_balance = delegator_info["current_balance"] + total_delegator_balance += staking_balance + + ratio = staking_balance / delegate_staking_balance + reward_item = RewardLog(address=address, type=reward_log.TYPE_DELEGATOR, staking_balance=staking_balance, current_balance=current_balance) reward_item.ratio = ratio reward_item.ratio0 = reward_item.ratio @@ -49,7 +54,8 @@ def calculate(self, reward_logs=None, total_reward_amount=None): reward_logs.append(reward_item) owners_rl = RewardLog(address=reward_log.TYPE_OWNERS_PARENT, type=reward_log.TYPE_OWNERS_PARENT, - balance=delegate_staking_balance - total_delegator_balance) + staking_balance=delegate_staking_balance - total_delegator_balance, + current_balance=0) owners_rl.ratio = (1 - ratio_sum) owners_rl.ratio0 = owners_rl.ratio diff --git a/src/calc/calculate_phase1.py b/src/calc/calculate_phase1.py index fc0731d8..2b67ebf2 100644 --- a/src/calc/calculate_phase1.py +++ b/src/calc/calculate_phase1.py @@ -38,15 +38,15 @@ def calculate(self, reward_data0, total_amount): # exclude requested addresses from reward list for rl0 in reward_data0: - total_balance += rl0.balance + total_balance += rl0.staking_balance if rl0.address in self.excluded_set: rl0.skip(desc=BY_CONFIGURATION, phase=self.phase) rewards.append(rl0) - total_balance_excluded += rl0.balance - elif MIN_DELEGATION_KEY in self.excluded_set and rl0.balance < self.min_delegation_amount: + total_balance_excluded += rl0.staking_balance + elif MIN_DELEGATION_KEY in self.excluded_set and rl0.staking_balance < self.min_delegation_amount: rl0.skip(desc=BY_MIN_DELEGATION, phase=self.phase) rewards.append(rl0) - total_balance_excluded += rl0.balance + total_balance_excluded += rl0.staking_balance else: # ratio will be replaced with actual ratio, read below rewards.append(rl0) @@ -55,7 +55,7 @@ def calculate(self, reward_data0, total_amount): # calculate new ratio using remaining balance for rl1 in self.filterskipped(rewards): - rl1.ratio = rl1.balance / new_total_balance + rl1.ratio = rl1.staking_balance / new_total_balance rl1.ratio1 = rl1.ratio # total reward amount needs to be diminished at the same rate total balance diminishes diff --git a/src/calc/calculate_phase2.py b/src/calc/calculate_phase2.py index ddaf347a..ea4961aa 100644 --- a/src/calc/calculate_phase2.py +++ b/src/calc/calculate_phase2.py @@ -39,16 +39,16 @@ def calculate(self, reward_data1, total_amount): # exclude requested addresses from reward list for rl1 in self.filterskipped(reward_data1): - total_balance += rl1.balance + total_balance += rl1.staking_balance if rl1.address in self.excluded_set: rl1.skip(desc=BY_CONFIGURATION, phase=self.phase) rewards.append(rl1) - total_balance_excluded += rl1.balance - elif MIN_DELEGATION_KEY in self.excluded_set and rl1.balance < self.min_delegation_amount: + total_balance_excluded += rl1.staking_balance + elif MIN_DELEGATION_KEY in self.excluded_set and rl1.staking_balance < self.min_delegation_amount: rl1.skip(desc=BY_MIN_DELEGATION, phase=self.phase) rewards.append(rl1) - total_balance_excluded += rl1.balance + total_balance_excluded += rl1.staking_balance else: # ratio2 will be replaced with actual ratio, read below rewards.append(rl1) @@ -57,7 +57,7 @@ def calculate(self, reward_data1, total_amount): # calculate new ratio using remaining balance for rl2 in self.filterskipped(rewards): - rl2.ratio = rl2.balance / new_total_balance + rl2.ratio = rl2.staking_balance / new_total_balance rl2.ratio2 = rl2.ratio # total reward amount remains the same diff --git a/src/calc/calculate_phase3.py b/src/calc/calculate_phase3.py index b7379054..9080588b 100644 --- a/src/calc/calculate_phase3.py +++ b/src/calc/calculate_phase3.py @@ -9,7 +9,7 @@ class CalculatePhase3(CalculatePhaseBase): """ -- Phase3 : Founders Phase -- - At stage 3, Founders record is created. Founders record is later on splitted into founder records, for each founder. + At stage 3, Founders record is created. Founders record is later split into founder records, for each founder. If any address is excluded at this stage, its reward is given to founders. Fee rates are set at this stage. """ @@ -37,7 +37,7 @@ def calculate(self, reward_data2, total_amount): rl2.skip(desc=BY_CONFIGURATION, phase=self.phase) new_rewards.append(rl2) total_excluded_ratio += rl2.ratio - elif MIN_DELEGATION_KEY in self.excluded_set and rl2.balance < self.min_delegation_amount: + elif MIN_DELEGATION_KEY in self.excluded_set and rl2.staking_balance < self.min_delegation_amount: rl2.skip(desc=BY_MIN_DELEGATION, phase=self.phase) new_rewards.append(rl2) total_excluded_ratio += rl2.ratio @@ -57,7 +57,7 @@ def calculate(self, reward_data2, total_amount): # create founders parent record if total_service_fee_ratio > 1e-6: # >0 - rl = RewardLog(address=TYPE_FOUNDERS_PARENT, type=TYPE_FOUNDERS_PARENT, balance=0) + rl = RewardLog(address=TYPE_FOUNDERS_PARENT, type=TYPE_FOUNDERS_PARENT, staking_balance=0, current_balance=0) rl.service_fee_rate = 0 rl.service_fee_ratio = 0 rl.ratio = total_service_fee_ratio diff --git a/src/calc/calculate_phase4.py b/src/calc/calculate_phase4.py index dd87d386..ede511ba 100644 --- a/src/calc/calculate_phase4.py +++ b/src/calc/calculate_phase4.py @@ -31,7 +31,7 @@ def calculate(self, reward_data3, total_amount): for rl3 in self.filterskipped(reward_data3): if rl3.type == TYPE_FOUNDERS_PARENT: for addr, ratio in self.founders_map.items(): - rl4 = RewardLog(addr, TYPE_FOUNDER, 0) + rl4 = RewardLog(addr, TYPE_FOUNDER, 0, 0) # new ratio is parent ratio * ratio of the founder rl4.ratio = ratio * rl3.ratio rl4.ratio4 = rl4.ratio @@ -46,7 +46,7 @@ def calculate(self, reward_data3, total_amount): elif rl3.type == TYPE_OWNERS_PARENT: for addr, ratio in self.owners_map.items(): - rl4 = RewardLog(addr, TYPE_OWNER, ratio * rl3.balance) + rl4 = RewardLog(addr, TYPE_OWNER, ratio * rl3.staking_balance, 0) # new ratio is parent ratio * ratio of the owner rl4.ratio = ratio * rl3.ratio rl4.ratio4 = rl4.ratio diff --git a/src/calc/calculate_phase5.py b/src/calc/calculate_phase5.py index e6d77335..35a767ef 100644 --- a/src/calc/calculate_phase5.py +++ b/src/calc/calculate_phase5.py @@ -25,5 +25,4 @@ def calculate(self, reward_data4, total_amount): if rl.address in self.addr_dest_dict: rl.paymentaddress = self.addr_dest_dict[rl.address] - return reward_data4, total_amount diff --git a/src/calc/calculate_phase6.py b/src/calc/calculate_phase6.py index 1754caf1..64a6b9b0 100644 --- a/src/calc/calculate_phase6.py +++ b/src/calc/calculate_phase6.py @@ -34,7 +34,7 @@ def calculate(self, reward_data5, total_amount): for addr, rl_list in payment_address_list_dict.items(): if len(rl_list) > 1: - total_balance = sum([rl.balance for rl in rl_list]) + total_balance = sum([rl.staking_balance for rl in rl_list]) total_ratio = sum([rl.ratio for rl in rl_list]) total_payment_amount = sum([rl.amount for rl in rl_list]) total_service_fee_amount = sum([rl.service_fee_amount for rl in rl_list]) diff --git a/src/calc/calculate_phase7.py b/src/calc/calculate_phase7.py new file mode 100644 index 00000000..4520d971 --- /dev/null +++ b/src/calc/calculate_phase7.py @@ -0,0 +1,39 @@ +from log_config import main_logger +from model import reward_log +from calc.calculate_phase_base import CalculatePhaseBase, BY_ZERO_BALANCE + +logger = main_logger + +class CalculatePhase7(CalculatePhaseBase): + """ + -- Phase7 : Check if current delegator balance is 0 -- + + At stage 7, check each delegate's current balance. If 0, and baker is not reactivating, + then mark payment as not-payable + """ + + def __init__(self, reactivate_zeroed) -> None: + super().__init__() + self.reactivate_zeroed = reactivate_zeroed + self.phase = 7 + + def calculate(self, reward_logs): + + reward_data7 = [] + + for delegate in reward_logs: + + # If delegate's current balance is 0, and we are NOT reactivating it, + # then mark address as being skipped with a description to be included + # in the CSV payment report + + if (delegate.type == reward_log.TYPE_DELEGATOR and delegate.current_balance == 0): + + if self.reactivate_zeroed: + delegate.needs_activation = True + else: + delegate.skip(BY_ZERO_BALANCE, self.phase) + + reward_data7.append(delegate) + + return reward_data7 diff --git a/src/calc/calculate_phase_base.py b/src/calc/calculate_phase_base.py index a9ed7b83..b46460c7 100644 --- a/src/calc/calculate_phase_base.py +++ b/src/calc/calculate_phase_base.py @@ -4,6 +4,7 @@ BY_CONFIGURATION = "Excluded by configuration" BY_MIN_DELEGATION = "Excluded by min delegation" +BY_ZERO_BALANCE = "Excluded by zero balance" class CalculatePhaseBase(ABC): diff --git a/src/calc/phased_payment_calculator.py b/src/calc/phased_payment_calculator.py index a7deb81f..73527437 100644 --- a/src/calc/phased_payment_calculator.py +++ b/src/calc/phased_payment_calculator.py @@ -49,7 +49,7 @@ def calculate(self, reward_provider_model): logger.debug("NO REWARDS to process!") return [], 0 - assert reward_provider_model.delegate_staking_balance == sum([rl.balance for rl in rwrd_logs]) + assert reward_provider_model.delegate_staking_balance == sum([rl.staking_balance for rl in rwrd_logs]) assert self.almost_equal(1, sum([rl.ratio for rl in rwrd_logs])) # calculate phase 1 @@ -86,7 +86,6 @@ def calculate(self, reward_provider_model): phase4 = CalculatePhase4(self.founders_map, self.owners_map) rwrd_logs, total_rwrd_amnt = phase4.calculate(rwrd_logs, total_rwrd_amnt) - # calculate amounts phase_last = CalculatePhaseFinal() rwrd_logs, total_rwrd_amnt = phase_last.calculate(rwrd_logs, total_rwrd_amnt) diff --git a/src/calc/test_calculatePhase0.py b/src/calc/test_calculatePhase0.py index f6e3fea8..fc5e8707 100644 --- a/src/calc/test_calculatePhase0.py +++ b/src/calc/test_calculatePhase0.py @@ -20,18 +20,19 @@ def test_calculate(self): phase0 = CalculatePhase0(model) reward_data, total_rewards = phase0.calculate() - staking_balance = int(model.delegate_staking_balance) + delegate_staking_balance = int(model.delegate_staking_balance) # total reward ratio is 1 self.assertTrue(1.0, sum(r.ratio0 for r in reward_data)) # check that ratio calculations are correct - delegators_balances = model.delegator_balance_dict + delegators_balances_dict = model.delegator_balance_dict # check ratios - for (address, balance), reward in zip(delegators_balances.items(),reward_data): + for (address, delegator_info), reward in zip(delegators_balances_dict.items(), reward_data): # ratio must be equal to stake/total staking balance - self.assertEqual(int(balance) / staking_balance, reward.ratio0) + delegator_staking_balance = int(delegator_info["staking_balance"]) + self.assertEqual(delegator_staking_balance / delegate_staking_balance, reward.ratio0) # last one is owners record self.assertTrue(reward_data[-1].type == reward_log.TYPE_OWNERS_PARENT) diff --git a/src/calc/test_calculatePhase1.py b/src/calc/test_calculatePhase1.py index fa923ca3..38c3cde8 100644 --- a/src/calc/test_calculatePhase1.py +++ b/src/calc/test_calculatePhase1.py @@ -11,7 +11,7 @@ def test_calculate(self): total_reward = 1000 for i, ratio in enumerate(ratios,start=1): - rl0 = RewardLog(address="addr" + str(i), type="D", balance=total_reward * ratio) + rl0 = RewardLog(address="addr" + str(i), type="D", staking_balance=total_reward * ratio, current_balance=0) rl0.ratio0 = ratio rewards.append(rl0) diff --git a/src/calc/test_calculatePhase2.py b/src/calc/test_calculatePhase2.py index 35a9a042..d5ec759d 100644 --- a/src/calc/test_calculatePhase2.py +++ b/src/calc/test_calculatePhase2.py @@ -11,11 +11,11 @@ def test_calculate(self): total_reward = 1000 for i, addr in enumerate(ratios, start=1): - rl0 = RewardLog(address="addr" + str(i), type="D", balance=total_reward * ratios[addr]) + rl0 = RewardLog(address="addr" + str(i), type="D", staking_balance=total_reward * ratios[addr], current_balance=0) rl0.ratio1 = ratios[addr] rewards.append(rl0) - rewards.append(RewardLog("addrdummy", "D", 0).skip("skipped for testing", 2)) + rewards.append(RewardLog("addrdummy", "D", 0, 0).skip("skipped for testing", 2)) excluded_set = {"addr1"} diff --git a/src/calc/test_calculatePhase3.py b/src/calc/test_calculatePhase3.py index 55fbb89c..da49defe 100644 --- a/src/calc/test_calculatePhase3.py +++ b/src/calc/test_calculatePhase3.py @@ -12,12 +12,12 @@ def test_calculate(self): total_reward = 1000 for i, ratio in enumerate(ratios, start=1): - rl0 = RewardLog(address="addr" + str(i), type="D", balance=total_reward * ratio) + rl0 = RewardLog(address="addr" + str(i), type="D", staking_balance=total_reward * ratio, current_balance=0) rl0.ratio = ratio rl0.ratio2 = ratio rewards.append(rl0) - rewards.append(RewardLog("addrdummy", "D", 0).skip("skipped for testing", 2)) + rewards.append(RewardLog("addrdummy", "D", 0, 0).skip("skipped for testing", 2)) excluded_set = {"addr1"} @@ -58,12 +58,12 @@ def test_calculate_sepecials(self): total_reward = 1000 for i, ratio in enumerate(ratios, start=1): - rl0 = RewardLog(address="addr" + str(i), type="D", balance=total_reward * ratio) + rl0 = RewardLog(address="addr" + str(i), type="D", staking_balance=total_reward * ratio, current_balance=0) rl0.ratio = ratio rl0.ratio2 = ratio rewards.append(rl0) - rewards.append(RewardLog("addrdummy", "D", 0).skip("skipped for testing", 2)) + rewards.append(RewardLog("addrdummy", "D", 0, 0).skip("skipped for testing", 2)) excluded_set = {"addr1"} supporters_set = {"addr2"} diff --git a/src/calc/test_calculatePhase4.py b/src/calc/test_calculatePhase4.py index 0fcbcab9..22419099 100644 --- a/src/calc/test_calculatePhase4.py +++ b/src/calc/test_calculatePhase4.py @@ -11,7 +11,7 @@ def test_calculate(self): total_reward = 1000 for i, ratio in enumerate(ratios, start=1): - rl0 = RewardLog(address="addr" + str(i), type="D", balance=total_reward * ratio) + rl0 = RewardLog(address="addr" + str(i), type="D", staking_balance=total_reward * ratio, current_balance=0) rl0.ratio = ratio rl0.ratio3 = ratio rewards.append(rl0) @@ -19,13 +19,12 @@ def test_calculate(self): rewards[0].type = TYPE_OWNERS_PARENT rewards[1].type = TYPE_FOUNDERS_PARENT - rewards.append(RewardLog("addrdummy", "D", 0).skip("skipped for testing", 3)) + rewards.append(RewardLog("addrdummy", "D", 0, 0).skip("skipped for testing", 3)) founders_map = {"addr1": 0.4, "addr2": 0.6} owners_map = {"addr1": 0.6, "addr2": 0.4} phase4 = CalculatePhase4(founders_map, owners_map) - new_rewards, new_total_reward = phase4.calculate(rewards, total_reward) # new_total_reward = total_reward diff --git a/src/calc/test_calculatePhase5.py b/src/calc/test_calculatePhase5.py index 783f9a34..5c9af4d5 100644 --- a/src/calc/test_calculatePhase5.py +++ b/src/calc/test_calculatePhase5.py @@ -11,12 +11,12 @@ def test_calculate(self): total_reward = 1000 for i, ratio in enumerate(ratios, start=1): - rl0 = RewardLog(address="addr" + str(i), type="D", balance=total_reward * ratio) + rl0 = RewardLog(address="addr" + str(i), type="D", staking_balance=total_reward * ratio, current_balance=0) rl0.ratio = ratio rl0.ratio4 = ratio rewards.append(rl0) - rewards.append(RewardLog("addrdummy","D",0).skip("skipped for testing",4)) + rewards.append(RewardLog("addrdummy", "D" , 0, 0).skip("skipped for testing",4)) phase5 = CalculatePhase5({"addr2":"addr1"}) diff --git a/src/cli/cmd_manager.py b/src/cli/cmd_manager.py index 7a88a32a..94efb937 100644 --- a/src/cli/cmd_manager.py +++ b/src/cli/cmd_manager.py @@ -1,5 +1,5 @@ +import os from subprocess import STDOUT, check_output, TimeoutExpired, CalledProcessError - from log_config import main_logger from util.client_utils import clear_terminal_chars @@ -26,6 +26,7 @@ def execute(self, cmd, verbose_override=None, timeout=None): logger.debug("--> Verbose : Command is |{}|".format(cmd)) try: + os.environ["TEZOS_CLIENT_UNSAFE_DISABLE_DISCLAIMER"] = "Y" output = check_output(cmd, shell=True, stderr=STDOUT, timeout=timeout, encoding='utf8') except TimeoutExpired as e: logger.info("Command timed out") diff --git a/src/configure.py b/src/configure.py index 3e27e79a..0c54df20 100644 --- a/src/configure.py +++ b/src/configure.py @@ -21,7 +21,8 @@ add_argument_verbose, add_argument_dry, add_argument_provider from log_config import main_logger from model.baking_conf import BakingConf, BAKING_ADDRESS, PAYMENT_ADDRESS, SERVICE_FEE, FOUNDERS_MAP, OWNERS_MAP, \ - MIN_DELEGATION_AMT, RULES_MAP, MIN_DELEGATION_KEY, DELEGATOR_PAYS_XFER_FEE, SPECIALS_MAP, SUPPORTERS_SET + MIN_DELEGATION_AMT, RULES_MAP, MIN_DELEGATION_KEY, DELEGATOR_PAYS_XFER_FEE, DELEGATOR_PAYS_RA_FEE, \ + REACTIVATE_ZEROED, SPECIALS_MAP, SUPPORTERS_SET from util.address_validator import AddressValidator from util.client_utils import get_client_path from util.dir_utils import get_payment_root, \ @@ -43,7 +44,9 @@ , 'mindelegationtarget' : "Specify where should shares of delegators failing to satisfy minimum delegation amount go. TOB: leave at balance, TOF: to founders, TOE: to everybody, default is TOB" , 'exclude' : "Add excluded address in form of PKH,target. Share of the exluded address will go to target. Possbile targets are= TOB: leave at balance, TOF: to founders, TOE: to everybody. Type enter to skip" , 'redirect' : "Add redirected address in form of PKH1,PKH2. Payments for PKH1 will go to PKH2. Type enter to skip" - , 'delegatorpays' : "Who is going to pay for transfer fees: 0 for delegator, 1 for delegate. Type enter for delegator" + , 'reactivatezeroed' : "If a destination address has 0 balance, should burn fee be paid to reactivate? 1 for Yes, 0 for No. Type enter for Yes" + , 'delegatorpaysxfrfee' : "Who is going to pay for transfer fees: 0 for delegator, 1 for delegate. Type enter for delegator" + , 'delegatorpaysrafee' : "Who is going to pay for 0 balance reactivation/burn fee: 0 for delegator, 1 for delegate. Type enter for delegator" , 'supporters' : "Add supporter address. Supporters do not pay service fee. Type enter to skip" , 'specials' : "Add special fee in form of PKH,fee. Given addresses will pay the specified fee rate. Type enter to skip" , 'final' : "Add excluded address in form of PKH,target. Possbile targets are= TOB: leave at balance, TOF: to founders, TOE: to everybody. Type enter to skip" @@ -253,7 +256,7 @@ def onredirect(input): printe("Invalid redirection entry: " + traceback.format_exc()) return -def ondelegatorpays(input): +def ondelegatorpaysxfrfee(input): try: if not input: input="0" @@ -264,6 +267,28 @@ def ondelegatorpays(input): return fsm.go() +def ondelegatorpaysrafee(input): + try: + if not input: + input="1" + global parser + parser.set(DELEGATOR_PAYS_RA_FEE, input!="1") + except: + printe("Invalid input: " + traceback.format_exc()) + return + fsm.go() + +def onreactivatezeroed(input): + try: + if not input: + input="1" + global parser + parser.set(REACTIVATE_ZEROED, input!="1") + except: + printe("Invalid input: " + traceback.format_exc()) + return + fsm.go() + def onfinal(input): pass @@ -276,7 +301,9 @@ def onfinal(input): ,'mindelegationtarget':onmindelegationtarget ,'exclude':onexclude ,'redirect':onredirect - ,'delegatorpays':ondelegatorpays + ,'reactivatezeroed':onreactivatezeroed + ,'delegatorpaysxfrfee':ondelegatorpaysxfrfee + ,'delegatorpaysrafee':ondelegatorpaysrafee ,'supporters':onsupporters ,'specials':onspecials ,'final':onfinal @@ -293,8 +320,10 @@ def onfinal(input): {'name': 'go', 'src': 'mindelegation', 'dst': 'mindelegationtarget'}, {'name': 'go', 'src': 'mindelegationtarget', 'dst': 'exclude'}, {'name': 'go', 'src': 'exclude', 'dst': 'redirect'}, - {'name': 'go', 'src': 'redirect', 'dst': 'delegatorpays'}, - {'name': 'go', 'src': 'delegatorpays', 'dst': 'specials'}, + {'name': 'go', 'src': 'redirect', 'dst': 'reactivatezeroed'}, + {'name': 'go', 'src': 'reactivatezeroed', 'dst': 'delegatorpaysrafee'}, + {'name': 'go', 'src': 'delegatorpaysrafee', 'dst': 'delegatorpaysxfrfee'}, + {'name': 'go', 'src': 'delegatorpaysxfrfee', 'dst': 'specials'}, {'name': 'go', 'src': 'specials', 'dst': 'supporters'}, {'name': 'go', 'src': 'supporters', 'dst': 'final'} ], 'callbacks': { diff --git a/src/main.py b/src/main.py index ca9d8a46..db1cf380 100644 --- a/src/main.py +++ b/src/main.py @@ -154,6 +154,8 @@ def main(args): c = PaymentConsumer(name='consumer' + str(i), payments_dir=payments_root, key_name=payment_address, client_path=client_path, payments_queue=payments_queue, node_addr=args.node_addr, wllt_clnt_mngr=wllt_clnt_mngr, args=args, verbose=args.verbose, dry_run=dry_run, + reactivate_zeroed=cfg.get_reactivate_zeroed(), + delegator_pays_ra_fee=cfg.get_delegator_pays_ra_fee(), delegator_pays_xfer_fee=cfg.get_delegator_pays_xfer_fee(), dest_map=cfg.get_dest_map(), network_config=network_config,publish_stats=publish_stats) time.sleep(1) diff --git a/src/model/baking_conf.py b/src/model/baking_conf.py index e8dc15d6..9b615516 100644 --- a/src/model/baking_conf.py +++ b/src/model/baking_conf.py @@ -9,7 +9,9 @@ SUPPORTERS_SET = 'supporters_set' PAYMENT_ADDRESS = 'payment_address' MIN_DELEGATION_AMT = 'min_delegation_amt' +REACTIVATE_ZEROED = 'reactivate_zeroed' DELEGATOR_PAYS_XFER_FEE = 'delegator_pays_xfer_fee' +DELEGATOR_PAYS_RA_FEE = 'delegator_pays_ra_fee' ### extensions FULL_SUPPORTERS_SET = "__full_supporters_set" EXCLUDED_DELEGATORS_SET_TOB = "__excluded_delegators_set_tob" @@ -69,9 +71,15 @@ def get_full_supporters_set(self): def get_min_delegation_amount(self): return self.get_attribute(MIN_DELEGATION_AMT) + def get_reactivate_zeroed(self): + return self.get_attribute(REACTIVATE_ZEROED) + def get_delegator_pays_xfer_fee(self): return self.get_attribute(DELEGATOR_PAYS_XFER_FEE) + def get_delegator_pays_ra_fee(self): + return self.get_attribute(DELEGATOR_PAYS_RA_FEE) + def get_rule_map(self): return self.get_attribute(RULES_MAP) diff --git a/src/model/reward_log.py b/src/model/reward_log.py index cf18639a..92e6e623 100644 --- a/src/model/reward_log.py +++ b/src/model/reward_log.py @@ -22,11 +22,13 @@ class RunMode(Enum): class RewardLog: - def __init__(self, address, type, balance) -> None: + def __init__(self, address, type, staking_balance, current_balance) -> None: super().__init__() - self.balance = balance + self.staking_balance = staking_balance + self.current_balance = current_balance self.address = address self.paymentaddress = address + self.needs_activation = False self.type = type self.desc = "" self.skipped = False @@ -61,11 +63,12 @@ def skip(self, desc, phase): return self def __repr__(self) -> str: - return "address: %s, type: %s, balance: %s, disabled:%s" % (self.address, self.type, self.balance, self.skipped) + return "Address: {}, Type: {}, SB: {}, CB: {}, Skipped: {}, NA: {}".format( + self.address, self.type, self.staking_balance, self.current_balance, self.skipped, self.needs_activation) @staticmethod def ExitInstance(): - return RewardLog(address=EXIT_PAYMENT_TYPE, type=EXIT_PAYMENT_TYPE, balance=0) + return RewardLog(address=EXIT_PAYMENT_TYPE, type=EXIT_PAYMENT_TYPE, staking_balance=0, current_balance=0) @staticmethod def ExternalInstance(file_name, address, amount): @@ -80,14 +83,14 @@ def cmp_by_skip_type_balance(rl1, rl2): TYPE_MERGED: 0} if rl1.skipped == rl2.skipped: if rl1.type == rl2.type: - if rl1.balance is None: + if rl1.staking_balance is None: return 1 - if rl2.balance is None: + if rl2.staking_balance is None: return -1 - if rl1.balance == rl2.balance: + if rl1.staking_balance == rl2.staking_balance: return 1 else: - return rl2.balance - rl1.balance + return rl2.staking_balance - rl1.staking_balance else: return types[rl2.type] - types[rl1.type] else: @@ -102,13 +105,13 @@ def cmp_by_type_balance(rl1, rl2): TYPE_MERGED: 0} if rl1.type == rl2.type: - if rl1.balance is None: + if rl1.staking_balance is None: return 1 - if rl2.balance is None: + if rl2.staking_balance is None: return -1 - if rl1.balance == rl2.balance: + if rl1.staking_balance == rl2.staking_balance: return 1 else: - return rl2.balance - rl1.balance + return rl2.staking_balance - rl1.staking_balance else: return types[rl2.type] - types[rl1.type] diff --git a/src/pay/batch_payer.py b/src/pay/batch_payer.py index 4475999b..6808bca7 100644 --- a/src/pay/batch_payer.py +++ b/src/pay/batch_payer.py @@ -20,7 +20,7 @@ COMM_HEAD = "rpc get /chains/main/blocks/head" COMM_COUNTER = "rpc get /chains/main/blocks/head/context/contracts/{}/counter" -CONTENT = '{"kind":"transaction","source":"%SOURCE%","destination":"%DESTINATION%","fee":"%fee%","counter":"%COUNTER%","gas_limit": "%gas_limit%", "storage_limit": "%storage_limit%","amount":"%AMOUNT%"}' +CONTENT = '{"kind":"transaction","source":"%SOURCE%","destination":"%DESTINATION%","fee":"%fee%","counter":"%COUNTER%","gas_limit":"%gas_limit%","storage_limit":"%storage_limit%","amount":"%AMOUNT%"}' FORGE_JSON = '{"branch": "%BRANCH%","contents":[%CONTENT%]}' RUNOPS_JSON = '{"branch": "%BRANCH%","contents":[%CONTENT%], "signature":"edsigtXomBKi5CTRf5cjATJWSyaRvhfYNHqSUGrn4SdbYRcGwQrUGjzEfQDTuqHhuA8b2d8NarZjz8TRf65WkpQmo423BtomS8Q"}' PREAPPLY_JSON = '[{"protocol":"%PROTOCOL%","branch":"%BRANCH%","contents":[%CONTENT%],"signature":"%SIGNATURE%"}]' @@ -33,9 +33,11 @@ FEE_INI = 'fee.ini' MUTEZ = 1e6 +RA_BURN_FEE = 257000 # 0.257 XTZ +RA_STORAGE = 300 class BatchPayer(): - def __init__(self, node_url, pymnt_addr, wllt_clnt_mngr, delegator_pays_xfer_fee, network_config): + def __init__(self, node_url, pymnt_addr, wllt_clnt_mngr, delegator_pays_ra_fee, delegator_pays_xfer_fee, network_config): super(BatchPayer, self).__init__() self.pymnt_addr = pymnt_addr self.node_url = node_url @@ -51,7 +53,7 @@ def __init__(self, node_url, pymnt_addr, wllt_clnt_mngr, delegator_pays_xfer_fee kttx = config['KTTX'] self.gas_limit = kttx['gas_limit'] - self.storage_limit = kttx['storage_limit'] + self.storage_limit = int(kttx['storage_limit']) self.default_fee = int(kttx['fee']) # section below is left to make sure no one using legacy configuration option @@ -62,15 +64,17 @@ def __init__(self, node_url, pymnt_addr, wllt_clnt_mngr, delegator_pays_xfer_fee raise Exception( "delegator_pays_xfer_fee is no longer read from fee.ini. It should be set in baking configuration file.") + self.delegator_pays_ra_fee = delegator_pays_ra_fee self.delegator_pays_xfer_fee = delegator_pays_xfer_fee # If delegator pays the fee, then the cutoff should be transaction-fee + 1 - # Ex: Delegator reward is 1800 mutez, txn fee is 1792 mutez, reward - fee = 8 mutez payable reward + # Ex: Delegator reward is 1800 mutez, txn fee is 1792 mutez, reward - txn fee = 8 mutez payable reward # If delegate pays fee, then cutoff is 1 mutez payable reward if self.delegator_pays_xfer_fee: - self.zero_threshold = self.default_fee + 1 + self.zero_threshold += self.default_fee logger.info("Transfer fee is {:.6f} XTZ and is paid by {}".format(self.default_fee/MUTEZ, "Delegator" if self.delegator_pays_xfer_fee else "Delegate")) + logger.info("Reactivation fee is {:.6f} XTZ and is paid by {}".format(RA_BURN_FEE/MUTEZ, "Delegator" if self.delegator_pays_ra_fee else "Delegate")) logger.info("Payment amount cutoff is {:.6f} XTZ".format(self.zero_threshold/MUTEZ)) # pymnt_addr has a length of 36 and starts with tz or KT then it is a public key has, else it is an alias @@ -125,8 +129,20 @@ def pay(self, payment_items_in, verbose=None, dry_run=None): # all unprocessed_payment_items are important (non-trivial) # gather up all unprocessed_payment_items that are greater than, or equal to the zero_threshold - # zero_threshold is either 1 mutez or the txn fee if delegator is not paying it - payment_items = [pi for pi in unprocessed_payment_items if pi.amount >= self.zero_threshold] + # zero_threshold is either 1 mutez or the txn fee if delegator is not paying it, and burn fee + payment_items = [] + sum_burn_fees = 0 + for pi in unprocessed_payment_items: + + zt = self.zero_threshold + if pi.needs_activation and self.delegator_pays_ra_fee: + # Need to apply this fee to only those which need reactivation + zt += RA_BURN_FEE + sum_burn_fees += RA_BURN_FEE + + if pi.amount >= zt: + payment_items.append(pi) + if not payment_items: logger.info("No payment items found, returning...") return payment_items_in, 0 @@ -136,6 +152,7 @@ def pay(self, payment_items_in, verbose=None, dry_run=None): payment_items_chunks = [payment_items[i:i + MAX_TX_PER_BLOCK] for i in range(0, len(payment_items), MAX_TX_PER_BLOCK)] total_amount_to_pay = sum([pl.amount for pl in payment_items]) + total_amount_to_pay += sum_burn_fees if not self.delegator_pays_xfer_fee: total_amount_to_pay += self.default_fee * len(payment_items) logger.info("Total amount to pay out is {:,} mutez.".format(total_amount_to_pay)) @@ -227,20 +244,28 @@ def attempt_single_batch(self, payment_records, op_counter, verbose=None, dry_ru content_list = [] for payment_item in payment_records: + + storage = self.storage_limit pymnt_amnt = payment_item.amount # expects in micro tezos + if payment_item.needs_activation: + storage += RA_STORAGE + if self.delegator_pays_ra_fee: + pymnt_amnt = max(pymnt_amnt - RA_BURN_FEE, 0) # ensure not less than 0 + if self.delegator_pays_xfer_fee: pymnt_amnt = max(pymnt_amnt - self.default_fee, 0) # ensure not less than 0 - # if, somehow, pymnt_amnt becomes 0, don't pay + # if pymnt_amnt becomes 0, don't pay if pymnt_amnt == 0: + logger.debug("Payment to {} became 0 after deducting fees. Skipping.".format(payment_item.paymentaddress)) continue op_counter.inc() content = CONTENT.replace("%SOURCE%", self.source).replace("%DESTINATION%", payment_item.paymentaddress) \ .replace("%AMOUNT%", str(pymnt_amnt)).replace("%COUNTER%", str(op_counter.get())) \ - .replace("%fee%", str(self.default_fee)).replace("%gas_limit%", self.gas_limit).replace("%storage_limit%", self.storage_limit) + .replace("%fee%", str(self.default_fee)).replace("%gas_limit%", self.gas_limit).replace("%storage_limit%", str(storage)) content_list.append(content) diff --git a/src/pay/payment_consumer.py b/src/pay/payment_consumer.py index 1e8e31c9..93a77be7 100644 --- a/src/pay/payment_consumer.py +++ b/src/pay/payment_consumer.py @@ -8,6 +8,7 @@ from NetworkConfiguration import is_mainnet from calc.calculate_phase5 import CalculatePhase5 from calc.calculate_phase6 import CalculatePhase6 +from calc.calculate_phase7 import CalculatePhase7 from emails.email_manager import EmailManager from log_config import main_logger from model.reward_log import cmp_by_type_balance, TYPE_MERGED, TYPE_FOUNDER, TYPE_OWNER, TYPE_DELEGATOR @@ -33,7 +34,8 @@ def count_and_log_failed(payment_logs): class PaymentConsumer(threading.Thread): def __init__(self, name, payments_dir, key_name, client_path, payments_queue, node_addr, wllt_clnt_mngr, - network_config, args=None, verbose=None, dry_run=None, delegator_pays_xfer_fee=True, dest_map=None, + network_config, args=None, verbose=None, dry_run=None, reactivate_zeroed=True, + delegator_pays_ra_fee=True, delegator_pays_xfer_fee=True, dest_map=None, publish_stats=True): super(PaymentConsumer, self).__init__() @@ -48,7 +50,9 @@ def __init__(self, name, payments_dir, key_name, client_path, payments_queue, no self.dry_run = dry_run self.mm = EmailManager() self.wllt_clnt_mngr = wllt_clnt_mngr + self.reactivate_zeroed = reactivate_zeroed self.delegator_pays_xfer_fee = delegator_pays_xfer_fee + self.delegator_pays_ra_fee = delegator_pays_ra_fee self.publish_stats = publish_stats self.args = args self.network_config = network_config @@ -60,7 +64,7 @@ def __init__(self, name, payments_dir, key_name, client_path, payments_queue, no def run(self): while True: try: - # 1- wait until a reward is present + # 1 - wait until a reward is present payment_batch = self.payments_queue.get(True) payment_items = payment_batch.batch @@ -79,19 +83,26 @@ def run(self): logger.info("Starting payments for cycle {}".format(pymnt_cycle)) + # Handle remapping of payment to alternate address phase5 = CalculatePhase5(self.dest_map) payment_items, _ = phase5.calculate(payment_items, None) + # Merge payments to same address phase6 = CalculatePhase6(addr_dest_dict=self.dest_map) payment_items, _ = phase6.calculate(payment_items, None) - # filter out non-payable items + # Filter zero-balance addresses based on config + phase7 = CalculatePhase7(self.reactivate_zeroed) + payment_items = phase7.calculate(payment_items) + + # Filter out non-payable items payment_items = [pi for pi in payment_items if pi.payable] payment_items.sort(key=functools.cmp_to_key(cmp_by_type_balance)) batch_payer = BatchPayer(self.node_addr, self.key_name, self.wllt_clnt_mngr, - self.delegator_pays_xfer_fee, self.network_config) + self.delegator_pays_ra_fee, self.delegator_pays_xfer_fee, + self.network_config) # 3- do the payment payment_logs, total_attempts = batch_payer.pay(payment_items, self.verbose, dry_run=self.dry_run) diff --git a/src/pay/payment_producer.py b/src/pay/payment_producer.py index c8b5f1c7..8988980a 100644 --- a/src/pay/payment_producer.py +++ b/src/pay/payment_producer.py @@ -223,8 +223,10 @@ def try_to_pay(self, pymnt_cycle, expected_reward = False): reward_model = self.reward_api.get_rewards_for_cycle_map(pymnt_cycle, expected_reward) else: reward_model = self.reward_api.get_rewards_for_cycle_map(pymnt_cycle) + # 2- calculate rewards reward_logs, total_amount = self.payment_calc.calculate(reward_model) + # set cycle info for rl in reward_logs: rl.cycle = pymnt_cycle total_amount_to_pay = sum([rl.amount for rl in reward_logs if rl.payable]) @@ -235,6 +237,7 @@ def try_to_pay(self, pymnt_cycle, expected_reward = False): # 5- send to payment consumer self.payments_queue.put(PaymentBatch(self, pymnt_cycle, reward_logs)) + # logger.info("Total payment amount is {:,} mutez. %s".format(total_amount_to_pay), # "" if self.delegator_pays_xfer_fee else "(Transfer fee is not included)") @@ -244,10 +247,9 @@ def try_to_pay(self, pymnt_cycle, expected_reward = False): # 6- create calculations report file. This file contains calculations details self.create_calculations_report(reward_logs, report_file_path, total_amount) - # 7- next cycle - # processing of cycle is done - logger.info( - "Reward creation is done for cycle {}, created {} rewards.".format(pymnt_cycle, len(reward_logs))) + + # 7- processing of cycle is done + logger.info("Reward creation is done for cycle {}, created {} rewards.".format(pymnt_cycle, len(reward_logs))) elif total_amount_to_pay == 0: logger.info("Total payment amount is 0. Nothing to pay!") @@ -285,10 +287,11 @@ def create_calculations_report(self, payment_logs, report_file_path, total_rewar writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) # write headers and total rewards writer.writerow( - ["address", "type", "balance", "ratio", "fee_ratio", "amount", "fee_amount", "fee_rate", "payable", + ["address", "type", "staked_balance", "current_balance", "ratio", "fee_ratio", "amount", "fee_amount", "fee_rate", "payable", "skipped", "atphase", "desc", "payment_address"]) - writer.writerow([self.baking_address, "B", sum([pl.balance for pl in payment_logs]), + writer.writerow([self.baking_address, "B", sum([pl.staking_balance for pl in payment_logs]), + "{0:f}".format(1.0), "{0:f}".format(1.0), "{0:f}".format(0.0), "{0:f}".format(total_rewards), @@ -300,7 +303,7 @@ def create_calculations_report(self, payment_logs, report_file_path, total_rewar for pymnt_log in payment_logs: # write row to csv file - array = [pymnt_log.address, pymnt_log.type, pymnt_log.balance, + array = [pymnt_log.address, pymnt_log.type, pymnt_log.staking_balance, pymnt_log.current_balance, "{0:.10f}".format(pymnt_log.ratio), "{0:.10f}".format(pymnt_log.service_fee_ratio), "{0:f}".format(pymnt_log.amount), @@ -312,12 +315,15 @@ def create_calculations_report(self, payment_logs, report_file_path, total_rewar pymnt_log.paymentaddress] writer.writerow(array) - logger.debug("Reward created for address %s type %s balance {:>10.2f} ratio {:.6f} fee_ratio {:.6f} " - "amount {:>10.6f} fee_amount {:>4.6f} fee_rate {:.2f} payable %s skipped %s atphase %s desc %s pay_addr %s" - .format(pymnt_log.balance / MUTEZ, pymnt_log.ratio, pymnt_log.service_fee_ratio, + logger.debug("Reward created for %s type: %s, stake bal: {:>10.2f}, cur bal: {:>10.2f}, ratio: {:.6f}, fee_ratio: {:.6f}, " + "amount: {:>10.6f}, fee_amount: {:>4.6f}, fee_rate: {:.2f}, payable: %s, skipped: %s, at-phase: %s, " + "desc: %s, pay_addr: %s" + .format(pymnt_log.staking_balance / MUTEZ, pymnt_log.current_balance / MUTEZ, + pymnt_log.ratio, pymnt_log.service_fee_ratio, pymnt_log.amount / MUTEZ, pymnt_log.service_fee_amount / MUTEZ, pymnt_log.service_fee_rate), pymnt_log.address, pymnt_log.type, pymnt_log.payable, pymnt_log.skipped, pymnt_log.skippedatphase, pymnt_log.desc, pymnt_log.paymentaddress) + logger.info("Calculation report is created at '{}'".format(report_file_path)) @staticmethod diff --git a/src/pay_for.py b/src/pay_for.py index 25a5f88f..696b046c 100644 --- a/src/pay_for.py +++ b/src/pay_for.py @@ -122,7 +122,7 @@ def main(args): key_name=args.paymentaddress, client_path=client_path, payments_queue=payments_queue, node_addr=args.node_addr, wllt_clnt_mngr=wllt_clnt_mngr, verbose=args.verbose, dry_run=dry_run, - delegator_pays_xfer_fee=False) + reactivate_zeroed=False, delegator_pays_ra_fee=False, delegator_pays_xfer_fee=False) time.sleep(1) c.start() diff --git a/src/rpc/rpc_reward_api.py b/src/rpc/rpc_reward_api.py index 2085fee1..b89d86d5 100644 --- a/src/rpc/rpc_reward_api.py +++ b/src/rpc/rpc_reward_api.py @@ -8,13 +8,13 @@ logger = main_logger +COMM_HEAD = "{}/chains/main/blocks/head" +COMM_DELEGATES = "{}/chains/main/blocks/{}/context/delegates/{}" +COMM_BLOCK = "{}/chains/main/blocks/{}" +COMM_SNAPSHOT = COMM_BLOCK + "/context/raw/json/cycle/{}/roll_snapshot" +COMM_DELEGATE_BALANCE = "{}/chains/main/blocks/{}/context/contracts/{}/balance" class RpcRewardApiImpl(RewardApi): - COMM_HEAD = "{}/chains/main/blocks/head" - COMM_DELEGATES = "{}/chains/main/blocks/{}/context/delegates/{}" - COMM_BLOCK = "{}/chains/main/blocks/{}" - COMM_SNAPSHOT = COMM_BLOCK + "/context/raw/json/cycle/{}/roll_snapshot" - COMM_DELEGATE_BALANCE = "{}/chains/main/blocks/{}/context/contracts/{}/balance" def __init__(self, nw, baking_address, node_url, verbose=True): super(RpcRewardApiImpl, self).__init__() @@ -30,24 +30,13 @@ def __init__(self, nw, baking_address, node_url, verbose=True): self.verbose = verbose - # replace protocol placeholder - self.COMM_HEAD = self.COMM_HEAD - self.COMM_DELEGATES = self.COMM_DELEGATES - self.COMM_BLOCK = self.COMM_BLOCK - self.COMM_SNAPSHOT = self.COMM_SNAPSHOT - self.COMM_DELEGATE_BALANCE = self.COMM_DELEGATE_BALANCE - - def get_nb_delegators(self, cycle, current_level): - _, delegators = self.__get_delegators_and_delgators_balance(cycle, current_level) - return len(delegators) - def get_rewards_for_cycle_map(self, cycle): current_level, current_cycle = self.__get_current_level() logger.debug("Current level {}, current cycle {}".format(current_level, current_cycle)) reward_data = {} reward_data["delegate_staking_balance"], reward_data[ - "delegators"] = self.__get_delegators_and_delgators_balance(cycle, current_level) + "delegators"] = self.__get_delegators_and_delgators_balances(cycle, current_level) reward_data["delegators_nb"] = len(reward_data["delegators"]) # Get last block in cycle where rewards are unfrozen @@ -69,14 +58,14 @@ def get_rewards_for_cycle_map(self, cycle): reward_model = RewardProviderModel(reward_data["delegate_staking_balance"], reward_data["total_rewards"], reward_data["delegators"]) - logger.debug("delegate_staking_balance={}, total_rewards = {}".format(reward_data["delegate_staking_balance"], + logger.debug("delegate_staking_balance = {}, total_rewards = {}".format(reward_data["delegate_staking_balance"], reward_data["total_rewards"])) logger.debug("delegators = {}".format(reward_data["delegators"])) return reward_model def __get_unfrozen_rewards(self, level_of_last_block_in_unfreeze_cycle, cycle): - request_metadata = self.COMM_BLOCK.format(self.node_url, level_of_last_block_in_unfreeze_cycle) + '/metadata' + request_metadata = COMM_BLOCK.format(self.node_url, level_of_last_block_in_unfreeze_cycle) + '/metadata' metadata = self.do_rpc_request(request_metadata) balance_updates = metadata["balance_updates"] unfrozen_rewards = unfrozen_fees = 0 @@ -121,49 +110,84 @@ def do_rpc_request(self, request, time_out=120): return response def __get_current_level(self): - head = self.do_rpc_request(self.COMM_HEAD.format(self.node_url)) + head = self.do_rpc_request(COMM_HEAD.format(self.node_url)) current_level = int(head["metadata"]["level"]["level"]) current_cycle = int(head["metadata"]["level"]["cycle"]) # head_hash = head["hash"] return current_level, current_cycle + + def __get_delegators_and_delgators_balances(self, cycle, current_level): - def __get_delegators_and_delgators_balance(self, cycle, current_level): - + # calculate the hash of the block for the chosen snapshot of the rewards cycle hash_snapshot_block = self.__get_snapshot_block_hash(cycle, current_level) if hash_snapshot_block == "": return 0, [] - request = self.COMM_DELEGATES.format(self.node_url, hash_snapshot_block, self.baking_address) + # construct RPC for getting list of delegates + get_delegates_request = COMM_DELEGATES.format(self.node_url, hash_snapshot_block, self.baking_address) delegate_staking_balance = 0 delegators = {} try: - response = self.do_rpc_request(request) + # get RPC response for delegates and staking balance + response = self.do_rpc_request(get_delegates_request) delegate_staking_balance = int(response["staking_balance"]) + # loop over delegates; get snapshot balance, and current balance delegators_addresses = response["delegated_contracts"] + d_a_len = len(delegators_addresses) + + if d_a_len == 0: + raise RpcRewardApiError("No delegators found") + + # Loop over delegators, get balances for idx, delegator in enumerate(delegators_addresses): - request = self.COMM_DELEGATE_BALANCE.format(self.node_url, hash_snapshot_block, delegator) - sleep(0.5) # be nice to public node service + # create new dictionary for each delegator + d_info = {"staking_balance": 0, "current_balance": 0} + + get_staking_balance_request = COMM_DELEGATE_BALANCE.format(self.node_url, hash_snapshot_block, delegator) + get_current_balance_request = COMM_DELEGATE_BALANCE.format(self.node_url, "head", delegator) - response = None + staking_balance_response = None + current_balance_response = None - while not response: + while not staking_balance_response: try: - response = self.do_rpc_request(request, time_out=5) + staking_balance_response = self.do_rpc_request(get_staking_balance_request, time_out=5) except: - logger.error("Fetching delegator info failed {}, will retry", delegator) + logger.debug("Fetching delegator staking balance failed {}, will retry", delegator) - delegators[delegator] = int(response) + d_info["staking_balance"] = int(staking_balance_response) + + sleep(0.4) # Be nice to public RPC since we are now making 2x the amount of RPC calls + + while not current_balance_response: + try: + current_balance_response = self.do_rpc_request(get_current_balance_request, time_out=5) + except: + logger.debug("Fetching delegator current balance failed {}, will retry", delegator) + + d_info["current_balance"] = int(current_balance_response) logger.debug( - "Delegator info ({}/{}) fetched: address {}, balance {}".format(idx, len(delegators_addresses), - delegator, delegators[delegator])) - except: - logger.warn('No delegators or unexpected error', exc_info=True) + "Delegator info ({}/{}) fetched: address {}, staked balance {}, current balance {} ".format( + idx+1, d_a_len, delegator, d_info["staking_balance"], d_info["current_balance"])) + + # "append" to master dict + delegators[delegator] = d_info + + # Sanity check. We should have fetched info for all delegates. If we didn't, something went wrong + d_len = len(delegators) + if d_a_len != d_len: + raise RpcRewardApiError("Did not collect info for all delegators, {}/{}".format(d_a_len, d_len)) + + except RpcRewardApiError as r: + logger.warn("RPC API Error: {}".format(r), exc_info=True) + except Exception as e: + logger.warn("Unexpected error: {}".format(e), exc_info=True) return delegate_staking_balance, delegators @@ -175,7 +199,7 @@ def __get_snapshot_block_hash(self, cycle, current_level): block_level = cycle * self.blocks_per_cycle + 1 if current_level - snapshot_level >= 0: - request = self.COMM_SNAPSHOT.format(self.node_url, block_level, cycle) + request = COMM_SNAPSHOT.format(self.node_url, block_level, cycle) chosen_snapshot = self.do_rpc_request(request) level_snapshot_block = (cycle - self.preserved_cycles - 2) * self.blocks_per_cycle + ( @@ -185,3 +209,6 @@ def __get_snapshot_block_hash(self, cycle, current_level): else: logger.info("Cycle too far in the future") return "" + +class RpcRewardApiError(Exception): + pass diff --git a/src/tzstats/tzstats_api_constants.py b/src/tzstats/tzstats_api_constants.py index 74a33beb..3afa0a93 100644 --- a/src/tzstats/tzstats_api_constants.py +++ b/src/tzstats/tzstats_api_constants.py @@ -1,4 +1,4 @@ -# Income +# Income/Rewards Breakdown idx_income_expected_income = 19 idx_income_total_income = 21 idx_income_total_bonds = 22 @@ -20,9 +20,12 @@ idx_income_lost_revelation_fees = 36 idx_income_lost_revelation_rewards = 37 -#snapshot -idx_baker_balance = 11 -idx_baker_delegated = 12 +# Cycle Snapshot +idx_balance = 0 +idx_baker_delegated = 1 +idx_delegator_address = 2 -idx_delegator_balance = 11 -idx_delegator_address = 15 +# Current balances +idx_cb_delegator_id = 0 +idx_cb_current_balance = 1 +idx_cb_delegator_address = 2 \ No newline at end of file diff --git a/src/tzstats/tzstats_reward_api.py b/src/tzstats/tzstats_reward_api.py index 787f79b7..ef97f32a 100644 --- a/src/tzstats/tzstats_reward_api.py +++ b/src/tzstats/tzstats_reward_api.py @@ -22,9 +22,7 @@ def get_rewards_for_cycle_map(self, cycle, expected_reward = False): root = self.helper.get_rewards_for_cycle(cycle, expected_reward, self.verbose) delegate_staking_balance = root["delegate_staking_balance"] - total_reward_amount = root["total_reward_amount"] + delegators_balances_dict = root["delegators_balances"] - delegators_balance = root["delegators_balance"] - - return RewardProviderModel(delegate_staking_balance, total_reward_amount, delegators_balance) + return RewardProviderModel(delegate_staking_balance, total_reward_amount, delegators_balances_dict) diff --git a/src/tzstats/tzstats_reward_provider_helper.py b/src/tzstats/tzstats_reward_provider_helper.py index 1b6f00fb..08f69fbc 100644 --- a/src/tzstats/tzstats_reward_provider_helper.py +++ b/src/tzstats/tzstats_reward_provider_helper.py @@ -1,5 +1,6 @@ import requests +from time import sleep from exception.api_provider import ApiProviderException from log_config import main_logger from tzstats.tzstats_api_constants import * @@ -7,7 +8,9 @@ logger = main_logger rewards_split_call = '/tables/income?address={}&cycle={}' -delegators_call = '/tables/snapshot?cycle={}&is_selected=1&delegate={}&limit=50000' +delegators_call = '/tables/snapshot?cycle={}&is_selected=1&delegate={}&columns=balance,delegated,address&limit=50000' +batch_current_balance_call = '/tables/account?delegate={}&columns=row_id,spendable_balance,address' +single_current_balance_call = '/tables/account?address={}&columns=row_id,spendable_balance,address' PREFIX_API = {'MAINNET': {'API_URL': 'http://api.tzstats.com'}, 'ZERONET': {'API_URL': 'http://api.zeronet.tzstats.com'}, @@ -29,18 +32,23 @@ def __init__(self, nw, baking_address): self.baking_address = baking_address def get_rewards_for_cycle(self, cycle, expected_reward = False, verbose=False): - ############# - root = {"delegate_staking_balance": 0, "total_reward_amount": 0, "delegators_balance": {}} + + root = {"delegate_staking_balance": 0, "total_reward_amount": 0, "delegators_balances": {}} + # + # Get rewards breakdown for cycle + # uri = self.api['API_URL'] + rewards_split_call.format(self.baking_address, cycle) + sleep(0.5) # be nice to tzstats + if verbose: - logger.debug("Requesting {}".format(uri)) + logger.debug("Requesting rewards breakdown, {}".format(uri)) resp = requests.get(uri, timeout=5) if verbose: - logger.debug("Response from tzstats is {}".format(resp)) + logger.debug("Response from tzstats is {}".format(resp.content.decode("utf8"))) if resp.status_code != 200: # This means something went wrong. @@ -50,18 +58,29 @@ def get_rewards_for_cycle(self, cycle, expected_reward = False, verbose=False): if expected_reward: root["total_reward_amount"] = int(1e6 * float(resp[idx_income_expected_income])) else: - root["total_reward_amount"] = int(1e6 * (float(resp[idx_income_baking_income]) + float(resp[idx_income_endorsing_income]) + float(resp[idx_income_seed_income]) + float(resp[idx_income_fees_income]) - float(resp[idx_income_lost_accusation_fees]) - float(resp[idx_income_lost_accusation_rewards]) - float(resp[idx_income_lost_revelation_fees]) - float(resp[idx_income_lost_revelation_rewards]))) - - + root["total_reward_amount"] = int(1e6 * (float(resp[idx_income_baking_income]) + + float(resp[idx_income_endorsing_income]) + + float(resp[idx_income_seed_income]) + + float(resp[idx_income_fees_income]) + - float(resp[idx_income_lost_accusation_fees]) + - float(resp[idx_income_lost_accusation_rewards]) + - float(resp[idx_income_lost_revelation_fees]) + - float(resp[idx_income_lost_revelation_rewards]))) + + # + # Get staking balances of delegators at snapshot block + # uri = self.api['API_URL'] + delegators_call.format(cycle - self.preserved_cycles - 2, self.baking_address) + sleep(0.5) # be nice to tzstats + if verbose: - logger.debug("Requesting {}".format(uri)) + logger.debug("Requesting staking balances of delegators, {}".format(uri)) resp = requests.get(uri, timeout=5) if verbose: - logger.debug("Response from tzstats is {}".format(resp)) + logger.debug("Response from tzstats is {}".format(resp.content.decode("utf8"))) if resp.status_code != 200: # This means something went wrong. @@ -70,9 +89,97 @@ def get_rewards_for_cycle(self, cycle, expected_reward = False, verbose=False): resp = resp.json() for delegator in resp: + if delegator[idx_delegator_address] == self.baking_address: - root["delegate_staking_balance"] = int(1e6 * (float(delegator[idx_baker_balance]) + float(delegator[idx_baker_delegated]))) + root["delegate_staking_balance"] = int(1e6 * (float(delegator[idx_balance]) + float(delegator[idx_baker_delegated]))) else: - root["delegators_balance"][delegator[idx_delegator_address]] = int(1e6 * float(delegator[idx_delegator_balance])) + delegator_info = {"staking_balance": 0, "current_balance": 0} + delegator_info["staking_balance"] = int(1e6 * float(delegator[idx_balance])) + root["delegators_balances"][delegator[idx_delegator_address]] = delegator_info + + # + # Get current balance of delegates + # + # This is done in 2 phases. 1) make a single API call to tzstats, retrieving an array + # of arrays with current balance of each delegator who "currently" delegates to delegate. There may + # be a case where the delegator has changed delegations and would therefor not be in this array. + # Thus, 2) determines which delegators are not in the first result, and makes individual + # calls to get their balance. This approach should reduce the overall number of API calls made to tzstats. + # + + # Phase 1 + # + uri = self.api['API_URL'] + batch_current_balance_call.format(self.baking_address) + + sleep(0.5) # be nice to tzstats + + if verbose: + logger.debug("Requesting current balance of delegators, phase 1, {}".format(uri)) + + resp = requests.get(uri, timeout=5) + + if verbose: + logger.debug("Response from tzstats is {}".format(resp.content.decode("utf8"))) + + if resp.status_code != 200: + # This means something went wrong. + raise ApiProviderException('GET {} {}'.format(uri, resp.status_code)) + + resp = resp.json() + + # Will use these two lists to determine who has/has not been fetched + staked_bal_delegators = root["delegators_balances"].keys() + curr_bal_delegators = [] + + for delegator in resp: + delegator_addr = delegator[idx_cb_delegator_address] + + # If delegator is in this batch, but has no staking balance for this reward cycle, + # then they must be a new delegator and are not receiving rewards at this time. + # We can ignore them. + if delegator_addr not in staked_bal_delegators: + continue + + root["delegators_balances"][delegator_addr]["current_balance"] = int(1e6 * float(delegator[idx_cb_current_balance])) + curr_bal_delegators.append(delegator_addr) + + # Phase 2 + # + + # Who was not in this result? + need_curr_balance_fetch = list(set(staked_bal_delegators) - set(curr_bal_delegators)) + + # Fetch individual not in original batch + if len(need_curr_balance_fetch) > 0: + + for d in need_curr_balance_fetch: + + uri = self.api['API_URL'] + single_current_balance_call.format(d) + + sleep(0.5) # be nice to tzstats + + if verbose: + logger.debug("Requesting current balance of delegator, phase 2, {}".format(uri)) + + resp = requests.get(uri, timeout=5) + + if verbose: + logger.debug("Response from tzstats is {}".format(resp.content.decode("utf8"))) + + if resp.status_code != 200: + # This means something went wrong. + raise ApiProviderException('GET {} {}'.format(uri, resp.status_code)) + + resp = resp.json()[0] + root["delegators_balances"][d]["current_balance"] = int(1e6 * float(resp[idx_cb_current_balance])) + curr_bal_delegators.append(d) + + # All done fetching balances. + # Sanity check. + n_curr_balance = len(curr_bal_delegators) + n_stake_balance = len(staked_bal_delegators) + + if n_curr_balance != n_stake_balance: + raise ApiProviderException('Did not fetch all balances {}/{}'.format(n_curr_balance, n_stake_balance)) return root