Fix tzstats and destination map issues (tezos-reward-distributor-orga…

…nization#132)

* Fix/tzstats

Fix tzstats issues: As noted in the last PR, tzstats was in testing period and was therefore not set as default. We noticed that the staking balance given by tzstats should be retrieved from the snapshot call and not from the income call, otherwise (differently from tzscan) the staking balance of the payout cycle and not of the snapshot cycle will be taken as reference.
Furthermore, we added the possibility to get reward data of upcoming cycles based on the expected income.
HOWEVER, since tzstats does not allow commercial use only under previous agreement and that could be taken for granted, we will keep on not using tzstats as default. PRPC will be still the default, using tzstats should be explicitly set through the -P tzstats flag, and under personal guarantee in case of commercial use.

* Fix payment address destination map

src/calc/phased_payment_calculator.py

@@ -86,6 +86,7 @@ def calculate(self, reward_provider_model):
         phase4 = CalculatePhase4(self.founders_map, self.owners_map)
         rwrd_logs, total_rwrd_amnt = phase4.calculate(rwrd_logs, total_rwrd_amnt)
 
+
         # calculate amounts
         phase_last = CalculatePhaseFinal()
         rwrd_logs, total_rwrd_amnt = phase_last.calculate(rwrd_logs, total_rwrd_amnt)

src/pay/batch_payer.py

@@ -248,7 +248,7 @@ def attempt_single_batch(self, payment_records, op_counter, verbose=None, dry_ru
 
             op_counter.inc()
 
-            content = CONTENT.replace("%SOURCE%", self.source).replace("%DESTINATION%", payment_item.address) \
+            content = CONTENT.replace("%SOURCE%", self.source).replace("%DESTINATION%", payment_item.paymentaddress) \
                 .replace("%AMOUNT%", str(pymnt_amnt)).replace("%COUNTER%", str(op_counter.get())) \
                 .replace("%fee%", self.default_fee).replace("%gas_limit%", self.gas_limit).replace("%storage_limit%", self.storage_limit)
 

src/pay/payment_producer.py

@@ -137,7 +137,15 @@ def run(self):
                 if pymnt_cycle <= crrnt_cycle - (self.nw_config['NB_FREEZE_CYCLE'] + 1) - self.release_override:
                     if not self.payments_queue.full():
 
-                        result = self.try_to_pay(pymnt_cycle)
+                        if pymnt_cycle >= crrnt_cycle:
+                            if self.reward_api.name == 'tzstats':
+                                result = self.try_to_pay(pymnt_cycle, expected_reward=True)
+                            else:
+                                logger.error("This feature is only possible using tzstats. Please consider changing the provider using the -P flag.")
+                                self.exit()
+                                break
+                        else:
+                            result = self.try_to_pay(pymnt_cycle)
 
                         if result:
                             # single run is done. Do not continue.
@@ -189,7 +197,7 @@ def run(self):
 
         return
 
-    def try_to_pay(self, pymnt_cycle):
+    def try_to_pay(self, pymnt_cycle, expected_reward = False):
         try:
             logger.info("Payment cycle is " + str(pymnt_cycle))
 
@@ -201,7 +209,10 @@ def try_to_pay(self, pymnt_cycle):
                 return True
 
             # 1- get reward data
-            reward_model = self.reward_api.get_rewards_for_cycle_map(pymnt_cycle)
+            if expected_reward:
+                reward_model = self.reward_api.get_rewards_for_cycle_map(pymnt_cycle, expected_reward)
+            else:
+                reward_model = self.reward_api.get_rewards_for_cycle_map(pymnt_cycle)
             # 2- calculate rewards
             reward_logs, total_amount = self.payment_calc.calculate(reward_model)
             # set cycle info

src/rpc/rpc_reward_api.py

@@ -20,6 +20,8 @@ class RpcRewardApiImpl(RewardApi):
     def __init__(self, nw, baking_address, node_url, verbose=True):
         super(RpcRewardApiImpl, self).__init__()
 
+        self.name = 'RPC'
+
         self.blocks_per_cycle = nw['BLOCKS_PER_CYCLE']
         self.preserved_cycles = nw['NB_FREEZE_CYCLE']
         self.blocks_per_roll_snapshot = nw['BLOCKS_PER_ROLL_SNAPSHOT']

src/tzstats/tzstats_api_constants.py

@@ -1,7 +1,5 @@
 # Income
-idx_income_balance = 4
-idx_income_delegated = 5
-
+idx_income_expected_income = 19
 idx_income_total_income = 21
 idx_income_total_bonds = 22
 
@@ -23,5 +21,8 @@
 idx_income_lost_revelation_rewards = 37
 
 #snapshot
+idx_baker_balance = 11
+idx_baker_delegated = 12
+
+idx_delegator_balance = 11
 idx_delegator_address = 15
-idx_delegator_balance = 11

src/tzstats/tzstats_reward_api.py

@@ -12,12 +12,14 @@ class TzStatsRewardApiImpl(RewardApi):
     def __init__(self, nw, baking_address, verbose=False):
         super().__init__()
 
+        self.name = 'tzstats'
+
         self.verbose = verbose
         self.logger = main_logger
         self.helper = TzStatsRewardProviderHelper(nw, baking_address)
 
-    def get_rewards_for_cycle_map(self, cycle):
-        root = self.helper.get_rewards_for_cycle(cycle, self.verbose)
+    def get_rewards_for_cycle_map(self, cycle, expected_reward = False):
+        root = self.helper.get_rewards_for_cycle(cycle, expected_reward, self.verbose)
 
         delegate_staking_balance = root["delegate_staking_balance"]
 

src/tzstats/tzstats_reward_provider_helper.py

@@ -28,7 +28,7 @@ def __init__(self, nw, baking_address):
 
         self.baking_address = baking_address
 
-    def get_rewards_for_cycle(self, cycle, verbose=False):
+    def get_rewards_for_cycle(self, cycle, expected_reward = False, verbose=False):
         #############
         root = {"delegate_staking_balance": 0, "total_reward_amount": 0, "delegators_balance": {}}
 
@@ -47,9 +47,11 @@ def get_rewards_for_cycle(self, cycle, verbose=False):
             raise TzStatsException('GET {} {}'.format(uri, resp.status_code))
 
         resp = resp.json()[0]
+        if expected_reward:
+            root["total_reward_amount"] = int(1e6 * float(resp[idx_income_expected_income]))
+        else:
+            root["total_reward_amount"] = int(1e6 * (float(resp[idx_income_baking_income]) + float(resp[idx_income_endorsing_income]) + float(resp[idx_income_seed_income]) +  float(resp[idx_income_fees_income])  -  float(resp[idx_income_lost_accusation_fees]) -  float(resp[idx_income_lost_accusation_rewards]) -  float(resp[idx_income_lost_revelation_fees]) -  float(resp[idx_income_lost_revelation_rewards])))
 
-        root["total_reward_amount"] = int( 1e6 * ( float(resp[idx_income_baking_income]) +  float(resp[idx_income_endorsing_income]) + float(resp[idx_income_seed_income]) +  float(resp[idx_income_fees_income])  -  float(resp[idx_income_lost_accusation_fees]) -  float(resp[idx_income_lost_accusation_rewards]) -  float(resp[idx_income_lost_revelation_fees]) -  float(resp[idx_income_lost_revelation_rewards])))
-        root["delegate_staking_balance"] = int( 1e6 * (float(resp[idx_income_balance]) + float(resp[idx_income_delegated])))
 
         uri = self.api['API_URL'] + delegators_call.format(cycle - self.preserved_cycles - 2, self.baking_address)
 
@@ -68,7 +70,9 @@ def get_rewards_for_cycle(self, cycle, verbose=False):
         resp = resp.json()
 
         for delegator in resp:
-            if delegator[idx_delegator_address] != self.baking_address:
+            if delegator[idx_delegator_address] == self.baking_address:
+                root["delegate_staking_balance"] = int(1e6 * (float(delegator[idx_baker_balance]) + float(delegator[idx_baker_delegated])))
+            else:
                 root["delegators_balance"][delegator[idx_delegator_address]] = int(1e6 * float(delegator[idx_delegator_balance]))
 
         return root

src/util/csv_payment_file_parser.py

@@ -36,4 +36,4 @@ def write(self, report_file, payment_logs):
 
             for pl in payment_logs:
                 # write row to csv file
-                csv_writer.writerow([pl.address, pl.type, pl.amount, pl.hash if pl.hash else "None", pl.paid.value])
+                csv_writer.writerow([pl.paymentaddress, pl.type, pl.amount, pl.hash if pl.hash else "None", pl.paid.value])