Remove trivial, alter minimum payout calculation (tezos-reward-distri…

…butor-organization#158)

* remove trivial/non-trivial concept
* increase float formatting to see entire value
* zero_threshold becomes 1mutez

src/pay/batch_payer.py

@@ -11,8 +11,6 @@
 from log_config import main_logger
 from util.rpc_utils import parse_json_response
 
-ZERO_THRESHOLD = 2e+3
-
 logger = main_logger
 
 MAX_TX_PER_BLOCK = 280
@@ -34,8 +32,7 @@
 COMM_WAIT = " wait for %OPERATION% to be included --confirmations {}".format(CONFIRMATIONS)
 
 FEE_INI = 'fee.ini'
-DUMMY_FEE = 1000
-
+MUTEZ = 1e6
 
 class BatchPayer():
     def __init__(self, node_url, pymnt_addr, wllt_clnt_mngr, delegator_pays_xfer_fee, network_config):
@@ -44,6 +41,7 @@ def __init__(self, node_url, pymnt_addr, wllt_clnt_mngr, delegator_pays_xfer_fee
         self.node_url = node_url
         self.wllt_clnt_mngr = wllt_clnt_mngr
         self.network_config = network_config
+        self.zero_threshold = 1    # 1 mutez = 0.000001 XTZ
 
         config = configparser.ConfigParser()
         if os.path.isfile(FEE_INI):
@@ -54,7 +52,7 @@ def __init__(self, node_url, pymnt_addr, wllt_clnt_mngr, delegator_pays_xfer_fee
         kttx = config['KTTX']
         self.gas_limit = kttx['gas_limit']
         self.storage_limit = kttx['storage_limit']
-        self.default_fee = kttx['fee']
+        self.default_fee = int(kttx['fee'])
 
         # section below is left to make sure no one using legacy configuration option
         self.delegator_pays_xfer_fee = config.getboolean('KTTX', 'delegator_pays_xfer_fee',
@@ -66,7 +64,14 @@ def __init__(self, node_url, pymnt_addr, wllt_clnt_mngr, delegator_pays_xfer_fee
 
         self.delegator_pays_xfer_fee = delegator_pays_xfer_fee
 
-        logger.debug("Transfer fee is paid by {}".format("Delegator" if self.delegator_pays_xfer_fee else "Delegate"))
+        # If delegator pays the fee, then the cutoff should be transaction-fee + 1
+        # Ex: Delegator reward is 1800 mutez, txn fee is 1792 mutez, reward - fee = 8 mutez payable reward
+        #     If delegate pays fee, then cutoff is 1 mutez payable reward
+        if self.delegator_pays_xfer_fee:
+            self.zero_threshold = self.default_fee + 1
+
+        logger.info("Transfer fee is {:.6f} XTZ and is paid by {}".format(self.default_fee/MUTEZ, "Delegator" if self.delegator_pays_xfer_fee else "Delegate"))
+        logger.info("Payment amount cutoff is {:.6f} XTZ".format(self.zero_threshold/MUTEZ))
 
         # pymnt_addr has a length of 36 and starts with tz or KT then it is a public key has, else it is an alias
         if len(self.pymnt_addr) == PKH_LENGTH and (
@@ -116,28 +121,25 @@ def pay(self, payment_items_in, verbose=None, dry_run=None):
 
         self.log_processed_items(payment_logs)
 
-        payment_items = [pi for pi in payment_items_in if not pi.paid.is_processed()]
+        unprocessed_payment_items = [pi for pi in payment_items_in if not pi.paid.is_processed()]
 
-        # separate trivial items, amounts less than zero_threshold are not trivial, no needed to be paid
-        non_trivial_payment_items = [pi for pi in payment_items if pi.amount < ZERO_THRESHOLD]
-        non_trivial_payment_items_total = sum([pl.amount for pl in non_trivial_payment_items])
-        if non_trivial_payment_items:
-            logger.info("{} payment items are not trivial, total of {:,} mutez".format(len(non_trivial_payment_items), non_trivial_payment_items_total))
-        self.log_non_trivial_items(non_trivial_payment_items)
-
-        trivial_payment_items = [pi for pi in payment_items if pi.amount >= ZERO_THRESHOLD]
-        if not trivial_payment_items:
-            logger.info("No trivial items found, returning...")
+        # all unprocessed_payment_items are important (non-trivial)
+        # gather up all unprocessed_payment_items that are greater than, or equal to the zero_threshold
+        # zero_threshold is either 1 mutez or the txn fee if delegator is not paying it
+        payment_items = [pi for pi in unprocessed_payment_items if pi.amount >= self.zero_threshold]
+        if not payment_items:
+            logger.info("No payment items found, returning...")
             return payment_items_in, 0
 
         # split payments into lists of MAX_TX_PER_BLOCK or less size
         # [list_of_size_MAX_TX_PER_BLOCK,list_of_size_MAX_TX_PER_BLOCK,list_of_size_MAX_TX_PER_BLOCK,...]
-        payment_items_chunks = [trivial_payment_items[i:i + MAX_TX_PER_BLOCK] for i in range(0, len(trivial_payment_items), MAX_TX_PER_BLOCK)]
+        payment_items_chunks = [payment_items[i:i + MAX_TX_PER_BLOCK] for i in range(0, len(payment_items), MAX_TX_PER_BLOCK)]
+
+        total_amount_to_pay = sum([pl.amount for pl in payment_items])
+        if not self.delegator_pays_xfer_fee: total_amount_to_pay += self.default_fee * len(payment_items)
 
-        total_amount_to_pay = sum([pl.amount for pl in trivial_payment_items])
-        if not self.delegator_pays_xfer_fee: total_amount_to_pay += int(self.default_fee) * len(trivial_payment_items)
-        logger.info("Total trivial amount to pay is {:,} mutez.".format(total_amount_to_pay))
-        logger.info("{} trivial payments will be done in {} batches".format(len(trivial_payment_items), len(payment_items_chunks)))
+        logger.info("Total amount to pay out is {:,} mutez.".format(total_amount_to_pay))
+        logger.info("{} payments will be done in {} batches".format(len(payment_items), len(payment_items_chunks)))
 
         total_attempts = 0
         op_counter = OpCounter()
@@ -151,26 +153,13 @@ def pay(self, payment_items_in, verbose=None, dry_run=None):
             payment_logs.extend(payments_log)
             total_attempts += attempt
 
-        # set non trivial items to DONE
-        for pi in non_trivial_payment_items:
-            pi.paid = PaymentStatus.DONE
-            pi.hash = ""
-
-        # add non trivial items
-        payment_logs.extend(non_trivial_payment_items)
-
         return payment_logs, total_attempts
 
     def log_processed_items(self, payment_logs):
         if payment_logs:
             for pl in payment_logs:
                 logger.debug("Reward already %s for cycle %s address %s amount %f tz type %s", pl.paid, pl.cycle, pl.address, pl.amount, pl.type)
 
-    def log_non_trivial_items(self, payment_logs):
-        if payment_logs:
-            for pl in payment_logs:
-                logger.debug("Reward not trivial for address %s amount %f tz type %s", pl.address, pl.amount, pl.type)
-
     def pay_single_batch(self, payment_items, op_counter, verbose=None, dry_run=None):
 
         max_try = 3
@@ -240,21 +229,23 @@ def attempt_single_batch(self, payment_records, op_counter, verbose=None, dry_ru
         for payment_item in payment_records:
             pymnt_amnt = payment_item.amount  # expects in micro tezos
 
-            assert pymnt_amnt >= ZERO_THRESHOLD # zero check, zero amounts needs to be filtered out earlier
-
             if self.delegator_pays_xfer_fee:
-                pymnt_amnt = max(pymnt_amnt - int(self.default_fee), 0)  # ensure not less than 0
+                pymnt_amnt = max(pymnt_amnt - self.default_fee, 0)  # ensure not less than 0
+
+            # if, somehow, pymnt_amnt becomes 0, don't pay
+            if pymnt_amnt == 0:
+                continue
 
             op_counter.inc()
 
             content = CONTENT.replace("%SOURCE%", self.source).replace("%DESTINATION%", payment_item.paymentaddress) \
                 .replace("%AMOUNT%", str(pymnt_amnt)).replace("%COUNTER%", str(op_counter.get())) \
-                .replace("%fee%", self.default_fee).replace("%gas_limit%", self.gas_limit).replace("%storage_limit%", self.storage_limit)
+                .replace("%fee%", str(self.default_fee)).replace("%gas_limit%", self.gas_limit).replace("%storage_limit%", self.storage_limit)
 
             content_list.append(content)
 
             if verbose:
-                logger.debug("Payment content: {}".format(content))
+                logger.info("Payment content: {}".format(content))
 
         contents_string = ",".join(content_list)
 

src/pay/payment_consumer.py

@@ -163,7 +163,7 @@ def create_payment_report(self, nb_failed, nb_injected, payment_logs, payment_cy
         logger.info("Payment report is created at '{}'".format(report_file))
 
         for pl in payment_logs:
-            logger.debug("Payment done for address %s type %s amount {:>8.2f} paid %s".format(pl.amount / MUTEZ), pl.address, pl.type, pl.paid)
+            logger.debug("Payment done for address %s type %s amount {:>10.6f} paid %s".format(pl.amount / MUTEZ), pl.address, pl.type, pl.paid)
 
         if self.publish_stats and not self.dry_run and (not self.args or is_mainnet(self.args.network)):
             stats_dict = self.create_stats_dict(nb_failed, nb_injected, payment_cycle, payment_logs, total_attempts)

src/pay/payment_producer.py

@@ -173,7 +173,7 @@ def run(self):
                         time.sleep(60 * 3)
                 # end of payment cycle check
                 else:
-                    logger.info("No pending payments for cycle {},current cycle is {}".format(pymnt_cycle, crrnt_cycle))
+                    logger.info("No pending payments for cycle {}, current cycle is {}".format(pymnt_cycle, crrnt_cycle))
 
                     # pending payments done. Do not wait any more.
                     if self.run_mode == RunMode.PENDING:
@@ -312,8 +312,8 @@ def create_calculations_report(self, payment_logs, report_file_path, total_rewar
                          pymnt_log.paymentaddress]
                 writer.writerow(array)
 
-                logger.debug("Reward created for address %s type %s balance {:>10.2f} ratio {:.8f} fee_ratio {:.6f} "
-                             "amount {:>8.2f} fee_amount {:.2f} fee_rate {:.2f} payable %s skipped %s atphase %s desc %s pay_addr %s"
+                logger.debug("Reward created for address %s type %s balance {:>10.2f} ratio {:.6f} fee_ratio {:.6f} "
+                             "amount {:>10.6f} fee_amount {:>4.6f} fee_rate {:.2f} payable %s skipped %s atphase %s desc %s pay_addr %s"
                              .format(pymnt_log.balance / MUTEZ, pymnt_log.ratio, pymnt_log.service_fee_ratio,
                                      pymnt_log.amount / MUTEZ, pymnt_log.service_fee_amount / MUTEZ,
                                      pymnt_log.service_fee_rate), pymnt_log.address, pymnt_log.type, pymnt_log.payable,