From cd80e2d8e86e22f51f9bd8985c3b039593bb699a Mon Sep 17 00:00:00 2001 From: Matthew Date: Sun, 22 Dec 2019 07:42:46 -0600 Subject: [PATCH] Remove trivial, alter minimum payout calculation (#158) * remove trivial/non-trivial concept * increase float formatting to see entire value * zero_threshold becomes 1mutez --- src/pay/batch_payer.py | 71 ++++++++++++++++--------------------- src/pay/payment_consumer.py | 2 +- src/pay/payment_producer.py | 6 ++-- 3 files changed, 35 insertions(+), 44 deletions(-) diff --git a/src/pay/batch_payer.py b/src/pay/batch_payer.py index c4533051..10bf636e 100644 --- a/src/pay/batch_payer.py +++ b/src/pay/batch_payer.py @@ -11,8 +11,6 @@ from log_config import main_logger from util.rpc_utils import parse_json_response -ZERO_THRESHOLD = 2e+3 - logger = main_logger MAX_TX_PER_BLOCK = 280 @@ -34,8 +32,7 @@ COMM_WAIT = " wait for %OPERATION% to be included --confirmations {}".format(CONFIRMATIONS) FEE_INI = 'fee.ini' -DUMMY_FEE = 1000 - +MUTEZ = 1e6 class BatchPayer(): def __init__(self, node_url, pymnt_addr, wllt_clnt_mngr, delegator_pays_xfer_fee, network_config): @@ -44,6 +41,7 @@ def __init__(self, node_url, pymnt_addr, wllt_clnt_mngr, delegator_pays_xfer_fee self.node_url = node_url self.wllt_clnt_mngr = wllt_clnt_mngr self.network_config = network_config + self.zero_threshold = 1 # 1 mutez = 0.000001 XTZ config = configparser.ConfigParser() if os.path.isfile(FEE_INI): @@ -54,7 +52,7 @@ def __init__(self, node_url, pymnt_addr, wllt_clnt_mngr, delegator_pays_xfer_fee kttx = config['KTTX'] self.gas_limit = kttx['gas_limit'] self.storage_limit = kttx['storage_limit'] - self.default_fee = kttx['fee'] + self.default_fee = int(kttx['fee']) # section below is left to make sure no one using legacy configuration option self.delegator_pays_xfer_fee = config.getboolean('KTTX', 'delegator_pays_xfer_fee', @@ -66,7 +64,14 @@ def __init__(self, node_url, pymnt_addr, wllt_clnt_mngr, delegator_pays_xfer_fee self.delegator_pays_xfer_fee = delegator_pays_xfer_fee - logger.debug("Transfer fee is paid by {}".format("Delegator" if self.delegator_pays_xfer_fee else "Delegate")) + # If delegator pays the fee, then the cutoff should be transaction-fee + 1 + # Ex: Delegator reward is 1800 mutez, txn fee is 1792 mutez, reward - fee = 8 mutez payable reward + # If delegate pays fee, then cutoff is 1 mutez payable reward + if self.delegator_pays_xfer_fee: + self.zero_threshold = self.default_fee + 1 + + logger.info("Transfer fee is {:.6f} XTZ and is paid by {}".format(self.default_fee/MUTEZ, "Delegator" if self.delegator_pays_xfer_fee else "Delegate")) + logger.info("Payment amount cutoff is {:.6f} XTZ".format(self.zero_threshold/MUTEZ)) # pymnt_addr has a length of 36 and starts with tz or KT then it is a public key has, else it is an alias if len(self.pymnt_addr) == PKH_LENGTH and ( @@ -116,28 +121,25 @@ def pay(self, payment_items_in, verbose=None, dry_run=None): self.log_processed_items(payment_logs) - payment_items = [pi for pi in payment_items_in if not pi.paid.is_processed()] + unprocessed_payment_items = [pi for pi in payment_items_in if not pi.paid.is_processed()] - # separate trivial items, amounts less than zero_threshold are not trivial, no needed to be paid - non_trivial_payment_items = [pi for pi in payment_items if pi.amount < ZERO_THRESHOLD] - non_trivial_payment_items_total = sum([pl.amount for pl in non_trivial_payment_items]) - if non_trivial_payment_items: - logger.info("{} payment items are not trivial, total of {:,} mutez".format(len(non_trivial_payment_items), non_trivial_payment_items_total)) - self.log_non_trivial_items(non_trivial_payment_items) - - trivial_payment_items = [pi for pi in payment_items if pi.amount >= ZERO_THRESHOLD] - if not trivial_payment_items: - logger.info("No trivial items found, returning...") + # all unprocessed_payment_items are important (non-trivial) + # gather up all unprocessed_payment_items that are greater than, or equal to the zero_threshold + # zero_threshold is either 1 mutez or the txn fee if delegator is not paying it + payment_items = [pi for pi in unprocessed_payment_items if pi.amount >= self.zero_threshold] + if not payment_items: + logger.info("No payment items found, returning...") return payment_items_in, 0 # split payments into lists of MAX_TX_PER_BLOCK or less size # [list_of_size_MAX_TX_PER_BLOCK,list_of_size_MAX_TX_PER_BLOCK,list_of_size_MAX_TX_PER_BLOCK,...] - payment_items_chunks = [trivial_payment_items[i:i + MAX_TX_PER_BLOCK] for i in range(0, len(trivial_payment_items), MAX_TX_PER_BLOCK)] + payment_items_chunks = [payment_items[i:i + MAX_TX_PER_BLOCK] for i in range(0, len(payment_items), MAX_TX_PER_BLOCK)] + + total_amount_to_pay = sum([pl.amount for pl in payment_items]) + if not self.delegator_pays_xfer_fee: total_amount_to_pay += self.default_fee * len(payment_items) - total_amount_to_pay = sum([pl.amount for pl in trivial_payment_items]) - if not self.delegator_pays_xfer_fee: total_amount_to_pay += int(self.default_fee) * len(trivial_payment_items) - logger.info("Total trivial amount to pay is {:,} mutez.".format(total_amount_to_pay)) - logger.info("{} trivial payments will be done in {} batches".format(len(trivial_payment_items), len(payment_items_chunks))) + logger.info("Total amount to pay out is {:,} mutez.".format(total_amount_to_pay)) + logger.info("{} payments will be done in {} batches".format(len(payment_items), len(payment_items_chunks))) total_attempts = 0 op_counter = OpCounter() @@ -151,14 +153,6 @@ def pay(self, payment_items_in, verbose=None, dry_run=None): payment_logs.extend(payments_log) total_attempts += attempt - # set non trivial items to DONE - for pi in non_trivial_payment_items: - pi.paid = PaymentStatus.DONE - pi.hash = "" - - # add non trivial items - payment_logs.extend(non_trivial_payment_items) - return payment_logs, total_attempts def log_processed_items(self, payment_logs): @@ -166,11 +160,6 @@ def log_processed_items(self, payment_logs): for pl in payment_logs: logger.debug("Reward already %s for cycle %s address %s amount %f tz type %s", pl.paid, pl.cycle, pl.address, pl.amount, pl.type) - def log_non_trivial_items(self, payment_logs): - if payment_logs: - for pl in payment_logs: - logger.debug("Reward not trivial for address %s amount %f tz type %s", pl.address, pl.amount, pl.type) - def pay_single_batch(self, payment_items, op_counter, verbose=None, dry_run=None): max_try = 3 @@ -240,21 +229,23 @@ def attempt_single_batch(self, payment_records, op_counter, verbose=None, dry_ru for payment_item in payment_records: pymnt_amnt = payment_item.amount # expects in micro tezos - assert pymnt_amnt >= ZERO_THRESHOLD # zero check, zero amounts needs to be filtered out earlier - if self.delegator_pays_xfer_fee: - pymnt_amnt = max(pymnt_amnt - int(self.default_fee), 0) # ensure not less than 0 + pymnt_amnt = max(pymnt_amnt - self.default_fee, 0) # ensure not less than 0 + + # if, somehow, pymnt_amnt becomes 0, don't pay + if pymnt_amnt == 0: + continue op_counter.inc() content = CONTENT.replace("%SOURCE%", self.source).replace("%DESTINATION%", payment_item.paymentaddress) \ .replace("%AMOUNT%", str(pymnt_amnt)).replace("%COUNTER%", str(op_counter.get())) \ - .replace("%fee%", self.default_fee).replace("%gas_limit%", self.gas_limit).replace("%storage_limit%", self.storage_limit) + .replace("%fee%", str(self.default_fee)).replace("%gas_limit%", self.gas_limit).replace("%storage_limit%", self.storage_limit) content_list.append(content) if verbose: - logger.debug("Payment content: {}".format(content)) + logger.info("Payment content: {}".format(content)) contents_string = ",".join(content_list) diff --git a/src/pay/payment_consumer.py b/src/pay/payment_consumer.py index c1b3a6db..1e8e31c9 100644 --- a/src/pay/payment_consumer.py +++ b/src/pay/payment_consumer.py @@ -163,7 +163,7 @@ def create_payment_report(self, nb_failed, nb_injected, payment_logs, payment_cy logger.info("Payment report is created at '{}'".format(report_file)) for pl in payment_logs: - logger.debug("Payment done for address %s type %s amount {:>8.2f} paid %s".format(pl.amount / MUTEZ), pl.address, pl.type, pl.paid) + logger.debug("Payment done for address %s type %s amount {:>10.6f} paid %s".format(pl.amount / MUTEZ), pl.address, pl.type, pl.paid) if self.publish_stats and not self.dry_run and (not self.args or is_mainnet(self.args.network)): stats_dict = self.create_stats_dict(nb_failed, nb_injected, payment_cycle, payment_logs, total_attempts) diff --git a/src/pay/payment_producer.py b/src/pay/payment_producer.py index e09fa46d..2a4fd03b 100644 --- a/src/pay/payment_producer.py +++ b/src/pay/payment_producer.py @@ -173,7 +173,7 @@ def run(self): time.sleep(60 * 3) # end of payment cycle check else: - logger.info("No pending payments for cycle {},current cycle is {}".format(pymnt_cycle, crrnt_cycle)) + logger.info("No pending payments for cycle {}, current cycle is {}".format(pymnt_cycle, crrnt_cycle)) # pending payments done. Do not wait any more. if self.run_mode == RunMode.PENDING: @@ -312,8 +312,8 @@ def create_calculations_report(self, payment_logs, report_file_path, total_rewar pymnt_log.paymentaddress] writer.writerow(array) - logger.debug("Reward created for address %s type %s balance {:>10.2f} ratio {:.8f} fee_ratio {:.6f} " - "amount {:>8.2f} fee_amount {:.2f} fee_rate {:.2f} payable %s skipped %s atphase %s desc %s pay_addr %s" + logger.debug("Reward created for address %s type %s balance {:>10.2f} ratio {:.6f} fee_ratio {:.6f} " + "amount {:>10.6f} fee_amount {:>4.6f} fee_rate {:.2f} payable %s skipped %s atphase %s desc %s pay_addr %s" .format(pymnt_log.balance / MUTEZ, pymnt_log.ratio, pymnt_log.service_fee_ratio, pymnt_log.amount / MUTEZ, pymnt_log.service_fee_amount / MUTEZ, pymnt_log.service_fee_rate), pymnt_log.address, pymnt_log.type, pymnt_log.payable,