Edit help section and repo cleanup (tezos-reward-distributor-organiza…

…tion#144) * Edit help section * Reorganize main function * Clean repo from unnecssary files * Rename tzstats exception to api provider exception * Cleanup fee.ini and document used values * Output operation hash by changing log level to info * This PR resolves issue tezos-reward-distributor-organization#115
dansan566 · Dec 15, 2019 · 6430e0e · 6430e0e
1 parent 2d5fb37
commit 6430e0e
Show file tree

Hide file tree

Showing 12 changed files with 118 additions and 99 deletions.
diff --git a/_config.yml b/_config.yml
diff --git a/fee.ini b/fee.ini
@@ -4,4 +4,7 @@ fee=1792
 [KTTX]
 storage_limit=0
 gas_limit=15385
-base=100
+
+# The used values for gas_limit correspond to protocol Babylon 2.0.1 and allow transactions from
+# implicit (tz) to implicit and implicit to manager.tz
+# Payouts from a manager.tz account are not supported using these values
diff --git a/src/cli/app.log b/src/cli/app.log
diff --git a/src/email.ini b/src/email.ini
diff --git a/src/exception/api_provider.py b/src/exception/api_provider.py
@@ -0,0 +1,2 @@
+class ApiProviderException(Exception):
+    pass
diff --git a/src/exception/tzstats.py b/src/exception/tzstats.py
diff --git a/src/launch_common.py b/src/launch_common.py
@@ -2,6 +2,8 @@
 
 from log_config import main_logger
 
+import argparse
+
 LINER = "--------------------------------------------"
 logger = main_logger
 
@@ -23,29 +25,93 @@ def print_banner(args, script_name):
         logger.info("DRY RUN MODE")
         logger.info(LINER)
 
+def parse_arguments():
+    parser = argparse.ArgumentParser()
+    add_argument_cycle(parser)
+    add_argument_mode(parser)
+    add_argument_release_override(parser)
+    add_argument_payment_offset(parser)
+    add_argument_network(parser)
+    add_argument_node_addr(parser)
+    add_argument_provider(parser)
+    add_argument_node_addr_public(parser)
+    add_argument_reports_base(parser)
+    add_argument_config_dir(parser)
+    add_argument_dry(parser)
+    add_argument_dry_no_consumer(parser)
+    add_argument_executable_dirs(parser)
+    add_argument_docker(parser)
+    add_argument_background_service(parser)
+    add_argument_stats(parser)
+    add_argument_verbose(parser)
+    args = parser.parse_args()
+    return args
+
+
+def add_argument_cycle(parser):
+    parser.add_argument("-C", "--initial_cycle",
+                        help="First cycle to start payment. For last released rewards, set to 0. Non-positive values "
+                             "are interpreted as: current cycle - abs(initial_cycle) - (NB_FREEZE_CYCLE+1). "
+                             "If not set application will continue from last payment made or last reward released.",
+                        type=int)
+
+
+def add_argument_mode(parser):
+    parser.add_argument("-M", "--run_mode",
+                        help="Waiting decision after making pending payments. 1: default option. Run forever. "
+                             "2: Run all pending payments and exit. 3: Run for one cycle and exit. "
+                             "Suitable to use with -C option.",
+                        default=1, choices=[1, 2, 3], type=int)
+
+
+class ReleaseOverrideAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        if not -11 <= values:
+            parser.error("Valid range for release-override({0}) is [-11,) ".format(option_string))
+
+        setattr(namespace, "release_override", values)
+
+
+def add_argument_release_override(parser):
+    parser.add_argument("-R", "--release_override",
+                        help="Override NB_FREEZE_CYCLE value. last released payment cycle will be "
+                             "(current_cycle-(NB_FREEZE_CYCLE+1)-release_override). Suitable for future payments. "
+                             "For future payments give negative values. Valid range is [-11,)",
+                        default=0, type=int, action=ReleaseOverrideAction)
+
+
+def add_argument_payment_offset(parser):
+    parser.add_argument("-O", "--payment_offset",
+                        help="Number of blocks to wait after a cycle starts before starting payments. "
+                             "This can be useful because cycle beginnings may be bussy.",
+                        default=0, type=int)
+
+
 def add_argument_network(parser):
-    parser.add_argument("-N", "--network", help="network name", choices=['ZERONET', 'ALPHANET', 'MAINNET'],
+    parser.add_argument("-N", "--network", help="Network name. Default is Mainnet, The test network of tezos is referred to as Alphanet even if the name changes with each protocol upgrade.",
+                        choices=['MAINNET', 'ZERONET', 'ALPHANET'],
                         default='MAINNET')
 
 
-def add_argument_reports_base(parser):
-    parser.add_argument("-r", "--reports_base", help="Base directory to create reports", default='~/pymnt/reports')
+def add_argument_node_addr(parser):
+    parser.add_argument("-A", "--node_addr", help="Node (host:port pair). Default is 127.0.0.1:8732. This is the main Tezos node used by the client for rpc queries and operation injections.", default='127.0.0.1:8732')
 
 
 def add_argument_provider(parser):
-    parser.add_argument("-P", "--reward_data_provider", help="where reward data is provided. prpc=public rpc", choices=['rpc','prpc','tzstats'],
+    parser.add_argument("-P", "--reward_data_provider", help="where reward data is provided. The default is the use of a public archive rpc node which is https://mainnet.tezrpc.me to query all needed data for reward calculations. If you prefer to use your own local node defined with the -A flag for getting reward data please set the provider to rpc (the local node MUST be an ARCHIVE node in this case). If you prefer using a public rpc node, please set the node URL using the -Ap flag. An alternative for providing reward data is tzstats, but pay attention for license in case of COMMERCIAL use!!", choices=['rpc','prpc','tzstats'],
                         default='prpc')
 
 
-def add_argument_config_dir(parser):
-    parser.add_argument("-f", "--config_dir", help="Directory to find baking configurations", default='~/pymnt/cfg')
+def add_argument_node_addr_public(parser):
+    parser.add_argument("-Ap", "--node_addr_public", help="Public node base url pair with protocol prefix. i.e. https://rpc.letzbake.com. This argument will only be used in case the provider is set to prpc. This node will only be used to query reward data and delegator list. It must be an ARCHIVE node. (default is https://mainnet.tezrpc.me)", default='https://mainnet.tezrpc.me')
 
 
-def add_argument_node_addr(parser):
-    parser.add_argument("-A", "--node_addr", help="Node host:port pair", default='127.0.0.1:8732')
+def add_argument_reports_base(parser):
+    parser.add_argument("-r", "--reports_base", help="Base directory to create reports", default='~/pymnt/reports')
 
-def add_argument_node_addr_public(parser):
-    parser.add_argument("-Ap", "--node_addr_public", help="Public node base url pair with protocol prefix. i.e. https://rpc.letzbake.com", default='https://mainnet.tezrpc.me')
+
+def add_argument_config_dir(parser):
+    parser.add_argument("-f", "--config_dir", help="Directory to find baking configurations", default='~/pymnt/cfg')
 
 
 def add_argument_dry(parser):
@@ -74,5 +140,18 @@ def add_argument_docker(parser):
                         action="store_true")
 
 
+def add_argument_background_service(parser):
+    parser.add_argument("-s", "--background_service",
+                        help="Marker to indicate that TRD is running in daemon mode. "
+                             "When not given it indicates that TRD is in interactive mode.",
+                        action="store_true")
+
+
+def add_argument_stats(parser):
+    parser.add_argument("-Dp", "--do_not_publish_stats",
+                        help="Do not publish anonymous usage statistics",
+                        action="store_true")
+
+
 def add_argument_verbose(parser):
     parser.add_argument("-V", "--verbose", help="Produces a lot of logs. Good for trouble shooting.", action="store_true")
diff --git a/src/main.py b/src/main.py
@@ -1,10 +1,10 @@
-import argparse
 import json
 import os
 import queue
 import sys
 import time
 
+from launch_common import print_banner, parse_arguments
 import version
 from Constants import RunMode
 from NetworkConfiguration import init_network_config
@@ -16,9 +16,6 @@
 from config.yaml_baking_conf_parser import BakingYamlConfParser
 from config.yaml_conf_parser import YamlConfParser
 from log_config import main_logger
-from launch_common import print_banner, add_argument_network, add_argument_provider, add_argument_reports_base, \
-    add_argument_config_dir, add_argument_node_addr, add_argument_dry, add_argument_dry_no_consumer, \
-    add_argument_executable_dirs, add_argument_docker, add_argument_verbose, add_argument_node_addr_public
 from model.baking_conf import BakingConf
 from pay.payment_consumer import PaymentConsumer
 from pay.payment_producer import PaymentProducer
@@ -28,12 +25,10 @@
 from util.process_life_cycle import ProcessLifeCycle
 
 LINER = "--------------------------------------------"
-
 NB_CONSUMERS = 1
 BUF_SIZE = 50
 payments_queue = queue.Queue(BUF_SIZE)
 logger = main_logger
-
 life_cycle = ProcessLifeCycle()
 
 
@@ -198,62 +193,13 @@ def get_latest_report_file(payments_root):
     return recent
 
 
-class ReleaseOverrideAction(argparse.Action):
-    def __call__(self, parser, namespace, values, option_string=None):
-        if not -11 <= values:
-            parser.error("Valid range for release-override({0}) is [-11,) ".format(option_string))
-
-        setattr(namespace, "release_override", values)
-
-
 if __name__ == '__main__':
 
     if not sys.version_info.major >= 3 and sys.version_info.minor>=6:
         raise Exception("Must be using Python 3.6 or later but it is {}.{}".format(sys.version_info.major,sys.version_info.minor ))
 
-    parser = argparse.ArgumentParser()
-
-    add_argument_network(parser)
-    add_argument_provider(parser)
-    add_argument_reports_base(parser)
-    add_argument_config_dir(parser)
-    add_argument_node_addr(parser)
-    add_argument_node_addr_public(parser)
-    add_argument_dry(parser)
-    add_argument_dry_no_consumer(parser)
-    add_argument_executable_dirs(parser)
-    add_argument_docker(parser)
-    add_argument_verbose(parser)
-
-    parser.add_argument("-s", "--background_service",
-                        help="Marker to indicate that TRD is running in daemon mode. "
-                             "When not given it indicates that TRD is in interactive mode.",
-                        action="store_true")
-    parser.add_argument("-Dp", "--do_not_publish_stats",
-                        help="Do not publish anonymous usage statistics",
-                        action="store_true")
-    parser.add_argument("-M", "--run_mode",
-                        help="Waiting decision after making pending payments. 1: default option. Run forever. "
-                             "2: Run all pending payments and exit. 3: Run for one cycle and exit. "
-                             "Suitable to use with -C option.",
-                        default=1, choices=[1, 2, 3], type=int)
-    parser.add_argument("-R", "--release_override",
-                        help="Override NB_FREEZE_CYCLE value. last released payment cycle will be "
-                             "(current_cycle-(NB_FREEZE_CYCLE+1)-release_override). Suitable for future payments. "
-                             "For future payments give negative values. Valid range is [-11,)",
-                        default=0, type=int, action=ReleaseOverrideAction)
-    parser.add_argument("-O", "--payment_offset",
-                        help="Number of blocks to wait after a cycle starts before starting payments. "
-                             "This can be useful because cycle beginnings may be bussy.",
-                        default=0, type=int)
-    parser.add_argument("-C", "--initial_cycle",
-                        help="First cycle to start payment. For last released rewards, set to 0. Non-positive values "
-                             "are interpreted as: current cycle - abs(initial_cycle) - (NB_FREEZE_CYCLE+1). "
-                             "If not set application will continue from last payment made or last reward released.",
-                        type=int)
-
-    args = parser.parse_args()
-    script_name = ""
-    print_banner(args, script_name)
+    args = parse_arguments()
+
+    print_banner(args, script_name="")
 
     main(args)
diff --git a/src/pay/batch_payer.py b/src/pay/batch_payer.py
@@ -52,7 +52,6 @@ def __init__(self, node_url, pymnt_addr, wllt_clnt_mngr, delegator_pays_xfer_fee
             logger.warn("File {} not found. Using default fee values".format(FEE_INI))
 
         kttx = config['KTTX']
-        self.base = kttx['base']
         self.gas_limit = kttx['gas_limit']
         self.storage_limit = kttx['storage_limit']
         self.default_fee = kttx['fee']
@@ -351,14 +350,14 @@ def attempt_single_batch(self, payment_records, op_counter, verbose=None, dry_ru
             return PaymentStatus.FAIL, ""
 
         operation_hash = parse_json_response(inject_command_response)
-        logger.debug("Operation hash is {}".format(operation_hash))
+        logger.info("Operation hash is {}".format(operation_hash))
 
         # wait for inclusion
-        logger.debug("Waiting for operation {} to be included. Please be patient until the block has {} confirmation(s)".format(operation_hash, CONFIRMATIONS))
+        logger.info("Waiting for operation {} to be included. Please be patient until the block has {} confirmation(s)".format(operation_hash, CONFIRMATIONS))
         try:
             cmd = self.comm_wait.replace("%OPERATION%", operation_hash)
             self.wllt_clnt_mngr.send_request(cmd, timeout=self.network_config[BLOCK_TIME_IN_SEC] * (CONFIRMATIONS + PATIENCE))
-            logger.debug("Operation {} is included".format(operation_hash))
+            logger.info("Operation {} is included".format(operation_hash))
         except TimeoutExpired:
             logger.warn("Operation {} wait is timed out. Not sure about the result!".format(operation_hash))
             return PaymentStatus.INJECTED, operation_hash

diff --git a/src/pay/payment_producer.py b/src/pay/payment_producer.py
@@ -9,7 +9,7 @@
 from log_config import main_logger
 from model.reward_log import RewardLog
 from model.rules_model import RulesModel
-from exception.tzstats import TzStatsException
+from exception.api_provider import ApiProviderException
 from requests import ReadTimeout, ConnectTimeout
 from pay.double_payment_check import check_past_payment
 from pay.payment_batch import PaymentBatch
@@ -193,9 +193,9 @@ def run(self):
                     # wait until current cycle ends
                     self.wait_until_next_cycle(nb_blocks_remaining)
 
-            except TzStatsException:
-                logger.debug("Tzstats error at reward loop", exc_info=True)
-                logger.info("Tzstats error at reward loop, will try again.")
+            except ApiProviderException:
+                logger.debug("{} API error at reward loop".format(self.reward_api.name), exc_info=True)
+                logger.info("{} API error at reward loop, will try again.".format(self.reward_api.name))
             except Exception:
                 logger.error("Error at payment producer loop", exc_info=True)
 
@@ -254,16 +254,16 @@ def try_to_pay(self, pymnt_cycle, expected_reward = False):
 
             return True
         except ReadTimeout:
-            logger.info("Tzstats call failed, will try again.")
-            logger.debug("Tzstats call failed", exc_info=False)
+            logger.info("API provider call failed, will try again.")
+            logger.debug("API provider call failed", exc_info=False)
             return False
         except ConnectTimeout:
-            logger.info("Tzstats connection failed, will try again.")
-            logger.debug("Tzstats connection failed", exc_info=False)
+            logger.info("API provider connection failed, will try again.")
+            logger.debug("API provider connection failed", exc_info=False)
             return False
-        except TzStatsException:
-            logger.info("Tzstats error at reward loop, will try again.")
-            logger.debug("Tzstats error at reward loop", exc_info=False)
+        except ApiProviderException:
+            logger.info("API provider error at reward loop, will try again.")
+            logger.debug("API provider error at reward loop", exc_info=False)
             return False
         except Exception:
             logger.error("Error at payment producer loop, will try again.", exc_info=True)

diff --git a/src/tzstats/tzstats_block_api.py b/src/tzstats/tzstats_block_api.py
@@ -1,7 +1,7 @@
 import requests
 
 from api.block_api import BlockApi
-from exception.tzstats import TzStatsException
+from exception.api_provider import ApiProviderException
 from log_config import main_logger
 
 logger = main_logger

diff --git a/src/tzstats/tzstats_reward_provider_helper.py b/src/tzstats/tzstats_reward_provider_helper.py
@@ -1,6 +1,6 @@
 import requests
 
-from exception.tzstats import TzStatsException
+from exception.api_provider import ApiProviderException
 from log_config import main_logger
 from tzstats.tzstats_api_constants import *
 
@@ -24,7 +24,7 @@ def __init__(self, nw, baking_address):
 
         self.api = PREFIX_API[nw['NAME']]
         if self.api is None:
-            raise TzStatsException("Unknown network {}".format(nw))
+            raise ApiProviderException("Unknown network {}".format(nw))
 
         self.baking_address = baking_address
 
@@ -44,7 +44,7 @@ def get_rewards_for_cycle(self, cycle, expected_reward = False, verbose=False):
 
         if resp.status_code != 200:
             # This means something went wrong.
-            raise TzStatsException('GET {} {}'.format(uri, resp.status_code))
+            raise ApiProviderException('GET {} {}'.format(uri, resp.status_code))
 
         resp = resp.json()[0]
         if expected_reward:
@@ -65,7 +65,7 @@ def get_rewards_for_cycle(self, cycle, expected_reward = False, verbose=False):
 
         if resp.status_code != 200:
             # This means something went wrong.
-            raise TzStatsException('GET {} {}'.format(uri, resp.status_code))
+            raise ApiProviderException('GET {} {}'.format(uri, resp.status_code))
 
         resp = resp.json()