diff --git a/fee.ini b/fee.ini index e58a0571..8b15d334 100644 --- a/fee.ini +++ b/fee.ini @@ -5,5 +5,4 @@ delegator_pays_xfer_fee=true [KTTX] storage_limit=0 gas_limit=10100 -base=100 - +base=100 \ No newline at end of file diff --git a/src/cli/__init__.py b/src/cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/cli/simple_client_manager.py b/src/cli/simple_client_manager.py new file mode 100644 index 00000000..d8653ee9 --- /dev/null +++ b/src/cli/simple_client_manager.py @@ -0,0 +1,42 @@ +import subprocess + +from util.client_utils import clear_terminal_chars + + +class SimpleClientManager: + def __init__(self, client_path, verbose=None) -> None: + super().__init__() + self.verbose = verbose + self.client_path = client_path + + def send_request(self, cmd): + whole_cmd = self.client_path + cmd + if self.verbose: + print("Command is |{}|".format(whole_cmd)) + + # execute client + process = subprocess.Popen(whole_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + bytes = [] + for b in process.stdout: + bytes.append(b) + + process.wait() + + buffer = b''.join(bytes).decode('utf-8') + + if self.verbose: + print("Answer is |{}|".format(buffer)) + + return buffer + + def sign(self, bytes, key_name): + response = self.send_request(" sign bytes 0x03{} for {}".format(bytes, key_name)) + + response = clear_terminal_chars(response) + + for line in response.splitlines(): + if "Signature" in line: + return line.strip("Signature:").strip() + + raise Exception("Signature not found in response '{}'".format(response.replace('\n', ''))) diff --git a/src/cli/test_walletClientManager.py b/src/cli/test_walletClientManager.py new file mode 100644 index 00000000..7659519f --- /dev/null +++ b/src/cli/test_walletClientManager.py @@ -0,0 +1,75 @@ +from unittest import TestCase + +from cli.wallet_client_manager import WalletClientManager + + +class TestWalletClientManager(TestCase): + def test_parse_get_manager_for_contract_response(self): + response = """ + Disclaimer: + The Tezos network is a new blockchain technology. + Users are solely responsible for any risks associated + with usage of the Tezos network. Users should do their + own research to determine if Tezos is the appropriate + platform for their needs and should apply judgement and + care in their network interactions. + + tz1PJnnddwa1P5jg5sChddigxfMccK8nwLiV (known as habanoz) + """ + clientManager = WalletClientManager(None) + manager = clientManager.parse_get_manager_for_contract_response(response) + self.assertEqual('tz1PJnnddwa1P5jg5sChddigxfMccK8nwLiV', manager) + + def test_parse_client_list_known_contracts_response(self): + response = """ + Disclaimer: + The Tezos network is a new blockchain technology. + Users are solely responsible for any risks associated + with usage of the Tezos network. Users should do their + own research to determine if Tezos is the appropriate + platform for their needs and should apply judgement and + care in their network interactions. + + newcontr: KT1XqEHigP5XumZy9i76QyVd6u93VD4HTqJK + habanoz: tz1fyvFH2pd3V9UEq5psqVokVBYkt7rHTKio + mainnetme: tz1a5GGJeyqeQ4ihZqbiRVcvj5rY5kMAt3Xa + """ + clientManager = WalletClientManager(None) + dict = clientManager.parse_client_list_known_contracts_response(response) + + self.assertTrue(dict['newcontr'] == 'KT1XqEHigP5XumZy9i76QyVd6u93VD4HTqJK') + self.assertTrue(dict['habanoz'] == 'tz1fyvFH2pd3V9UEq5psqVokVBYkt7rHTKio') + self.assertTrue(dict['mainnetme'] == 'tz1a5GGJeyqeQ4ihZqbiRVcvj5rY5kMAt3Xa') + + def test_parse_list_known_addresses_response(self): + response = """ + Disclaimer: + The Tezos network is a new blockchain technology. + Users are solely responsible for any risks associated + with usage of the Tezos network. Users should do their + own research to determine if Tezos is the appropriate + platform for their needs and should apply judgement and + care in their network interactions. + + habanoz: tz1fyvFH2pd3V9UEq5psqVokVBYkt7rHTKio (unencrypted sk known) + mainnetme: tz1a5GGJeyqeQ4ihZqbiRVcvj5rY5kMAt3Xa (tcp sk known) + zeronetme: tz1MZ72sJEVen3Qgc7uWvqKhKFJW84bNGd6T (unencrypted sk not known) + """ + + clientManager = WalletClientManager(None) + dict = clientManager.parse_list_known_addresses_response(response) + + habanoz = dict['tz1fyvFH2pd3V9UEq5psqVokVBYkt7rHTKio'] + + self.assertEqual(habanoz['alias'], 'habanoz') + self.assertEqual(habanoz['sk'], True) + + mainnetme = dict['tz1a5GGJeyqeQ4ihZqbiRVcvj5rY5kMAt3Xa'] + + self.assertEqual(mainnetme['alias'], 'mainnetme') + self.assertEqual(mainnetme['sk'], True) + + zeronetme = dict['tz1MZ72sJEVen3Qgc7uWvqKhKFJW84bNGd6T'] + + self.assertEqual(zeronetme['alias'], 'zeronetme') + self.assertEqual(zeronetme['sk'], False) diff --git a/src/cli/wallet_client_manager.py b/src/cli/wallet_client_manager.py new file mode 100644 index 00000000..ebfc5353 --- /dev/null +++ b/src/cli/wallet_client_manager.py @@ -0,0 +1,141 @@ +from cli.simple_client_manager import SimpleClientManager +from util.address_validator import AddressValidator +from util.client_utils import clear_terminal_chars, not_indicator_line + + +class WalletClientManager(SimpleClientManager): + + def __init__(self, client_path, contr_dict_by_alias=None, + addr_dict_by_pkh=None, managers=None, verbose=None) -> None: + super().__init__(client_path, verbose) + + self.managers = managers + if self.managers is None: + self.managers = dict() + + self.contr_dict_by_alias = contr_dict_by_alias + self.addr_dict_by_pkh = addr_dict_by_pkh + self.address_dict = None + + def get_manager_for_contract(self, pkh): + + if pkh.startswith('tz'): + return pkh + + if pkh in self.managers: + return self.managers[pkh] + + response = self.send_request(" get manager for " + pkh) + + response = clear_terminal_chars(response) + + manager = self.parse_get_manager_for_contract_response(response) + + try: + AddressValidator("manager").validate(manager) + except Exception as e: + raise Exception("Invalid response from client '{}'".format(response),e) + self.managers[pkh] = manager + + return manager + + def parse_get_manager_for_contract_response(self, response): + manager = None + for line in response.splitlines(): + line = line.strip() + if line.startswith("tz"): + line = line.replace(" (", ':') + manager, alias_plus = line.split(":", maxsplit=1) + break + + if self.verbose: + print("Manager address is : {}".format(manager)) + + return manager + + def generate_address_dict(self): + if self.address_dict is not None: + return self.address_dict + + self.address_dict = {} + + if self.contr_dict_by_alias is None: + self.contr_dict_by_alias = self.list_known_contracts_by_alias() + + if self.addr_dict_by_pkh is None: + self.addr_dict_by_pkh = self.list_known_addresses_by_pkh() + + for pkh, dict_alias_sk in self.addr_dict_by_pkh.items(): + self.address_dict[pkh] = {"pkh": pkh, "originated": False, "alias": dict_alias_sk['alias'], + "sk": dict_alias_sk['sk'], "manager": pkh} + + for alias, pkh in self.contr_dict_by_alias.items(): + if pkh.startswith("KT"): + manager = self.get_manager_for_contract(pkh) + manager_sk = self.addr_dict_by_pkh[manager]['sk'] + + self.address_dict[pkh] = {"pkh": pkh, "originated": True, "alias": alias, "sk": manager_sk, + "manager": manager} + + def list_known_contracts_by_alias(self): + response = self.send_request(" list known contracts") + + response = clear_terminal_chars(response) + + dict = self.parse_client_list_known_contracts_response(response) + + return dict + + def parse_client_list_known_contracts_response(self, response): + dict = {} + for line in response.splitlines(): + line = line.strip() + if ":" in line and not_indicator_line(line): + alias, pkh = line.split(":", maxsplit=1) + dict[alias.strip()] = pkh.strip() + if self.verbose: + print("known contracts: {}".format(dict)) + return dict + + def list_known_addresses_by_pkh(self): + + response = self.send_request(" list known addresses") + + response = clear_terminal_chars(response) + + dict = self.parse_list_known_addresses_response(response) + + return dict + + def get_known_contact_by_alias(self, alias): + + if self.contr_dict_by_alias is None: + self.contr_dict_by_alias = self.list_known_contracts_by_alias() + + return self.contr_dict_by_alias[alias] + + def get_known_contacts_by_alias(self): + + if self.contr_dict_by_alias is None: + self.contr_dict_by_alias = self.list_known_contracts_by_alias() + + return self.contr_dict_by_alias + + def parse_list_known_addresses_response(self, response): + dict = {} + + for line in response.splitlines(): + line = line.strip() + if ":" in line and not_indicator_line(line): + alias, pkh_plus_braces = line.split(":", maxsplit=1) + pkh_plus_braces = pkh_plus_braces.replace(' (', ':') + pkh, sk_section = pkh_plus_braces.split(":", maxsplit=1) + sk_known = "sk known" in sk_section + pkh = pkh.strip() + alias = alias.strip() + dict[pkh] = {"alias": alias, "sk": sk_known} + + if self.verbose: + print("known addresses: {}".format(dict)) + + return dict diff --git a/src/config/yaml_app_conf_parser.py b/src/config/yaml_app_conf_parser.py index a11e0386..aac2a9f0 100644 --- a/src/config/yaml_app_conf_parser.py +++ b/src/config/yaml_app_conf_parser.py @@ -19,10 +19,9 @@ class AppYamlConfParser(YamlConfParser): - def __init__(self, yaml_text, known_contracts, managers, verbose=None) -> None: + def __init__(self, yaml_text, wllt_clnt_mngr, verbose=None) -> None: super().__init__(yaml_text, verbose) - self.known_contracts = known_contracts - self.managers = managers + self.wllt_clnt_mngr = wllt_clnt_mngr def parse(self): yaml_conf_dict = super().parse() @@ -99,14 +98,14 @@ def __validate_payment_address(self, conf_obj): if len(pymnt_addr) == PKH_LENGHT and (pymnt_addr.startswith("KT") or pymnt_addr.startswith("tz")): conf_obj[('%s_type' % PAYMENT_ADDRESS)] = AddrType.KT if pymnt_addr.startswith("KT") else AddrType.TZ conf_obj[('%s_pkh' % PAYMENT_ADDRESS)] = pymnt_addr - conf_obj[('%s_manager' % PAYMENT_ADDRESS)] = self.managers[pymnt_addr] + conf_obj[('%s_manager' % PAYMENT_ADDRESS)] = self.wllt_clnt_mngr.get_manager_for_contract(pymnt_addr) else: - if pymnt_addr in self.known_contracts: - pkh = self.known_contracts[pymnt_addr] + if pymnt_addr in self.wllt_clnt_mngr.get_known_contacts_by_alias(): + pkh = self.wllt_clnt_mngr.get_known_contact_by_alias(pymnt_addr) conf_obj[('%s_type' % PAYMENT_ADDRESS)] = AddrType.KTALS if pkh.startswith("KT") else AddrType.TZALS conf_obj[('%s_pkh' % PAYMENT_ADDRESS)] = pkh - conf_obj[('%s_manager' % PAYMENT_ADDRESS)] = self.managers[pkh] + conf_obj[('%s_manager' % PAYMENT_ADDRESS)] = self.wllt_clnt_mngr.get_manager_for_contract(pkh) else: raise Exception("Payment Address ({}) cannot be translated into a PKH or alias".format(pymnt_addr)) diff --git a/src/main.py b/src/main.py index 197c7fc0..e5f65aa5 100644 --- a/src/main.py +++ b/src/main.py @@ -11,8 +11,10 @@ from NetworkConfiguration import network_config_map from calc.payment_calculator import PaymentCalculator from calc.service_fee_calculator import ServiceFeeCalculator +from cli.wallet_client_manager import WalletClientManager from config.config_parser import ConfigParser from config.yaml_app_conf_parser import AppYamlConfParser +from config.yaml_conf_parser import YamlConfParser from log_config import main_logger from model.payment_log import PaymentRecord from pay.double_payment_check import check_past_payment @@ -306,6 +308,11 @@ def main(args): if config_dir and not os.path.exists(config_dir): os.makedirs(config_dir) + network_config = network_config_map[args.network] + client_path = get_client_path([x.strip() for x in args.executable_dirs.split(',')], args.docker, network_config, + args.verbose) + logger.debug("Client command is {}".format(client_path)) + config_file = None for file in os.listdir(config_dir): if file.endswith(".yaml"): @@ -315,13 +322,27 @@ def main(args): raise Exception( "Unable to find any '.yaml' configuration files inside configuration directory({})".format(config_dir)) + master_config_file_path = os.path.join(config_dir, "master.yaml") config_file_path = os.path.join(config_dir, config_file) logger.info("Loading configuration file {}".format(config_file_path)) - managers = {'tz1boot1pK9h2BVGXdyvfQSv8kd1LQM6H889': 'tz1boot1pK9h2BVGXdyvfQSv8kd1LQM6H889'} - known_contracts = {} - - parser = AppYamlConfParser(ConfigParser.load_file(config_file_path), known_contracts, managers) + master_cfg={} + if os.path.isfile(master_config_file_path): + master_parser = YamlConfParser(ConfigParser.load_file(master_config_file_path)) + master_cfg = master_parser.parse() + + managers = None + contracts_by_alias = None + addresses_by_pkh = None + if 'managers' in master_cfg: + managers = master_cfg['managers'] + if 'contracts_by_alias' in master_cfg: + contracts_by_alias = master_cfg['contracts_by_alias'] + if 'addresses_by_pkh' in master_cfg: + addresses_by_pkh = master_cfg['addresses_by_pkh'] + + wllt_clnt_mngr = WalletClientManager(client_path, contracts_by_alias, addresses_by_pkh, managers) + parser = AppYamlConfParser(ConfigParser.load_file(config_file_path), wllt_clnt_mngr) parser.parse() parser.validate() cfg = parser.get_conf_obj() @@ -351,10 +372,6 @@ def main(args): run_mode = RunMode(args.run_mode) node_addr = args.node_addr payment_offset = args.payment_offset - network_config = network_config_map[args.network] - client_path = get_client_path([x.strip() for x in args.executable_dirs.split(',')], args.docker, network_config, - args.verbose) - logger.debug("Client command is {}".format(client_path)) validate_release_override(args.release_override) @@ -391,7 +408,7 @@ def main(args): for i in range(NB_CONSUMERS): c = PaymentConsumer(name='consumer' + str(i), payments_dir=payments_root, key_name=payment_address, client_path=client_path, payments_queue=payments_queue, node_addr=node_addr, - verbose=args.verbose, dry_run=dry_run) + wllt_clnt_mngr=wllt_clnt_mngr, verbose=args.verbose, dry_run=dry_run) time.sleep(1) c.start() try: diff --git a/src/pay/batch_payer.py b/src/pay/batch_payer.py index 1fe8537a..43ebb97b 100644 --- a/src/pay/batch_payer.py +++ b/src/pay/batch_payer.py @@ -4,23 +4,23 @@ import base58 import os from log_config import main_logger -from util.client_utils import client_list_known_contracts, sign, check_response, send_request +from util.client_utils import check_response from util.rpc_utils import parse_json_response import configparser logger = main_logger -COMM_HEAD = "{} rpc get http://{}/chains/main/blocks/head" -COMM_COUNTER = "{} rpc get http://{}/chains/main/blocks/head/context/contracts/{}/counter" +COMM_HEAD = " rpc get http://{}/chains/main/blocks/head" +COMM_COUNTER = " rpc get http://{}/chains/main/blocks/head/context/contracts/{}/counter" CONTENT = '{"kind":"transaction","source":"%SOURCE%","destination":"%DESTINATION%","fee":"%fee%","counter":"%COUNTER%","gas_limit": "%gas_limit%", "storage_limit": "%storage_limit%","amount":"%AMOUNT%"}' FORGE_JSON = '{"branch": "%BRANCH%","contents":[%CONTENT%]}' RUNOPS_JSON = '{"branch": "%BRANCH%","contents":[%CONTENT%], "signature":"edsigtXomBKi5CTRf5cjATJWSyaRvhfYNHqSUGrn4SdbYRcGwQrUGjzEfQDTuqHhuA8b2d8NarZjz8TRf65WkpQmo423BtomS8Q"}' PREAPPLY_JSON = '[{"protocol":"%PROTOCOL%","branch":"%BRANCH%","contents":[%CONTENT%],"signature":"%SIGNATURE%"}]' -COMM_FORGE = "{} rpc post http://%NODE%/chains/main/blocks/head/helpers/forge/operations with '%JSON%'" -COMM_RUNOPS = "{} rpc post http://%NODE%/chains/main/blocks/head/helpers/scripts/run_operation with '%JSON%'" -COMM_PREAPPLY = "{} rpc post http://%NODE%/chains/main/blocks/head/helpers/preapply/operations with '%JSON%'" -COMM_INJECT = "{} %LOG% rpc post http://%NODE%/injection/operation with '\"%OPERATION_HASH%\"'" -COMM_WAIT = "{} wait for %OPERATION% to be included ---confirmations 5" +COMM_FORGE = " rpc post http://%NODE%/chains/main/blocks/head/helpers/forge/operations with '%JSON%'" +COMM_RUNOPS = " rpc post http://%NODE%/chains/main/blocks/head/helpers/scripts/run_operation with '%JSON%'" +COMM_PREAPPLY = " rpc post http://%NODE%/chains/main/blocks/head/helpers/preapply/operations with '%JSON%'" +COMM_INJECT = " %LOG% rpc post http://%NODE%/injection/operation with '\"%OPERATION_HASH%\"'" +COMM_WAIT = " wait for %OPERATION% to be included ---confirmations 5" MAX_TX_PER_BLOCK = 280 PKH_LENGHT = 36 @@ -29,11 +29,11 @@ class BatchPayer(): - def __init__(self, node_url, client_path, key_name): + def __init__(self, node_url, pymnt_addr, wllt_clnt_mngr): super(BatchPayer, self).__init__() - self.key_name = key_name + self.pymnt_addr = pymnt_addr self.node_url = node_url - self.client_path = client_path + self.wllt_clnt_mngr = wllt_clnt_mngr config = configparser.ConfigParser() if os.path.isfile(FEE_INI): @@ -46,25 +46,27 @@ def __init__(self, node_url, client_path, key_name): self.gas_limit = kttx['gas_limit'] self.storage_limit = kttx['storage_limit'] self.default_fee = kttx['fee'] - self.delegator_pays_xfer_fee = config.getboolean('KTTX', 'delegator_pays_xfer_fee', fallback=True) # Must use getboolean otherwise parses as string + self.delegator_pays_xfer_fee = config.getboolean('KTTX', 'delegator_pays_xfer_fee', + fallback=True) # Must use getboolean otherwise parses as string - # key_name has a length of 36 and starts with tz or KT then it is a public key has, else it is an alias - if len(self.key_name) == PKH_LENGHT and (self.key_name.startswith("KT") or self.key_name.startswith("tz")): - self.source = self.key_name + # pymnt_addr has a length of 36 and starts with tz or KT then it is a public key has, else it is an alias + if len(self.pymnt_addr) == PKH_LENGHT and ( + self.pymnt_addr.startswith("KT") or self.pymnt_addr.startswith("tz")): + self.source = self.pymnt_addr else: - known_contracts = client_list_known_contracts(self.client_path) - if self.key_name in known_contracts: - self.source = known_contracts[self.key_name] + known_contracts = self.wllt_clnt_mngr.client_list_known_contracts() + if self.pymnt_addr in known_contracts: + self.source = known_contracts[self.pymnt_addr] else: - raise Exception("key_name cannot be translated into a PKH or alias: {}".format(self.key_name)) + raise Exception("pymnt_addr cannot be translated into a PKH or alias: {}".format(self.pymnt_addr)) - self.comm_head = COMM_HEAD.format(self.client_path, self.node_url) - self.comm_counter = COMM_COUNTER.format(self.client_path, self.node_url, self.source) - self.comm_runops = COMM_RUNOPS.format(self.client_path).replace("%NODE%", self.node_url) - self.comm_forge = COMM_FORGE.format(self.client_path).replace("%NODE%", self.node_url) - self.comm_preapply = COMM_PREAPPLY.format(self.client_path).replace("%NODE%", self.node_url) - self.comm_inject = COMM_INJECT.format(self.client_path).replace("%NODE%", self.node_url) - self.comm_wait = COMM_WAIT.format(self.client_path) + self.comm_head = COMM_HEAD.format(self.node_url) + self.comm_counter = COMM_COUNTER.format(self.node_url, self.source) + self.comm_runops = COMM_RUNOPS.format().replace("%NODE%", self.node_url) + self.comm_forge = COMM_FORGE.format().replace("%NODE%", self.node_url) + self.comm_preapply = COMM_PREAPPLY.format().replace("%NODE%", self.node_url) + self.comm_inject = COMM_INJECT.format().replace("%NODE%", self.node_url) + self.comm_wait = COMM_WAIT.format() def pay(self, payment_items, verbose=None, dry_run=None): # split payments into lists of MAX_TX_PER_BLOCK or less size @@ -114,10 +116,10 @@ def wait_random(self): sleep(slp_tm) def pay_single_batch(self, payment_records, verbose=None, dry_run=None): - counter = parse_json_response(send_request(self.comm_counter, verbose)) + counter = parse_json_response(self.wllt_clnt_mngr.send_request(self.comm_counter)) counter = int(counter) - head = parse_json_response(send_request(self.comm_head, verbose)) + head = parse_json_response(self.wllt_clnt_mngr.send_request(self.comm_head)) branch = head["hash"] protocol = head["metadata"]["protocol"] @@ -150,7 +152,7 @@ def pay_single_batch(self, payment_records, verbose=None, dry_run=None): runops_json = RUNOPS_JSON.replace('%BRANCH%', branch).replace("%CONTENT%", contents_string) runops_command_str = self.comm_runops.replace("%JSON%", runops_json) if verbose: logger.debug("runops_command_str is |{}|".format(runops_command_str)) - runops_command_response = send_request(runops_command_str, verbose) + runops_command_response = self.wllt_clnt_mngr.send_request(runops_command_str) if not check_response(runops_command_response): error_desc = parse_json_response(runops_command_response) # for content in runops_command_response["contents"]: @@ -166,14 +168,14 @@ def pay_single_batch(self, payment_records, verbose=None, dry_run=None): forge_json = FORGE_JSON.replace('%BRANCH%', branch).replace("%CONTENT%", contents_string) forge_command_str = self.comm_forge.replace("%JSON%", forge_json) if verbose: logger.debug("forge_command_str is |{}|".format(forge_command_str)) - forge_command_response = send_request(forge_command_str, verbose) + forge_command_response = self.wllt_clnt_mngr.send_request(forge_command_str) if not check_response(forge_command_response): logger.error("Error in forge response '{}'".format(forge_command_response)) return False, "" # sign the operations bytes = parse_json_response(forge_command_response, verbose=verbose) - signed_bytes = sign(self.client_path, bytes, self.key_name, verbose=verbose) + signed_bytes = self.wllt_clnt_mngr.sign(bytes, self.pymnt_addr, verbose=verbose) # pre-apply operations logger.debug("Preapplying the operations") @@ -182,7 +184,7 @@ def pay_single_batch(self, payment_records, verbose=None, dry_run=None): preapply_command_str = self.comm_preapply.replace("%JSON%", preapply_json) if verbose: logger.debug("preapply_command_str is |{}|".format(preapply_command_str)) - preapply_command_response = send_request(preapply_command_str, verbose) + preapply_command_response = self.wllt_clnt_mngr.send_request(preapply_command_str) if not check_response(preapply_command_response): logger.error("Error in preapply response '{}'".format(preapply_command_response)) return False, "" @@ -213,7 +215,7 @@ def pay_single_batch(self, payment_records, verbose=None, dry_run=None): inject_command_str = self.comm_inject.replace("%OPERATION_HASH%", signed_operation_bytes) inject_command_str = inject_command_str.replace("%LOG%", "-l" if verbose else "") if verbose: logger.debug("inject_command_str is |{}|".format(inject_command_str)) - inject_command_response = send_request(inject_command_str, verbose) + inject_command_response = self.wllt_clnt_mngr.send_request(inject_command_str) if not check_response(inject_command_response): logger.error("Error in inject response '{}'".format(inject_command_response)) return False, "" @@ -223,7 +225,7 @@ def pay_single_batch(self, payment_records, verbose=None, dry_run=None): # wait for inclusion logger.debug("Waiting for operation {} to be included".format(operation_hash)) - send_request(self.comm_wait.replace("%OPERATION%", operation_hash), verbose) + self.wllt_clnt_mngr.send_request(self.comm_wait.replace("%OPERATION%", operation_hash)) logger.debug("Operation {} is included".format(operation_hash)) return True, operation_hash diff --git a/src/pay/payment_consumer.py b/src/pay/payment_consumer.py index 1962dd9b..ef8965a6 100644 --- a/src/pay/payment_consumer.py +++ b/src/pay/payment_consumer.py @@ -25,7 +25,7 @@ def count_and_log_failed(payment_logs, pymnt_cycle): class PaymentConsumer(threading.Thread): - def __init__(self, name, payments_dir, key_name, client_path, payments_queue, node_addr, verbose=None, + def __init__(self, name, payments_dir, key_name, client_path, payments_queue, node_addr, wllt_clnt_mngr, verbose=None, dry_run=None): super(PaymentConsumer, self).__init__() @@ -38,6 +38,7 @@ def __init__(self, name, payments_dir, key_name, client_path, payments_queue, no self.verbose = verbose self.dry_run = dry_run self.mm = EmailManager() + self.wllt_clnt_mngr=wllt_clnt_mngr logger.debug('Consumer "%s" created', self.name) @@ -64,7 +65,7 @@ def run(self): # payment_log = regular_payer.pay(payment_items[0], self.verbose, dry_run=self.dry_run) # payment_logs = [payment_log] - batch_payer = BatchPayer(self.node_addr, self.client_path, self.key_name) + batch_payer = BatchPayer(self.node_addr, self.key_name, self.wllt_clnt_mngr) # 3- do the payment payment_logs = batch_payer.pay(payment_items, self.verbose, dry_run=self.dry_run) diff --git a/src/rpc/rpc_reward_calculator_will_come_here.txt b/src/rpc/rpc_reward_calculator_will_come_here.txt new file mode 100644 index 00000000..e69de29b diff --git a/src/test/client_response_parse.py b/src/test/client_response_parse.py index 4c396fc3..2c9ee9f9 100644 --- a/src/test/client_response_parse.py +++ b/src/test/client_response_parse.py @@ -55,6 +55,73 @@ "BMeLBs4C2RbZos4anjmGvP3GMa6hA7froTpxZN7HsNtRZyvK72r" """ +test_str = """ +Warning: + + This is NOT the Tezos Mainnet. + The Tezos Mainnet is not yet released. + + The node you are connecting to claims to be running on the + Tezos Betanet EXPERIMENTAL NETWORK. + Betanet is a pre-release experimental network and comes with no warranty. + Use your fundraiser keys on this network AT YOUR OWN RISK. + All transactions happening on the Betanet are expected to be valid in the Mainnet. + If in doubt, we recommend that you wait for the Mainnet lunch. + +Error: + Rpc request failed: + - meth: POST + - uri: http://localhost:8732/chains/main/blocks/head/helpers/preapply/operations + - error: Oups! It looks like we forged an invalid HTTP request. + [ { "protocol": "PsYLVpVvgbLhAhoqAkMFUo6gudkJ9weNXhUYCiLDzcUpFpkk8Wt", + "branch": "BLh62ZiNsBiLnQZiuUsQzTdXkjWgeLAMryYJywV9z4wCZsjTL8h", + "contents": + [ { "kind": "transaction", + "source": "tz1aZoYGSEoGpzWmitPaCJw6HQCkz5YSi1ow", + "destination": "KT1PEZ91VnphKodWSfuvCcjXrA29zfHsgUxt", "fee": "0", + "counter": "316261", "gas_limit": "200", "storage_limit": "0", + "amount": "31431000" }, + { "kind": "transaction", + "source": "tz1aZoYGSEoGpzWmitPaCJw6HQCkz5YSi1ow", + "destination": "KT1Ao8UXNJ9Dz71Wx3m8yzYNdnNQp2peqtMc", "fee": "0", + "counter": "316262", "gas_limit": "200", "storage_limit": "0", + "amount": "3117000" }, + { "kind": "transaction", + "source": "tz1aZoYGSEoGpzWmitPaCJw6HQCkz5YSi1ow", + "destination": "KT18bVwvyLBR1GAM1rBoiHzEXVNtXb5C3vEU", "fee": "0", + "counter": "316263", "gas_limit": "200", "storage_limit": "0", + "amount": "2830000" }, + { "kind": "transaction", + "source": "tz1aZoYGSEoGpzWmitPaCJw6HQCkz5YSi1ow", + "destination": "KT1HuhLZ3Rg45bRnSVssA6KEVXqbKbjzsmPH", "fee": "0", + "counter": "316264", "gas_limit": "200", "storage_limit": "0", + "amount": "1000" }, + { "kind": "transaction", + "source": "tz1aZoYGSEoGpzWmitPaCJw6HQCkz5YSi1ow", + "destination": "KT18bVwvyLBR1GAM1rBoiHzEXVNtXb5C3vEU", "fee": "0", + "counter": "316265", "gas_limit": "200", "storage_limit": "0", + "amount": "40000" }, + { "kind": "transaction", + "source": "tz1aZoYGSEoGpzWmitPaCJw6HQCkz5YSi1ow", + "destination": "KT1PEZ91VnphKodWSfuvCcjXrA29zfHsgUxt", "fee": "0", + "counter": "316266", "gas_limit": "200", "storage_limit": "0", + "amount": "431000" }, + { "kind": "transaction", + "source": "tz1aZoYGSEoGpzWmitPaCJw6HQCkz5YSi1ow", + "destination": "KT18bVwvyLBR1GAM1rBoiHzEXVNtXb5C3vEU", "fee": "0", + "counter": "316267", "gas_limit": "200", "storage_limit": "0", + "amount": "75000" }, + { "kind": "transaction", + "source": "tz1aZoYGSEoGpzWmitPaCJw6HQCkz5YSi1ow", + "destination": "KT1PEZ91VnphKodWSfuvCcjXrA29zfHsgUxt", "fee": "0", + "counter": "316268", "gas_limit": "200", "storage_limit": "0", + "amount": "75000" } ], + "signature": + "edsigtiGP1KZTXrjwbUtNB2FiLKLD4zttATB73XGpkPcvicfMnSo7wBgQiWDUKh9aaeLsQywNVGBRW8aTF8Jh9PmwkCn6BF6b" } ] +""" + + + def strip_disclaimer(client_response): idx = client_response.find("{") diff --git a/src/util/client_utils.py b/src/util/client_utils.py index 648798dd..31e69182 100644 --- a/src/util/client_utils.py +++ b/src/util/client_utils.py @@ -1,78 +1,10 @@ -import subprocess -import re - import os +import re DOCKER_CLIENT_EXE = "%network%.sh" DOCKER_CLIENT_EXE_SUFFIX = " client" REGULAR_CLIENT_EXE = "tezos-client" -test_str = """ -Warning: - - This is NOT the Tezos Mainnet. - The Tezos Mainnet is not yet released. - - The node you are connecting to claims to be running on the - Tezos Betanet EXPERIMENTAL NETWORK. - Betanet is a pre-release experimental network and comes with no warranty. - Use your fundraiser keys on this network AT YOUR OWN RISK. - All transactions happening on the Betanet are expected to be valid in the Mainnet. - If in doubt, we recommend that you wait for the Mainnet lunch. - -Error: - Rpc request failed: - - meth: POST - - uri: http://localhost:8732/chains/main/blocks/head/helpers/preapply/operations - - error: Oups! It looks like we forged an invalid HTTP request. - [ { "protocol": "PsYLVpVvgbLhAhoqAkMFUo6gudkJ9weNXhUYCiLDzcUpFpkk8Wt", - "branch": "BLh62ZiNsBiLnQZiuUsQzTdXkjWgeLAMryYJywV9z4wCZsjTL8h", - "contents": - [ { "kind": "transaction", - "source": "tz1aZoYGSEoGpzWmitPaCJw6HQCkz5YSi1ow", - "destination": "KT1PEZ91VnphKodWSfuvCcjXrA29zfHsgUxt", "fee": "0", - "counter": "316261", "gas_limit": "200", "storage_limit": "0", - "amount": "31431000" }, - { "kind": "transaction", - "source": "tz1aZoYGSEoGpzWmitPaCJw6HQCkz5YSi1ow", - "destination": "KT1Ao8UXNJ9Dz71Wx3m8yzYNdnNQp2peqtMc", "fee": "0", - "counter": "316262", "gas_limit": "200", "storage_limit": "0", - "amount": "3117000" }, - { "kind": "transaction", - "source": "tz1aZoYGSEoGpzWmitPaCJw6HQCkz5YSi1ow", - "destination": "KT18bVwvyLBR1GAM1rBoiHzEXVNtXb5C3vEU", "fee": "0", - "counter": "316263", "gas_limit": "200", "storage_limit": "0", - "amount": "2830000" }, - { "kind": "transaction", - "source": "tz1aZoYGSEoGpzWmitPaCJw6HQCkz5YSi1ow", - "destination": "KT1HuhLZ3Rg45bRnSVssA6KEVXqbKbjzsmPH", "fee": "0", - "counter": "316264", "gas_limit": "200", "storage_limit": "0", - "amount": "1000" }, - { "kind": "transaction", - "source": "tz1aZoYGSEoGpzWmitPaCJw6HQCkz5YSi1ow", - "destination": "KT18bVwvyLBR1GAM1rBoiHzEXVNtXb5C3vEU", "fee": "0", - "counter": "316265", "gas_limit": "200", "storage_limit": "0", - "amount": "40000" }, - { "kind": "transaction", - "source": "tz1aZoYGSEoGpzWmitPaCJw6HQCkz5YSi1ow", - "destination": "KT1PEZ91VnphKodWSfuvCcjXrA29zfHsgUxt", "fee": "0", - "counter": "316266", "gas_limit": "200", "storage_limit": "0", - "amount": "431000" }, - { "kind": "transaction", - "source": "tz1aZoYGSEoGpzWmitPaCJw6HQCkz5YSi1ow", - "destination": "KT18bVwvyLBR1GAM1rBoiHzEXVNtXb5C3vEU", "fee": "0", - "counter": "316267", "gas_limit": "200", "storage_limit": "0", - "amount": "75000" }, - { "kind": "transaction", - "source": "tz1aZoYGSEoGpzWmitPaCJw6HQCkz5YSi1ow", - "destination": "KT1PEZ91VnphKodWSfuvCcjXrA29zfHsgUxt", "fee": "0", - "counter": "316268", "gas_limit": "200", "storage_limit": "0", - "amount": "75000" } ], - "signature": - "edsigtiGP1KZTXrjwbUtNB2FiLKLD4zttATB73XGpkPcvicfMnSo7wBgQiWDUKh9aaeLsQywNVGBRW8aTF8Jh9PmwkCn6BF6b" } ] -""" - - def get_client_path(search_paths, docker=None, network_config=None, verbose=None): client_exe = REGULAR_CLIENT_EXE if docker: @@ -86,28 +18,6 @@ def get_client_path(search_paths, docker=None, network_config=None, verbose=None raise Exception("Client executable not found. Review --executable_dirs, --docker and --network parameters") - -def send_request(cmd, verbose=None): - if verbose: - print("Command is |{}|".format(cmd)) - - # execute client - process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - bytes = [] - for b in process.stdout: - bytes.append(b) - - process.wait() - - buffer = b''.join(bytes).decode('utf-8') - - if verbose: - print("Answer is |{}|".format(buffer)) - - return buffer - - def check_response(response): if "Error:" in response or "error" in response or "invalid" in response or "Unexpected server answer" in response: return False @@ -130,103 +40,5 @@ def clear_terminal_chars(content): result = ansi_escape.sub('', content) return result - -def get_manager_for_contract(client_cmd, pkh, verbose): - response = send_request(client_cmd + " get manager for " + pkh, verbose) - - response = clear_terminal_chars(response) - - return dict - - -def parse_get_manager_for_contract_response(response, verbose=None): - manager = None - for line in response.splitlines(): - line = line.strip() - if line.startswith("tz"): - line = line.replace(" (", ':') - manager, alias_plus = line.split(":", maxsplit=1) - break - - if verbose: - print("Manager address is : {}".format(manager)) - - return manager - - -def client_generate_address_dict(client_cmd, verbose: None): - contr_dict = client_list_known_contracts(client_cmd, verbose) - addr_dict = client_list_known_addresses(client_cmd, verbose) - - for alias, pkh in contr_dict.items(): - if pkh.startswith("KT"): - manager = get_manager_for_contract(client_cmd, pkh, verbose) - manager_sk = addr_dict[manager]['sk'] - addr_dict[pkh] = {"pkh": pkh, "originated": True, "alias": alias, "sk": manager_sk, "manager": manager} - - -def client_list_known_contracts(client_cmd, verbose=None): - response = send_request(client_cmd + " list known contracts", verbose) - - response = clear_terminal_chars(response) - - dict = parse_client_list_known_contracts_response(response, verbose) - - return dict - - -def parse_client_list_known_contracts_response(response, verbose=None): - dict = {} - for line in response.splitlines(): - line = line.strip() - if ":" in line and not_indicator_line(line): - alias, pkh = line.split(":", maxsplit=1) - dict[alias.strip()] = pkh.strip() - if verbose: - print("known contracts: {}".format(dict)) - return dict - - -def client_list_known_addresses(client_cmd, verbose=None): - response = send_request(client_cmd + " list known addresses", verbose) - - response = clear_terminal_chars(response) - - dict = parse_list_known_addresses_response(response, verbose) - - return dict - - def not_indicator_line(line): return "Warning" not in line[0:15] and "Disclaimer" not in line[0:15] - - -def parse_list_known_addresses_response(response, verbose=None): - dict = {} - - for line in response.splitlines(): - line = line.strip() - if ":" in line and not_indicator_line(line): - alias, pkh_plus_braces = line.split(":", maxsplit=1) - pkh_plus_braces = pkh_plus_braces.replace(' (', ':') - pkh, sk_section = pkh_plus_braces.split(":", maxsplit=1) - sk_known = "sk known" in sk_section - pkh = pkh.strip() - alias = alias.strip() - dict[pkh] = {"pkh": pkh, "originated": False, "alias": alias, "sk": sk_known, "manager": pkh} - if verbose: - print("known addresses: {}".format(dict)) - - return dict - - -def sign(client_cmd, bytes, key_name, verbose=None): - response = send_request(client_cmd + " sign bytes 0x03{} for {}".format(bytes, key_name), verbose) - - response = clear_terminal_chars(response) - - for line in response.splitlines(): - if "Signature" in line: - return line.strip("Signature:").strip() - - raise Exception("Signature not found in response '{}'".format(response)) diff --git a/src/util/test_parse_list_known_addresses_response.py b/src/util/test_parse_list_known_addresses_response.py deleted file mode 100644 index 2d79430d..00000000 --- a/src/util/test_parse_list_known_addresses_response.py +++ /dev/null @@ -1,77 +0,0 @@ -from unittest import TestCase - -from util.client_utils import parse_list_known_addresses_response, parse_get_manager_for_contract_response, \ - parse_client_list_known_contracts_response - - -class TestParse_client_utils(TestCase): - def test_parse_list_known_addresses_response(self): - response = """ - Disclaimer: - The Tezos network is a new blockchain technology. - Users are solely responsible for any risks associated - with usage of the Tezos network. Users should do their - own research to determine if Tezos is the appropriate - platform for their needs and should apply judgement and - care in their network interactions. - - habanoz: tz1fyvFH2pd3V9UEq5psqVokVBYkt7rHTKio (unencrypted sk known) - mainnetme: tz1a5GGJeyqeQ4ihZqbiRVcvj5rY5kMAt3Xa (tcp sk known) - zeronetme: tz1MZ72sJEVen3Qgc7uWvqKhKFJW84bNGd6T (unencrypted sk not known) - """ - - dict = parse_list_known_addresses_response(response) - - habanoz = dict['tz1fyvFH2pd3V9UEq5psqVokVBYkt7rHTKio'] - - self.assertEqual(habanoz['alias'], 'habanoz') - self.assertEqual(habanoz['sk'], True) - self.assertEqual(habanoz['pkh'], 'tz1fyvFH2pd3V9UEq5psqVokVBYkt7rHTKio') - - mainnetme = dict['tz1a5GGJeyqeQ4ihZqbiRVcvj5rY5kMAt3Xa'] - - self.assertEqual(mainnetme['alias'], 'mainnetme') - self.assertEqual(mainnetme['sk'], True) - self.assertEqual(mainnetme['pkh'], 'tz1a5GGJeyqeQ4ihZqbiRVcvj5rY5kMAt3Xa') - - zeronetme = dict['tz1MZ72sJEVen3Qgc7uWvqKhKFJW84bNGd6T'] - - self.assertEqual(zeronetme['alias'], 'zeronetme') - self.assertEqual(zeronetme['sk'], False) - self.assertEqual(zeronetme['pkh'], 'tz1MZ72sJEVen3Qgc7uWvqKhKFJW84bNGd6T') - - def test_get_manager(self): - response = """ - Disclaimer: - The Tezos network is a new blockchain technology. - Users are solely responsible for any risks associated - with usage of the Tezos network. Users should do their - own research to determine if Tezos is the appropriate - platform for their needs and should apply judgement and - care in their network interactions. - -tz1PJnnddwa1P5jg5sChddigxfMccK8nwLiV (known as habanoz) - """ - - manager = parse_get_manager_for_contract_response(response) - self.assertEqual('tz1PJnnddwa1P5jg5sChddigxfMccK8nwLiV', manager) - - def test_parse_client_list_known_contracts_response(self): - response = """ - Disclaimer: - The Tezos network is a new blockchain technology. - Users are solely responsible for any risks associated - with usage of the Tezos network. Users should do their - own research to determine if Tezos is the appropriate - platform for their needs and should apply judgement and - care in their network interactions. - -newcontr: KT1XqEHigP5XumZy9i76QyVd6u93VD4HTqJK -habanoz: tz1fyvFH2pd3V9UEq5psqVokVBYkt7rHTKio -mainnetme: tz1a5GGJeyqeQ4ihZqbiRVcvj5rY5kMAt3Xa - """ - - dict = parse_client_list_known_contracts_response(response) - self.assertTrue(dict['newcontr']=='KT1XqEHigP5XumZy9i76QyVd6u93VD4HTqJK') - self.assertTrue(dict['habanoz']=='tz1fyvFH2pd3V9UEq5psqVokVBYkt7rHTKio') - self.assertTrue(dict['mainnetme']=='tz1a5GGJeyqeQ4ihZqbiRVcvj5rY5kMAt3Xa') \ No newline at end of file