From 4494abf7e2f874d18f57757560d21f10dbb953bc Mon Sep 17 00:00:00 2001 From: Huseyin ABANOZ Date: Sat, 30 Nov 2019 15:14:58 +0300 Subject: [PATCH 1/2] - retry rpc call - ssl option for email --- src/emails/email_manager.py | 15 ++++++++--- src/emails/email_worker.py | 14 ++++++++-- src/rpc/rpc_reward_api.py | 54 +++++++++++++++++++++++++------------ 3 files changed, 61 insertions(+), 22 deletions(-) diff --git a/src/emails/email_manager.py b/src/emails/email_manager.py index bd20f2ba..b7f1e935 100644 --- a/src/emails/email_manager.py +++ b/src/emails/email_manager.py @@ -13,6 +13,7 @@ HOST = "smtp.host" PORT = "smtp.port" SENDER = "sender" +USE_SSL = "use.ssl" EMAIL_INI_PATH = "./email.ini" @@ -29,13 +30,22 @@ def __init__(self): config.read(EMAIL_INI_PATH) default = config['DEFAULT'] + + use_ssl = self.getUseSslOrFalse(default) + self.default = default if self.all_set(default): - self.email_sender = EmailSender(default[HOST], int(default[PORT]), default[USER], - default[PASS], default[SENDER]) + self.email_sender = EmailSender(default[HOST], int(default[PORT]), default[USER], default[PASS], + default[SENDER], use_ssl) else: logger.info("If you want to send emails, populate email.ini file under current working directory.") + def getUseSslOrFalse(self, default): + use_ssl = False + if USE_SSL in default: + use_ssl = default[USE_SSL] + return use_ssl + def all_set(self, default): return default[USER] and default[PASS] and default[HOST] and default[PORT] and default[SENDER] and default[ RECIPIENTS] @@ -60,4 +70,3 @@ def send_payment_mail(self, cyle, payments_file, nb_failed, nb_unknown): self.default["recipients"], [payments_file]) logger.debug("Report email sent for cycle {}.".format(cyle)) - diff --git a/src/emails/email_worker.py b/src/emails/email_worker.py index 2d60c1c2..eda9dfcd 100644 --- a/src/emails/email_worker.py +++ b/src/emails/email_worker.py @@ -7,8 +7,9 @@ class EmailSender(): - def __init__(self, host, port, user, passw, sender): + def __init__(self, host, port, user, passw, sender, use_ssl: False): super(EmailSender, self).__init__() + self.use_ssl = use_ssl self.host = host self.port = port self.sender = sender @@ -36,7 +37,16 @@ def send(self, title, message, recipients, attachments): part['Content-Disposition'] = 'attachment; filename="%s"' % basename(f) msg.attach(part) - smtp = smtplib.SMTP(self.host, self.port) + if self.use_ssl: + smtp = smtplib.SMTP_SSL(self.host, self.port) + else: + smtp = smtplib.SMTP(self.host, self.port) + smtp.login(self.user, self.password) smtp.sendmail(self.sender, recipient_string, msg.as_string()) smtp.close() + + +if __name__ == '__main__': + sender = EmailSender("---", 587, "---", "---", "---") + sender.send("---", "---", "---", []) diff --git a/src/rpc/rpc_reward_api.py b/src/rpc/rpc_reward_api.py index b350afac..74039cf1 100644 --- a/src/rpc/rpc_reward_api.py +++ b/src/rpc/rpc_reward_api.py @@ -10,7 +10,6 @@ class RpcRewardApiImpl(RewardApi): - COMM_HEAD = "{}/chains/main/blocks/head" COMM_DELEGATES = "{}/chains/main/blocks/{}/context/delegates/{}" COMM_BLOCK = "{}/chains/main/blocks/{}" @@ -47,13 +46,17 @@ def get_rewards_for_cycle_map(self, cycle): logger.debug("Current level {}, current cycle {}".format(current_level, current_cycle)) reward_data = {} - reward_data["delegate_staking_balance"], reward_data["delegators"] = self.__get_delegators_and_delgators_balance(cycle, current_level) + reward_data["delegate_staking_balance"], reward_data[ + "delegators"] = self.__get_delegators_and_delgators_balance(cycle, current_level) reward_data["delegators_nb"] = len(reward_data["delegators"]) # Get last block in cycle where rewards are unfrozen - level_of_last_block_in_unfreeze_cycle = (cycle+self.preserved_cycles+1) * self.blocks_per_cycle + level_of_last_block_in_unfreeze_cycle = (cycle + self.preserved_cycles + 1) * self.blocks_per_cycle - logger.debug("Cycle {}, preserved cycles {}, blocks per cycle {}, last_block_cycle {}".format(cycle, self.preserved_cycles, self.blocks_per_cycle, level_of_last_block_in_unfreeze_cycle)) + logger.debug("Cycle {}, preserved cycles {}, blocks per cycle {}, last_block_cycle {}".format(cycle, + self.preserved_cycles, + self.blocks_per_cycle, + level_of_last_block_in_unfreeze_cycle)) if current_level - level_of_last_block_in_unfreeze_cycle >= 0: unfrozen_rewards = self.__get_unfrozen_rewards(level_of_last_block_in_unfreeze_cycle, cycle) @@ -63,9 +66,11 @@ def get_rewards_for_cycle_map(self, cycle): logger.warn("Please wait until the rewards and fees for cycle {} are unfrozen".format(cycle)) reward_data["total_rewards"] = 0 - reward_model = RewardProviderModel(reward_data["delegate_staking_balance"], reward_data["total_rewards"], reward_data["delegators"]) + reward_model = RewardProviderModel(reward_data["delegate_staking_balance"], reward_data["total_rewards"], + reward_data["delegators"]) - logger.debug("delegate_staking_balance={}, total_rewards = {}".format(reward_data["delegate_staking_balance"],reward_data["total_rewards"])) + logger.debug("delegate_staking_balance={}, total_rewards = {}".format(reward_data["delegate_staking_balance"], + reward_data["total_rewards"])) logger.debug("delegators = {}".format(reward_data["delegators"])) return reward_model @@ -83,24 +88,29 @@ def __get_unfrozen_rewards(self, level_of_last_block_in_unfreeze_cycle, cycle): if int(balance_update["cycle"]) == cycle or int(balance_update["change"]) < 0: if balance_update["category"] == "rewards": unfrozen_rewards = -int(balance_update["change"]) - logger.debug("[__get_unfrozen_rewards] Found balance update for reward {}".format(balance_update)) + logger.debug( + "[__get_unfrozen_rewards] Found balance update for reward {}".format(balance_update)) elif balance_update["category"] == "fees": unfrozen_fees = -int(balance_update["change"]) - logger.debug("[__get_unfrozen_rewards] Found balance update for fee {}".format(balance_update)) + logger.debug( + "[__get_unfrozen_rewards] Found balance update for fee {}".format(balance_update)) else: - logger.debug("[__get_unfrozen_rewards] Found balance update, not including: {}".format(balance_update)) + logger.debug("[__get_unfrozen_rewards] Found balance update, not including: {}".format( + balance_update)) else: - logger.debug("[__get_unfrozen_rewards] Found balance update, cycle does not match or change is non-zero, not including: {}".format(balance_update)) + logger.debug( + "[__get_unfrozen_rewards] Found balance update, cycle does not match or change is non-zero, not including: {}".format( + balance_update)) return unfrozen_fees + unfrozen_rewards - def do_rpc_request(self, request): + def do_rpc_request(self, request, time_out=120): if self.verbose: logger.debug("[do_rpc_request] Requesting URL {}".format(request)) - sleep(0.1) # be nice to public node service + sleep(0.1) # be nice to public node service - resp = requests.get(request) + resp = requests.get(request, timeout=time_out) if resp.status_code != 200: raise Exception("Request '{} failed with status code {}".format(request, resp.status_code)) @@ -117,8 +127,6 @@ def __get_current_level(self): return current_level, current_cycle - - def __get_delegators_and_delgators_balance(self, cycle, current_level): hash_snapshot_block = self.__get_snapshot_block_hash(cycle, current_level) @@ -138,6 +146,17 @@ def __get_delegators_and_delgators_balance(self, cycle, current_level): for idx, delegator in enumerate(delegators_addresses): request = self.COMM_DELEGATE_BALANCE.format(self.node_url, hash_snapshot_block, delegator) response = self.do_rpc_request(request) + + sleep(0.5) # be nice to public node service + + response = None + + while not response: + try: + response = self.do_rpc_request(request, time_out=5) + except: + logger.error("Fetching delegator info failed {}, will retry", delegator) + delegators[delegator] = int(response["balance"]) logger.debug( @@ -151,7 +170,7 @@ def __get_delegators_and_delgators_balance(self, cycle, current_level): def __get_snapshot_block_hash(self, cycle, current_level): snapshot_level = (cycle - self.preserved_cycles) * self.blocks_per_cycle + 1 - logger.debug("Reward cycle {}, snapshot level {}".format(cycle,snapshot_level)) + logger.debug("Reward cycle {}, snapshot level {}".format(cycle, snapshot_level)) block_level = cycle * self.blocks_per_cycle + 1 @@ -165,7 +184,8 @@ def __get_snapshot_block_hash(self, cycle, current_level): logger.error("Too few or too many possible snapshots found!") return "" - level_snapshot_block = (cycle - self.preserved_cycles - 2) * self.blocks_per_cycle + (chosen_snapshot+1) * self.blocks_per_roll_snapshot + level_snapshot_block = (cycle - self.preserved_cycles - 2) * self.blocks_per_cycle + ( + chosen_snapshot + 1) * self.blocks_per_roll_snapshot return level_snapshot_block else: From 2939ad84cfda1d90b80cba9811c08583ae6dee57 Mon Sep 17 00:00:00 2001 From: Huseyin ABANOZ Date: Sat, 30 Nov 2019 15:16:43 +0300 Subject: [PATCH 2/2] -use_ssl added --- examples/email.ini | 1 + src/emails/email_manager.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/email.ini b/examples/email.ini index 7c6f8266..9157ae4c 100644 --- a/examples/email.ini +++ b/examples/email.ini @@ -5,3 +5,4 @@ smtp.host = smtp.port = sender = recipients = +use_ssl= \ No newline at end of file diff --git a/src/emails/email_manager.py b/src/emails/email_manager.py index b7f1e935..54697c09 100644 --- a/src/emails/email_manager.py +++ b/src/emails/email_manager.py @@ -56,7 +56,7 @@ def check_and_create_ini(self): with open(EMAIL_INI_PATH, "w") as f: f.writelines(["[DEFAULT]\n", USER + NL, PASS + NL, HOST + NL, PORT + NL, SENDER + NL, - RECIPIENTS + NL]) + RECIPIENTS + NL + USE_SSL+NL]) def send_payment_mail(self, cyle, payments_file, nb_failed, nb_unknown): if not self.email_sender: