From 173e0779cd0f1fcbb2d157ec412979d3d73532c2 Mon Sep 17 00:00:00 2001 From: Tim Kimber Date: Sun, 6 Oct 2024 16:14:43 +0100 Subject: [PATCH 1/2] Wait for Finalize to finish processing as well as Order --- getssl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/getssl b/getssl index 2c49a82f..c508c793 100755 --- a/getssl +++ b/getssl @@ -1632,7 +1632,7 @@ get_auth_dns() { # get the authoritative dns server for a domain (sets primary_n if [[ "$out" == *SERVFAIL* ]]; then debug Output from "$HAS_DIG_OR_DRILL $DNS_CHECK_OPTIONS ${gad_s} NS ${gad_d}" contains SERVFAIL debug "$out" - sleep 2 + sleep 5 fi ((i++)) done @@ -1824,6 +1824,13 @@ get_certificate() { # get certificate for csr, if all domains validated. else # APIv2 info "Requesting Finalize Link" send_signed_request "$FinalizeLink" "{\"csr\": \"$der\"}" "needbase64" + # if ACME response is processing (still creating certificates) then wait and try again. + while [[ "$response_status" == "processing" ]]; do + info "ACME server still Finalizing certificates" + sleep 5 + send_signed_request "$FinalizeLink" "" + done + info Requesting Order Link debug "order link was $OrderLink" send_signed_request "$OrderLink" "" @@ -1833,6 +1840,7 @@ get_certificate() { # get certificate for csr, if all domains validated. sleep 5 send_signed_request "$OrderLink" "" done + info "Requesting certificate" CertData=$(json_get "$response" "certificate") send_signed_request "$CertData" "" "" "$gc_fullchain" From 88f5d2ed17781addad96a6a216ee8be6f786f4ec Mon Sep 17 00:00:00 2001 From: Tim Kimber Date: Tue, 8 Oct 2024 16:32:01 +0100 Subject: [PATCH 2/2] Fix intermittent test failures by retrying on DNS failure and waiting for finalize order to return Valid --- getssl | 62 ++++++++++++++++++++++++++++++------------ test/README-Testing.md | 3 +- test/run-test.sh | 5 +++- 3 files changed, 50 insertions(+), 20 deletions(-) diff --git a/getssl b/getssl index c508c793..122bd4ea 100755 --- a/getssl +++ b/getssl @@ -539,11 +539,11 @@ check_challenge_completion() { # checks with the ACME server if our challenge is break; fi - # if ACME response is that their check gave an invalid response, error exit + # if ACME response is "invalid" then abandon the order request - returns error so it can be retried if [[ "$status" == "invalid" ]] ; then err_detail=$(echo "$response" | grep "detail") - # TODO need to check for "DNS problem: SERVFAIL looking up CAA ..." and retry - error_exit "$domain:Verify error:$err_detail" + info "$domain:Verify error:$err_detail" + return 1 fi # if ACME response is pending (they haven't completed checks yet) @@ -557,6 +557,7 @@ check_challenge_completion() { # checks with the ACME server if our challenge is debug "sleep 5 secs before testing verify again" sleep 5 done + return 0 } check_challenge_completion_dns() { # perform validation via DNS challenge @@ -1469,8 +1470,12 @@ for d in "${alldomains[@]}"; do # let Let's Encrypt check check_challenge_completion "${uri}" "${d}" "${keyauthorization}" - + result=$? del_dns_rr "${d}" "${auth_key}" + if [[ $result -eq 1 ]]; then + # check_challenge_completion failed with "invalid" - order creation cancelled, return error so we can retryS + return 1 + fi else # set up the correct http token for verification if [[ $API -eq 1 ]]; then # get the token from the http component @@ -1523,6 +1528,7 @@ for d in "${alldomains[@]}"; do fi check_challenge_completion "$uri" "$d" "$keyauthorization" + result=$? debug "remove token from ${DOMAIN_ACL}" IFS=\; read -r -a token_locations <<<"$DOMAIN_ACL" @@ -1552,12 +1558,17 @@ for d in "${alldomains[@]}"; do rm -f "${t_loc:?}/${token:?}" fi done + if [[ $result -eq 1 ]]; then + # check_challenge_completion failed with "invalid" - order creation cancelled, return error so we can retryS + return 1 + fi fi # increment domain-counter ((dn++)) fi done # end of ... loop through domains for cert ( from SANS list) #end of verify each domain. +return 0 } get_auth_dns() { # get the authoritative dns server for a domain (sets primary_ns ) @@ -1824,21 +1835,22 @@ get_certificate() { # get certificate for csr, if all domains validated. else # APIv2 info "Requesting Finalize Link" send_signed_request "$FinalizeLink" "{\"csr\": \"$der\"}" "needbase64" - # if ACME response is processing (still creating certificates) then wait and try again. - while [[ "$response_status" == "processing" ]]; do - info "ACME server still Finalizing certificates" - sleep 5 - send_signed_request "$FinalizeLink" "" - done - info Requesting Order Link - debug "order link was $OrderLink" + info Checking Finalize status + debug "POST-as-GET order link ($OrderLink) to check status" send_signed_request "$OrderLink" "" - # if ACME response is processing (still creating certificates) then wait and try again. - while [[ "$response_status" == "processing" ]]; do + + # if ACME response is pending (they haven't completed checks yet) or ready (awaiting finalization) + # or processing (still creating certificates) then wait and check again. + count=0 + while [[ "$response_status" != "valid" ]]; do info "ACME server still Processing certificates" sleep 5 send_signed_request "$OrderLink" "" + ((count++)) + if [[ $count -gt 10 ]]; then + error_exit "Finalize failed - checked server 10 times but certificate still not ready" + fi done info "Requesting certificate" @@ -3480,11 +3492,25 @@ else read -r -a alldomains <<< "$(echo "$DOMAIN,$SANS" | sed "s/,/ /g")" fi -if [[ $API -eq 2 ]]; then - create_order -fi +# Try again if order creation fails (means check_challenge_completion returned "invalid" - generally DNS failure) +retry=0 +while [[ $retry -lt 3 ]] +do + if [[ $API -eq 2 ]]; then + create_order + fi -fulfill_challenges + fulfill_challenges + result=$? + if [[ $result -eq 0 ]]; then + break + fi + ((retry++)) +done + +if [[ $retry -ge 3 ]]; then + error_exit "$domain: fulfill_challenges failed 3 times" +fi # Verification has been completed for all SANS, so request certificate. info "Verification completed, obtaining certificate." diff --git a/test/README-Testing.md b/test/README-Testing.md index 5dd18640..f321b397 100644 --- a/test/README-Testing.md +++ b/test/README-Testing.md @@ -44,7 +44,8 @@ For individual accounts, \ is your github account name. 1. Start `pebble` and `challtestsrv` using ```docker compose up -d --build``` 2. ```run-test.sh /getssl/test/debug-test.sh ``` -3. e.g. `test/run-test.sh ubuntu /getssl/test/debug-test.sh -d /getssl/test/test-config/getssl-http01-cfg` +3. e.g. `test/run-test.sh ubuntu /getssl/test/debug-test.sh -d /getssl/test/test-config/getssl-http01.cfg` +4. or (`test/run-test.sh ubuntu /getssl/test/debug-test.sh -d getssl-http01.cfg`) ## TODO diff --git a/test/run-test.sh b/test/run-test.sh index 62bee669..379aaaa9 100755 --- a/test/run-test.sh +++ b/test/run-test.sh @@ -3,6 +3,8 @@ if [ $# -eq 0 ]; then echo "Usage: $(basename "$0") []" echo "e.g. $(basename "$0") alpine bats /getssl/test" + echo "e.g. $(basename "$0") ubuntu 11-mixed-case.bats" + echo "e.g. $(basename "$0") ubuntu /getssl/test/debug-test.sh -d getssl-http01.cfg" exit 1 fi OS=$1 @@ -10,7 +12,7 @@ OS=$1 if [ $# -gt 1 ]; then shift COMMAND=$* - if [[ $COMMAND != bash ]]; then + if [[ $COMMAND != bash ]] && [[ $COMMAND != /getssl/test/debug-test.sh* ]]; then if [[ $COMMAND != "bats /getssl/test"* ]]; then if [[ $COMMAND == /getssl/test* ]]; then COMMAND="bats $COMMAND" @@ -27,6 +29,7 @@ if [ $# -gt 1 ]; then else COMMAND="bats /getssl/test --timing" fi +echo "Running $COMMAND" REPO="" if [ -n "$GITHUB_REPOSITORY" ] ; then