Merge pull request #12 from square/replicationfix

add status check for secret replication to replica regions

.gitignore

@@ -1,3 +1,4 @@
 *.out
 *.tmp
 *.swp
+vendor/

go.mod

@@ -4,9 +4,11 @@ go 1.20
 
 require (
 	github.com/aws/aws-lambda-go v1.16.0
-	github.com/aws/aws-sdk-go v1.34.0
+	github.com/aws/aws-sdk-go v1.44.276
+
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/go-test/deep v1.0.6
 )
 
-require github.com/jmespath/go-jmespath v0.3.0 // indirect
+require github.com/jmespath/go-jmespath v0.4.0 // indirect
+

go.sum

@@ -1,31 +1,57 @@
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/aws/aws-lambda-go v1.16.0 h1:9+Pp1/6cjEXYhwadp8faFXKSOWt7/tHRCnQxQmKvVwM=
 github.com/aws/aws-lambda-go v1.16.0/go.mod h1:FEwgPLE6+8wcGBTe5cJN3JWurd1Ztm9zN4jsXsjzKKw=
-github.com/aws/aws-sdk-go v1.34.0 h1:brux2dRrlwCF5JhTL7MUT3WUwo9zfDHZZp3+g3Mvlmo=
-github.com/aws/aws-sdk-go v1.34.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aws/aws-sdk-go v1.44.276 h1:ywPlx9C5Yc482dUgAZ9bHpQ6onVvJvYE9FJWsNDCEy0=
+github.com/aws/aws-sdk-go v1.44.276/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-test/deep v1.0.6 h1:UHSEyLZUwX9Qoi99vVwvewiMC8mM2bf7XEM2nqvzEn8=
 github.com/go-test/deep v1.0.6/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
-github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
-github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

rotate.go

@@ -66,6 +66,10 @@ type Config struct {
 	// EventReceiver receives events during the four-step password rotation process.
 	// If none is provided, NullEventReceiver is used. See EventReceiver for more details.
 	EventReceiver EventReceiver
+
+	// ReplicationWait governs the duration password rotation lambda will wait for
+	// secret replication to secondary regions to complete
+	ReplicationWait time.Duration
 }
 
 // InvokedBySecretsManager returns true if the event is from Secrets Manager.
@@ -98,6 +102,7 @@ type Rotator struct {
 	clientRequestToken string
 	secretId           string
 	startTime          time.Time
+	replicationWait    time.Duration
 }
 
 // NewRotator creates a new Rotator.
@@ -114,11 +119,12 @@ func NewRotator(cfg Config) *Rotator {
 		ss = RandomPassword{}
 	}
 	return &Rotator{
-		sm:     cfg.SecretsManager,
-		db:     cfg.PasswordSetter,
-		ss:     ss,
-		event:  event,
-		skipDb: cfg.SkipDatabase,
+		sm:              cfg.SecretsManager,
+		db:              cfg.PasswordSetter,
+		ss:              ss,
+		event:           event,
+		skipDb:          cfg.SkipDatabase,
+		replicationWait: cfg.ReplicationWait,
 	}
 }
 
@@ -523,6 +529,12 @@ func (r *Rotator) FinishSecret(ctx context.Context, event map[string]string) err
 	downtime := now.Sub(r.startTime)
 	log.Printf("password downtime: %dms", downtime.Milliseconds())
 
+	// Wait for secret replication to complete to all replica regions
+	err = r.checkSecretReplicationStatus()
+	if err != nil {
+		return err
+	}
+
 	// Remove AWSPENDING label
 	debug("removing AWSPENDING from version id = %v", *newSecret.VersionId)
 	_, err = r.sm.UpdateSecretVersionStage(&secretsmanager.UpdateSecretVersionStageInput{
@@ -602,6 +614,52 @@ func (r *Rotator) rollback(ctx context.Context, creds db.NewPassword, rotationSt
 	return errRotationFailed // always return this error
 }
 
+// checks that secret have been replicated to all replica regions
+// this is necessary between multiple calls of UpdateSecretVersionStage
+// to guard against arace condition in AWS that leaves secret replication
+// stuck indefinitely.
+func (r *Rotator) checkSecretReplicationStatus() error {
+	log.Println("checking secret replication status")
+	waitDuration := DEFAULT_REPLICATION_WAIT
+	if r.replicationWait > 0 {
+		waitDuration = r.replicationWait
+	}
+
+	startTime := time.Now()
+	for time.Now().Sub(startTime) < waitDuration {
+		secret, err := r.sm.DescribeSecret(&secretsmanager.DescribeSecretInput{
+			SecretId: aws.String(r.secretId),
+		})
+		if err != nil {
+			return err
+		}
+		if secret == nil {
+			return fmt.Errorf("expected an non null secret for secretId %v but received null", r.secretId)
+		}
+		replicationSyncComplete := true
+		for _, status := range secret.ReplicationStatus {
+			if status == nil {
+				replicationSyncComplete = false
+				log.Println("encountered null replication status")
+				break
+			}
+			if *status.Status != secretsmanager.StatusTypeInSync {
+				replicationSyncComplete = false
+				log.Printf("replication status still in (%v) in region (%v) expecting (%v)\n", *status.Status, *status.Region, secretsmanager.StatusTypeInSync)
+				break
+			}
+		}
+		// only return success if all secret replica regions are in sync all
+		// other cases are treated as errors
+		if replicationSyncComplete {
+			log.Println("secret replication sync completed successfully")
+			return nil // success
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	return fmt.Errorf("timeout waiting for secret replication StatusTypeInSync = true")
+}
+
 // --------------------------------------------------------------------------
 
 var (
@@ -616,6 +674,10 @@ var (
 	DebugSecret = false
 
 	debugLog = log.New(os.Stderr, "DEBUG ", log.LstdFlags|log.Lmicroseconds|log.Lshortfile|log.LUTC)
+
+	// DEFAULT_REPLICATION_WAIT is the default duration that password rotation lambda will
+	// wait for secret replication to secondary regions to complete
+	DEFAULT_REPLICATION_WAIT = 30 * time.Second
 )
 
 func debugSecret(msg string, v ...interface{}) {

rotate_test.go

@@ -12,7 +12,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/secretsmanager"
 	"github.com/go-test/deep"
 
-	"github.com/square/password-rotation-lambda/v2"
+	rotate "github.com/square/password-rotation-lambda/v2"
 	"github.com/square/password-rotation-lambda/v2/db"
 	"github.com/square/password-rotation-lambda/v2/test"
 )
@@ -432,6 +432,7 @@ func TestStepFinishSecret(t *testing.T) {
 	// calls UpdateSecretVersionStage to move the current label to the pending
 	// secret by ID.
 	var gotUpdateInput *secretsmanager.UpdateSecretVersionStageInput
+	var gotDescribeInput *secretsmanager.DescribeSecretInput
 	sm := test.MockSecretsManager{
 		GetSecretValueFunc: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) {
 			switch *input.VersionStage {
@@ -464,6 +465,10 @@ func TestStepFinishSecret(t *testing.T) {
 				Name: aws.String("sercetName"),
 			}, nil
 		},
+		DescribeSecretFunc: func(input *secretsmanager.DescribeSecretInput) (*secretsmanager.DescribeSecretOutput, error) {
+			gotDescribeInput = input
+			return &secretsmanager.DescribeSecretOutput{}, nil
+		},
 	}
 
 	// Create a new Rotator to test
@@ -493,6 +498,88 @@ func TestStepFinishSecret(t *testing.T) {
 	if diff := deep.Equal(gotUpdateInput, expectUpdateInput); diff != nil {
 		t.Log(diff)
 	}
+	expectedDescribeInput := &secretsmanager.DescribeSecretInput{
+		SecretId: aws.String("def"),
+	}
+	if diff := deep.Equal(expectedDescribeInput, gotDescribeInput); diff != nil {
+		t.Log(diff)
+	}
+}
+
+func TestStepFinishSecretFailure(t *testing.T) {
+	// Test that finish secret fails when describesecret replication status
+	// does not go to InSync
+	retryWait := 1 * time.Second
+
+	sm := test.MockSecretsManager{
+		GetSecretValueFunc: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) {
+			switch *input.VersionStage {
+			case rotate.AWSCURRENT:
+				return &secretsmanager.GetSecretValueOutput{
+					ARN:           aws.String("arn"),
+					Name:          aws.String("sercetName"),
+					SecretString:  &secretString1,
+					VersionId:     aws.String("v1"),
+					VersionStages: []*string{aws.String(rotate.AWSCURRENT)},
+					CreatedDate:   &now,
+				}, nil
+			case rotate.AWSPENDING:
+				return &secretsmanager.GetSecretValueOutput{
+					ARN:           aws.String("arn"),
+					Name:          aws.String("sercetName"),
+					SecretString:  &secretString2,
+					VersionId:     aws.String("v2"),
+					VersionStages: []*string{aws.String(rotate.AWSPENDING)},
+					CreatedDate:   &now,
+				}, nil
+			default:
+				return nil, nil
+			}
+		},
+		UpdateSecretVersionStageFunc: func(input *secretsmanager.UpdateSecretVersionStageInput) (*secretsmanager.UpdateSecretVersionStageOutput, error) {
+			return &secretsmanager.UpdateSecretVersionStageOutput{
+				ARN:  aws.String("arn"),
+				Name: aws.String("sercetName"),
+			}, nil
+		},
+		DescribeSecretFunc: func(input *secretsmanager.DescribeSecretInput) (*secretsmanager.DescribeSecretOutput, error) {
+			return &secretsmanager.DescribeSecretOutput{
+				ReplicationStatus: []*secretsmanager.ReplicationStatusType{
+					&secretsmanager.ReplicationStatusType{
+						Region: aws.String("us-west-2"),
+						Status: aws.String(secretsmanager.StatusTypeInProgress),
+					},
+				},
+			}, nil
+		},
+	}
+	startTime := time.Now()
+	// Create a new Rotator to test
+	r := rotate.NewRotator(rotate.Config{
+		SecretsManager:  sm,
+		SecretSetter:    test.MockSecretSetter{},
+		PasswordSetter:  test.MockPasswordSetter{},
+		ReplicationWait: retryWait,
+	})
+
+	// Simulate testSecret event from Secrets Manager
+	event := map[string]string{
+		"ClientRequestToken": "abc",
+		"SecretId":           "def",
+		"Step":               "finishSecret",
+	}
+	_, err := r.Handler(context.TODO(), event)
+
+	// expect an error
+	if err == nil {
+		t.Error(err)
+	}
+
+	finishTime := time.Now()
+	testDuration := finishTime.Sub(startTime)
+	if testDuration < retryWait {
+		t.Errorf("expected test to take %v duration but it finished in %v", retryWait, testDuration)
+	}
 }
 
 func TestUserInvoke(t *testing.T) {

test/mock_aws.go

@@ -14,6 +14,7 @@ type MockSecretsManager struct {
 	GetSecretValueFunc           func(*secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error)
 	PutSecretValueFunc           func(*secretsmanager.PutSecretValueInput) (*secretsmanager.PutSecretValueOutput, error)
 	UpdateSecretVersionStageFunc func(*secretsmanager.UpdateSecretVersionStageInput) (*secretsmanager.UpdateSecretVersionStageOutput, error)
+	DescribeSecretFunc           func(input *secretsmanager.DescribeSecretInput) (*secretsmanager.DescribeSecretOutput, error)
 }
 
 func (m MockSecretsManager) GetSecretValue(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) {
@@ -37,6 +38,13 @@ func (m MockSecretsManager) UpdateSecretVersionStage(input *secretsmanager.Updat
 	return nil, nil
 }
 
+func (m MockSecretsManager) DescribeSecret(input *secretsmanager.DescribeSecretInput) (*secretsmanager.DescribeSecretOutput, error) {
+	if m.DescribeSecretFunc != nil {
+		return m.DescribeSecretFunc(input)
+	}
+	return nil, nil
+}
+
 // --------------------------------------------------------------------------
 
 type MockRDSClient struct {