From f053ca6833ca77e3ff9a62153685d8726c71d68b Mon Sep 17 00:00:00 2001 From: Aiden Keating Date: Thu, 2 Apr 2020 13:22:19 +0100 Subject: [PATCH 1/2] add check for cached or out of date elasticache snapshots ensure that elasticache snapshots that are found through the tagging api but may not longer exist are not attmepted to be deleted, and gracefully handle the errors if a previously deleted snapshot is attempted to be deleted again when it can no longer be found. verification: - locate a cluster with phantom/cached snapshots - run 'make build/cli' - run './cli cleanup -t elasticache:snapshot' - ensure the result does not contain snapshots that do not exist through the aws console - run './cli cleanup -t elasticache:snapshot --dry-run=false' - ensure the result is not a failure due to cached, not found, snapshots trying to be deleted' --- pkg/aws/manager_elasticache_snapshot.go | 26 +++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/pkg/aws/manager_elasticache_snapshot.go b/pkg/aws/manager_elasticache_snapshot.go index 6f463bb..0ff1687 100644 --- a/pkg/aws/manager_elasticache_snapshot.go +++ b/pkg/aws/manager_elasticache_snapshot.go @@ -59,8 +59,20 @@ func (r *ElasticacheSnapshotManager) DeleteResourcesForCluster(clusterId string, for _, resourceTagMapping := range resourceOutput.ResourceTagMappingList { snapshotARN := aws.StringValue(resourceTagMapping.ResourceARN) snapshotARNElements := strings.Split(snapshotARN, ":") + snapshotName := snapshotARNElements[len(snapshotARNElements)-1] + snapshotLogger := r.logger.WithField(loggingKeySnapshot, snapshotName) + describeSnapshotsOutput, err := r.elasticacheClient.DescribeSnapshots(&elasticache.DescribeSnapshotsInput{ + SnapshotName: aws.String(snapshotName), + }) + if err != nil { + return nil, errors.WrapLog(err, "failed to get elasticache snapshot", r.logger) + } + if len(describeSnapshotsOutput.Snapshots) == 0 { + snapshotLogger.Debug("no snapshot found, assuming caching issue in aws, skipping") + continue + } snapshotsToDelete = append(snapshotsToDelete, &basicResource{ - Name: snapshotARNElements[len(snapshotARNElements)-1], + Name: snapshotName, ARN: snapshotARN, }) } @@ -85,9 +97,15 @@ func (r *ElasticacheSnapshotManager) DeleteResourcesForCluster(clusterId string, SnapshotName: aws.String(snapshot.Name), } if _, err := r.elasticacheClient.DeleteSnapshot(deleteSnapshotInput); err != nil { - if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == elasticache.ErrCodeInvalidSnapshotStateFault { - snapshotLogger.Debug("snapshot is in a deleting state, ignoring error") - continue + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == elasticache.ErrCodeInvalidSnapshotStateFault { + snapshotLogger.Debug("snapshot is in a deleting state, ignoring error") + continue + } + if awsErr.Code() == elasticache.ErrCodeSnapshotNotFoundFault { + snapshotLogger.Debug("snapshot is not found, assuming already removed or aws caching, ignoring error") + continue + } } return nil, errors.WrapLog(err, "failed to delete snapshot", r.logger) } From 713cb853e8ceaaa2ca9d96df0eefbab61a65e62d Mon Sep 17 00:00:00 2001 From: Aiden Keating Date: Thu, 2 Apr 2020 13:26:54 +0100 Subject: [PATCH 2/2] bump version to 0.2.2 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index c20e1ac..86db2a7 100644 --- a/version/version.go +++ b/version/version.go @@ -2,5 +2,5 @@ package version var ( //Version Version of the Cluster Service - Version = "0.2.1" + Version = "0.2.2" )