Command recreate rds to fix aws cf issue

pkg/api/controllers.go

@@ -1493,3 +1493,32 @@ func (s *Server) SystemUpdate(c *stdapi.Context) error {
 func (s *Server) Workers(c *stdapi.Context) error {
 	return stdapi.Errorf(404, "not available via api")
 }
+
+func (s *Server) IsDBSnapshotComplete(c *stdapi.Context) error {
+	snapshot := c.Var("snapshot")
+
+	v, err := s.provider(c).WithContext(c.Context()).IsDBSnapshotComplete(snapshot)
+	if err != nil {
+		return err
+	}
+
+	return c.RenderJSON(v)
+}
+
+func (s *Server) SetDBDeletionProtectionAndCreateSnapShot(c *stdapi.Context) error {
+	app := c.Var("app")
+	resource := c.Var("resource")
+	snapshot := c.Var("snapshot")
+
+	v, err := s.provider(c).WithContext(c.Context()).SetDBDeletionProtectionAndCreateSnapShot(app, resource, snapshot)
+	if err != nil {
+		return err
+	}
+
+	return c.RenderJSON(v)
+}
+
+func (s *Server) DeleteDB(c *stdapi.Context) error {
+	resource := c.Var("resource")
+	return s.provider(c).WithContext(c.Context()).DeleteDB(resource)
+}

pkg/api/routes.go

@@ -60,6 +60,9 @@ func (s *Server) setupRoutes(r stdapi.Router) {
 	r.Route("GET", "/apps/{app}/services/{name}/metrics", s.ServiceMetrics)
 	r.Route("POST", "/apps/{app}/services/{name}/restart", s.ServiceRestart)
 	r.Route("PUT", "/apps/{app}/services/{name}", s.ServiceUpdate)
+	r.Route("GET", "/rds/snapshots/{snapshot}", s.IsDBSnapshotComplete)
+	r.Route("DELETE", "/rds/{resource}", s.DeleteDB)
+	r.Route("POST", "/apps/{app}/resources/{resource}/deletion-protection-and-snapshot/{snapshot}", s.SetDBDeletionProtectionAndCreateSnapShot)
 	r.Route("GET", "/system", s.SystemGet)
 	r.Route("", "", s.SystemInstall)
 	r.Route("PUT", "/system/jwt/rotate", s.SystemJwtSignKeyRotate)

pkg/cli/helpers.go

@@ -13,6 +13,7 @@ import (
 	"sort"
 	"strings"
 	"time"
+	"unicode"
 
 	"github.com/convox/rack/pkg/helpers"
 	"github.com/convox/rack/pkg/structs"
@@ -432,3 +433,14 @@ func waitForResourceRunning(rack sdk.Interface, c *stdcli.Context, resource stri
 		return r.Status == "running", nil
 	})
 }
+
+// only letter and digits will be kept
+func filterString(input string) string {
+	var builder strings.Builder
+	for _, r := range input {
+		if unicode.IsLetter(r) || unicode.IsDigit(r) {
+			builder.WriteRune(r)
+		}
+	}
+	return builder.String()
+}

pkg/cli/resources.go

@@ -7,7 +7,9 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
+	"github.com/convox/rack/pkg/helpers"
 	"github.com/convox/rack/pkg/options"
 	"github.com/convox/rack/pkg/structs"
 	"github.com/convox/rack/sdk"
@@ -130,6 +132,24 @@ func init() {
 		Usage:     "<resource>",
 		Validate:  stdcli.Args(1),
 	})
+
+	register("resources fix cf", "fix CF rds resource", ResourcesFixCf, stdcli.CommandOptions{
+		Flags: []stdcli.Flag{
+			stdcli.StringFlag("snapshot", "", "db snapshot name"),
+			stdcli.StringFlag("olddb", "", "old db"),
+			flagRack, flagApp,
+		},
+		Invisible: true,
+		Usage:     "<resource>",
+		Validate:  stdcli.Args(1),
+	})
+
+	register("resources db delete", "delete db", ResourcesDeleteDB, stdcli.CommandOptions{
+		Flags:     []stdcli.Flag{flagRack, flagApp},
+		Invisible: true,
+		Usage:     "<resource>",
+		Validate:  stdcli.Args(1),
+	})
 }
 
 func Resources(rack sdk.Interface, c *stdcli.Context) error {
@@ -692,3 +712,118 @@ func RackResourcesUrl(rack sdk.Interface, c *stdcli.Context) error {
 
 	return nil
 }
+
+func ResourcesFixCf(rack sdk.Interface, c *stdcli.Context) error {
+	appName := app(c)
+	resourceName := c.Arg(0)
+	snapshot := c.String("snapshot")
+	oldDb := ""
+	var err error
+	if snapshot == "" {
+		snapshot = filterString(fmt.Sprintf("db%s-%s-%d", appName, resourceName, time.Now().Unix()))
+		c.Writef("Creating snapshot: %s\n", snapshot)
+		if oldDb, err = rack.SetDBDeletionProtectionAndCreateSnapShot(appName, resourceName, snapshot); err != nil {
+			return err
+		}
+	} else if c.String("olddb") != "" {
+		oldDb = c.String("olddb")
+		c.Writef("Using snapshot: %s and db: %s\n", snapshot, oldDb)
+	}
+
+	rs, err := rack.ReleaseList(appName, structs.ReleaseListOptions{Limit: options.Int(1)})
+	if err != nil {
+		return err
+	}
+
+	if len(rs) == 0 {
+		return fmt.Errorf("no releases to promote")
+	}
+
+	releaseId := rs[0].Id
+
+	a, err := rack.AppGet(appName)
+	if err != nil {
+		return err
+	}
+
+	if a.Status != "running" {
+		c.Startf("Waiting for app to be ready")
+		if err := helpers.WaitForAppRunning(rack, appName); err != nil {
+			return err
+		}
+		c.OK()
+	}
+
+	c.Writef("DB identifier: %s\n", oldDb)
+
+	c.Startf("Promoting <release>%s</release> to remove resource: %s", releaseId, resourceName)
+	if err := rack.ReleasePromote(appName, releaseId, structs.ReleasePromoteOptions{
+		Xignore: &resourceName,
+	}); err != nil {
+		return err
+	}
+	c.Writef("\n")
+	if err := helpers.WaitForAppWithLogs(rack, c, appName); err != nil && !strings.Contains(err.Error(), "rollback") {
+		return err
+	}
+	c.OK()
+
+	c.Startf("Waiting for app to be ready")
+	if err := helpers.WaitForAppRunning(rack, appName); err != nil {
+		return err
+	}
+	c.OK()
+
+	c.Startf("Waiting for snapshot to be ready")
+	for {
+		complete, err := rack.IsDBSnapshotComplete(snapshot)
+		if err != nil {
+			return err
+		}
+
+		if complete {
+			break
+		}
+		time.Sleep(30 * time.Second)
+	}
+	c.OK()
+
+	c.Startf("Promoting <release>%s</release> to recreate resource: %s", releaseId, resourceName)
+	if err := rack.ReleasePromote(appName, releaseId, structs.ReleasePromoteOptions{
+		Xrds:      &resourceName,
+		Xsnapshot: &snapshot,
+	}); err != nil {
+		return err
+	}
+
+	c.Writef("\n")
+	if err := helpers.WaitForAppWithLogs(rack, c, appName); err != nil {
+		return err
+	}
+	c.OK()
+
+	c.Startf("Promoting <release>%s</release> to rsync the resource: %s", releaseId, resourceName)
+	if err := rack.ReleasePromote(appName, releaseId, structs.ReleasePromoteOptions{}); err != nil {
+		return err
+	}
+
+	c.Writef("\n")
+	if err := helpers.WaitForAppWithLogs(rack, c, appName); err != nil {
+		return err
+	}
+	c.OK()
+
+	c.Writef("\n")
+
+	c.Writef("Resource is fixed. Manually delete the old db if everything is alright: convox resources db delete %s", oldDb)
+
+	return nil
+}
+
+func ResourcesDeleteDB(rack sdk.Interface, c *stdcli.Context) error {
+	resourceName := c.Arg(0)
+	if err := rack.DeleteDB(resourceName); err != nil {
+		return err
+	}
+	return c.OK()
+}