From 12e49ad3fe2d53b9fb79befb42d4ae497052a0b8 Mon Sep 17 00:00:00 2001
From: Sam Nguyen <118077+dtjm@users.noreply.github.com>
Date: Fri, 10 May 2024 12:04:36 -0700
Subject: [PATCH] add ability to include internal topics in `bootstrap` (#192)

* add ability to include double-underscore topics in `bootstrap`

* update flag name and readme

* fix formatting
---
 README.md                        | 4 ++++
 cmd/topicctl/subcmd/bootstrap.go | 8 ++++++++
 pkg/cli/cli.go                   | 3 ++-
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 75db8444..eb1bf995 100644
--- a/README.md
+++ b/README.md
@@ -145,6 +145,10 @@ The `bootstrap` subcommand creates apply topic configs from the existing topics
 cluster. This can be used to "import" topics not created or previously managed by topicctl.
 The output can be sent to either a directory (if the `--output` flag is set) or `stdout`.
 
+By default, this does not include internal topics such as `__consumer_offsets`.
+If you would like to have these topics included,
+pass the `--allow-internal-topics` flag.
+
 #### check
 
 ```
diff --git a/cmd/topicctl/subcmd/bootstrap.go b/cmd/topicctl/subcmd/bootstrap.go
index 18c3f04d..7010bae0 100644
--- a/cmd/topicctl/subcmd/bootstrap.go
+++ b/cmd/topicctl/subcmd/bootstrap.go
@@ -21,6 +21,8 @@ type bootstrapCmdConfig struct {
 	outputDir     string
 	overwrite     bool
 
+	allowInternalTopics bool
+
 	shared sharedOptions
 }
 
@@ -52,6 +54,11 @@ func init() {
 		false,
 		"Overwrite existing configs in output directory",
 	)
+	bootstrapCmd.Flags().BoolVar(
+		&bootstrapConfig.allowInternalTopics,
+		"allow-internal-topics",
+		false,
+		"Include topics that start with __ (typically these are internal topics)")
 
 	addSharedConfigOnlyFlags(bootstrapCmd, &bootstrapConfig.shared)
 	bootstrapCmd.MarkFlagRequired("cluster-config")
@@ -92,5 +99,6 @@ func bootstrapRun(cmd *cobra.Command, args []string) error {
 		bootstrapConfig.excludeRegexp,
 		bootstrapConfig.outputDir,
 		bootstrapConfig.overwrite,
+		bootstrapConfig.allowInternalTopics,
 	)
 }
diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go
index 19b41067..ee20f497 100644
--- a/pkg/cli/cli.go
+++ b/pkg/cli/cli.go
@@ -208,6 +208,7 @@ func (c *CLIRunner) BootstrapTopics(
 	excludeRegexpStr string,
 	outputDir string,
 	overwrite bool,
+	allowInternalTopics bool,
 ) error {
 	topicInfoObjs, err := c.adminClient.GetTopics(ctx, topics, false)
 	if err != nil {
@@ -226,7 +227,7 @@ func (c *CLIRunner) BootstrapTopics(
 	topicConfigs := []config.TopicConfig{}
 
 	for _, topicInfo := range topicInfoObjs {
-		if strings.HasPrefix(topicInfo.Name, "__") {
+		if !allowInternalTopics && strings.HasPrefix(topicInfo.Name, "__") {
 			// Never include underscore topics
 			continue
 		} else if !matchRegexp.MatchString(topicInfo.Name) {