Automatically uncompress all spec data returned by the visitor packag…

…e. (#1124)

cmd/registry/cmd/compute/complexity/complexity.go

@@ -17,9 +17,7 @@ package complexity
 import (
 	"context"
 	"fmt"
-	"strings"
 
-	"github.com/apigee/registry/cmd/registry/compress"
 	"github.com/apigee/registry/cmd/registry/tasks"
 	"github.com/apigee/registry/pkg/connection"
 	"github.com/apigee/registry/pkg/log"
@@ -133,11 +131,6 @@ func (task *computeComplexityTask) Run(ctx context.Context) error {
 	relation := "complexity"
 	log.Debugf(ctx, "Computing %s/artifacts/%s", task.specName, relation)
 	contents := spec.GetContents()
-	if strings.Contains(spec.GetMimeType(), "+gzip") {
-		if contents, err = compress.GUnzippedBytes(contents); err != nil {
-			return err
-		}
-	}
 	var complexity *metrics.Complexity
 	if mime.IsOpenAPIv2(spec.GetMimeType()) {
 		document, err := oas2.ParseDocument(contents)

cmd/registry/cmd/compute/lint/lint.go

@@ -74,10 +74,10 @@ func Command() *cobra.Command {
 			// Iterate through a collection of specs and evaluate each.
 			err = visitor.ListSpecs(ctx, client, spec, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
 				taskQueue <- &computeLintTask{
-					client:   client,
-					specName: spec.Name,
-					linter:   linter,
-					dryRun:   dryRun,
+					client: client,
+					spec:   spec,
+					linter: linter,
+					dryRun: dryRun,
 				}
 				return nil
 			})
@@ -92,29 +92,23 @@ func Command() *cobra.Command {
 }
 
 type computeLintTask struct {
-	client   connection.RegistryClient
-	specName string
-	linter   string
-	dryRun   bool
+	client connection.RegistryClient
+	spec   *rpc.ApiSpec
+	linter string
+	dryRun bool
 }
 
 func (task *computeLintTask) String() string {
-	return fmt.Sprintf("compute %s/lint-%s", task.specName, task.linter)
+	return fmt.Sprintf("compute %s/lint-%s", task.spec.Name, task.linter)
 }
 
 func lintRelation(linter string) string {
 	return "lint-" + linter
 }
 
 func (task *computeLintTask) Run(ctx context.Context) error {
-	request := &rpc.GetApiSpecRequest{
-		Name: task.specName,
-	}
-	spec, err := task.client.GetApiSpec(ctx, request)
-	if err != nil {
-		return err
-	}
-	data, err := visitor.GetBytesForSpec(ctx, task.client, spec)
+	spec := task.spec
+	err := visitor.FetchSpecContents(ctx, task.client, spec)
 	if err != nil {
 		return err
 	}
@@ -127,7 +121,7 @@ func (task *computeLintTask) Run(ctx context.Context) error {
 		}
 		relation = lintRelation(task.linter)
 		log.Debugf(ctx, "Computing %s/artifacts/%s", spec.Name, relation)
-		lint, err = NewLintFromOpenAPI(spec.Name, data, task.linter)
+		lint, err = NewLintFromOpenAPI(spec.Name, spec.Contents, task.linter)
 		if err != nil {
 			return fmt.Errorf("error processing OpenAPI: %s (%s)", spec.Name, err.Error())
 		}
@@ -140,7 +134,7 @@ func (task *computeLintTask) Run(ctx context.Context) error {
 		}
 		relation = lintRelation(task.linter)
 		log.Debugf(ctx, "Computing %s/artifacts/%s", spec.Name, relation)
-		lint, err = NewLintFromZippedProtos(spec.Name, data)
+		lint, err = NewLintFromZippedProtos(spec.Name, spec.Contents)
 		if err != nil {
 			return fmt.Errorf("error processing protos: %s (%s)", spec.Name, err.Error())
 		}

cmd/registry/cmd/compute/vocabulary/vocabulary.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/apigee/registry/cmd/registry/compress"
 	"github.com/apigee/registry/cmd/registry/tasks"
 	"github.com/apigee/registry/pkg/connection"
 	"github.com/apigee/registry/pkg/log"
@@ -28,7 +27,6 @@ import (
 	"github.com/apigee/registry/rpc"
 	"github.com/google/gnostic/metrics/vocabulary"
 	"github.com/spf13/cobra"
-	"google.golang.org/grpc/metadata"
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/proto"
 
@@ -81,18 +79,18 @@ func Command() *cobra.Command {
 			if parsed.RevisionID == "" {
 				err = visitor.ListSpecs(ctx, client, parsed.Spec(), filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
 					taskQueue <- &computeVocabularyTask{
-						client:   client,
-						specName: spec.GetName(),
-						dryRun:   dryRun,
+						client: client,
+						spec:   spec,
+						dryRun: dryRun,
 					}
 					return nil
 				})
 			} else {
 				err = visitor.ListSpecRevisions(ctx, client, parsed, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
 					taskQueue <- &computeVocabularyTask{
-						client:   client,
-						specName: spec.GetName(),
-						dryRun:   dryRun,
+						client: client,
+						spec:   spec,
+						dryRun: dryRun,
 					}
 					return nil
 				})
@@ -109,62 +107,53 @@ func Command() *cobra.Command {
 }
 
 type computeVocabularyTask struct {
-	client   connection.RegistryClient
-	specName string
-	dryRun   bool
+	client connection.RegistryClient
+	spec   *rpc.ApiSpec
+	dryRun bool
 }
 
 func (task *computeVocabularyTask) String() string {
-	return "compute vocabulary " + task.specName
+	return "compute vocabulary " + task.spec.Name
 }
 
 func (task *computeVocabularyTask) Run(ctx context.Context) error {
-	ctx = metadata.AppendToOutgoingContext(ctx, "accept-encoding", "gzip")
-	contents, err := task.client.GetApiSpecContents(ctx, &rpc.GetApiSpecContentsRequest{
-		Name: task.specName,
-	})
-	if err != nil {
+	if err := visitor.FetchSpecContents(ctx, task.client, task.spec); err != nil {
 		return err
 	}
-	if mime.IsGZipCompressed(contents.ContentType) {
-		contents.Data, err = compress.GUnzippedBytes(contents.Data)
-		if err != nil {
-			return err
-		}
-	}
 
-	log.Debugf(ctx, "Computing %s/artifacts/vocabulary", task.specName)
+	log.Debugf(ctx, "Computing %s/artifacts/vocabulary", task.spec.Name)
 	var vocab *metrics.Vocabulary
 
-	if mime.IsOpenAPIv2(contents.GetContentType()) {
-		document, err := oas2.ParseDocument(contents.GetData())
+	if mime.IsOpenAPIv2(task.spec.GetMimeType()) {
+		document, err := oas2.ParseDocument(task.spec.GetContents())
 		if err != nil {
-			log.FromContext(ctx).WithError(err).Errorf("Invalid OpenAPI: %s", task.specName)
+			log.FromContext(ctx).WithError(err).Errorf("Invalid OpenAPI: %s", task.spec.Name)
 			return nil
 		}
 		vocab = vocabulary.NewVocabularyFromOpenAPIv2(document)
-	} else if mime.IsOpenAPIv3(contents.GetContentType()) {
-		document, err := oas3.ParseDocument(contents.GetData())
+	} else if mime.IsOpenAPIv3(task.spec.GetMimeType()) {
+		document, err := oas3.ParseDocument(task.spec.GetContents())
 		if err != nil {
-			log.FromContext(ctx).WithError(err).Errorf("Invalid OpenAPI: %s", task.specName)
+			log.FromContext(ctx).WithError(err).Errorf("Invalid OpenAPI: %s", task.spec.Name)
 			return nil
 		}
 		vocab = vocabulary.NewVocabularyFromOpenAPIv3(document)
-	} else if mime.IsDiscovery(contents.GetContentType()) {
-		document, err := discovery.ParseDocument(contents.GetData())
+	} else if mime.IsDiscovery(task.spec.GetMimeType()) {
+		document, err := discovery.ParseDocument(task.spec.GetContents())
 		if err != nil {
-			log.FromContext(ctx).WithError(err).Errorf("Invalid Discovery: %s", task.specName)
+			log.FromContext(ctx).WithError(err).Errorf("Invalid Discovery: %s", task.spec.Name)
 			return nil
 		}
 		vocab = vocabulary.NewVocabularyFromDiscovery(document)
-	} else if mime.IsProto(contents.GetContentType()) && mime.IsZipArchive(contents.GetContentType()) {
-		vocab, err = NewVocabularyFromZippedProtos(contents.GetData())
+	} else if mime.IsProto(task.spec.GetMimeType()) && mime.IsZipArchive(task.spec.GetMimeType()) {
+		var err error
+		vocab, err = NewVocabularyFromZippedProtos(task.spec.GetContents())
 		if err != nil {
-			log.FromContext(ctx).WithError(err).Errorf("Error processing protos: %s", task.specName)
+			log.FromContext(ctx).WithError(err).Errorf("Error processing protos: %s", task.spec.Name)
 			return nil
 		}
 	} else {
-		return fmt.Errorf("we don't know how to compute the vocabulary of %s", task.specName)
+		return fmt.Errorf("we don't know how to compute the vocabulary of %s", task.spec.Name)
 	}
 
 	if task.dryRun {
@@ -177,7 +166,7 @@ func (task *computeVocabularyTask) Run(ctx context.Context) error {
 		return err
 	}
 	return visitor.SetArtifact(ctx, task.client, &rpc.Artifact{
-		Name:     task.specName + "/artifacts/vocabulary",
+		Name:     task.spec.Name + "/artifacts/vocabulary",
 		MimeType: mime.MimeTypeForMessageType("gnostic.metrics.Vocabulary"),
 		Contents: messageData,
 	})

cmd/registry/cmd/diff/diff.go

@@ -23,7 +23,6 @@ import (
 	"github.com/apigee/registry/cmd/registry/compress"
 	"github.com/apigee/registry/pkg/connection"
 	"github.com/apigee/registry/pkg/log"
-	"github.com/apigee/registry/pkg/mime"
 	"github.com/apigee/registry/pkg/names"
 	"github.com/apigee/registry/pkg/visitor"
 	"github.com/apigee/registry/rpc"
@@ -196,22 +195,7 @@ func printDiff(spec1, spec2 *rpc.ApiSpec) error {
 			}
 		}
 	} else {
-		var err error
-		contents1 := spec1.Contents
-		if mime.IsGZipCompressed(spec1.MimeType) {
-			contents1, err = compress.GUnzippedBytes(contents1)
-			if err != nil {
-				return err
-			}
-		}
-		contents2 := spec2.Contents
-		if mime.IsGZipCompressed(spec2.MimeType) {
-			contents2, err = compress.GUnzippedBytes(contents2)
-			if err != nil {
-				return err
-			}
-		}
-		diff := computeDiff(contents1, contents2, spec1.Name, spec2.Name)
+		diff := computeDiff(spec1.Contents, spec2.Contents, spec1.Name, spec2.Name)
 		if len(diff) > 0 {
 			fmt.Println(diff)
 		}

cmd/registry/cmd/get/get.go

@@ -19,9 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"strings"
 
-	"github.com/apigee/registry/cmd/registry/compress"
 	"github.com/apigee/registry/cmd/registry/patch"
 	"github.com/apigee/registry/pkg/connection"
 	"github.com/apigee/registry/pkg/encoding"
@@ -221,11 +219,7 @@ func (v *getVisitor) SpecHandler() visitor.SpecHandler {
 			if err := visitor.FetchSpecContents(ctx, v.registryClient, message); err != nil {
 				return err
 			}
-			contents := message.GetContents()
-			if strings.Contains(message.GetMimeType(), "+gzip") {
-				contents, _ = compress.GUnzippedBytes(contents)
-			}
-			v.results = append(v.results, contents)
+			v.results = append(v.results, message.GetContents())
 			return nil
 		case "yaml":
 			spec, err := patch.NewApiSpec(ctx, v.registryClient, message, v.nested)

cmd/registry/conformance/conformance-task.go

@@ -94,7 +94,7 @@ func (task *ComputeConformanceTask) String() string {
 func (task *ComputeConformanceTask) Run(ctx context.Context) error {
 	log.Debugf(ctx, "Computing conformance report %s/artifacts/%s", task.Spec.GetName(), conformanceReportId(task.StyleguideId))
 
-	data, err := visitor.GetBytesForSpec(ctx, task.Client, task.Spec)
+	err := visitor.FetchSpecContents(ctx, task.Client, task.Spec)
 	if err != nil {
 		return err
 	}
@@ -111,10 +111,10 @@ func (task *ComputeConformanceTask) Run(ctx context.Context) error {
 	defer os.RemoveAll(root)
 
 	if mime.IsZipArchive(task.Spec.GetMimeType()) {
-		_, err = compress.UnzipArchiveToPath(data, root)
+		_, err = compress.UnzipArchiveToPath(task.Spec.GetContents(), root)
 	} else {
 		// Write the file to the temporary directory.
-		err = os.WriteFile(filepath.Join(root, name), data, 0644)
+		err = os.WriteFile(filepath.Join(root, name), task.Spec.GetContents(), 0644)
 	}
 	if err != nil {
 		return err

cmd/registry/patch/project.go

@@ -20,9 +20,7 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
-	"strings"
 
-	"github.com/apigee/registry/cmd/registry/compress"
 	"github.com/apigee/registry/cmd/registry/tasks"
 	"github.com/apigee/registry/gapic"
 	"github.com/apigee/registry/pkg/connection"
@@ -31,7 +29,6 @@ import (
 	"github.com/apigee/registry/pkg/names"
 	"github.com/apigee/registry/pkg/visitor"
 	"github.com/apigee/registry/rpc"
-	"google.golang.org/grpc/metadata"
 	"gopkg.in/yaml.v3"
 )
 
@@ -489,18 +486,11 @@ func (task *exportSpecTask) Run(ctx context.Context) error {
 	if task.message.Filename == "" {
 		return nil
 	}
-	ctx = metadata.AppendToOutgoingContext(ctx, "accept-encoding", "gzip")
-	contents, err := task.client.GetApiSpecContents(ctx, &rpc.GetApiSpecContentsRequest{
-		Name: task.message.GetName(),
-	})
+	err = visitor.FetchSpecContents(ctx, task.client, task.message)
 	if err != nil {
 		return err
 	}
-	data := contents.GetData()
-	if strings.Contains(contents.GetContentType(), "+gzip") {
-		data, _ = compress.GUnzippedBytes(data)
-	}
-	return os.WriteFile(filepath.Join(parentDir, task.message.Filename), data, 0644)
+	return os.WriteFile(filepath.Join(parentDir, task.message.Filename), task.message.GetContents(), 0644)
 }
 
 type exportDeploymentTask struct {

pkg/mime/types.go

@@ -69,6 +69,11 @@ func IsGZipCompressed(mimeType string) bool {
 	return strings.Contains(mimeType, "+gzip")
 }
 
+// GUnzippedType returns an equivalent uncompressed MIME type for a compressed type.
+func GUnzippedType(mimeType string) string {
+	return strings.Replace(mimeType, "+gzip", "", 1)
+}
+
 // IsZipArchive returns true if a MIME type represents a type stored as a multifile Zip archive.
 func IsZipArchive(mimeType string) bool {
 	return strings.Contains(mimeType, "+zip")

pkg/mime/types_test.go

@@ -72,6 +72,12 @@ func TestOpenAPIMimeTypes(t *testing.T) {
 			if IsZipArchive(value) {
 				t.Errorf("%s is incorrectly recognized as a zip archive", value)
 			}
+			if IsGZipCompressed(value) {
+				unzipped := GUnzippedType(value)
+				if IsGZipCompressed(unzipped) {
+					t.Errorf("failed to remove compression from type %q", value)
+				}
+			}
 		})
 	}
 }