Add support for more apihub resources, add YAML import of API-owned a…

…rtifacts. (#503)

* protos for apihub display settings and references

* add generated code for new resources

* print display settings and references using "registry get"

* extend "upload artifact" to display settings and references

* Add support for YAML apply/export of display settings and references.

This includes a general upgrade from yaml/v2 to yaml/v3.

* Fix linter error.

* Update YAML import/export tests for yaml.v3

The yaml.v3 library uses different indentation than v2,
so the "golden" yaml needed an update to match what was
generated by v3. It also uses 4-character spacing, which
seems excessive for our applications, so this adds a helper
that provides 2-character marshalling.

I also noticed that the comparison order was wrong
in calls to cmp.Diff and fixed that.

* Combine parallel switch cases.

* Add artifact to sample API YAML in apply/testdata

* Remove duplicate "Exporting" message.

* When applying patches, skip non-yaml files and log each file imported.

* Allow ?recursive={true,false} on fileurls to control directory upload

* Fix linter error.

* Use if-with-initializer as suggested

* rework if conditions in directory walks

* restructure patch constructors

* Fix some error strings (removing initial capitalization)

* Add comment explaining polymorphic artifact marshalling/unmarshalling.

cmd/registry/cmd/apply/apply_test.go

@@ -88,7 +88,7 @@ func TestApply(t *testing.T) {
 			t.Fatalf("ExportApi(%+v) returned an error: %s", got, err)
 		}
 
-		if diff := cmp.Diff(actual, expected); diff != "" {
+		if diff := cmp.Diff(expected, actual); diff != "" {
 			t.Errorf("GetApi(%q) returned unexpected diff: (-want +got):\n%s", got, diff)
 		}
 	}
@@ -122,7 +122,7 @@ func TestApply(t *testing.T) {
 			t.Fatalf("ExportArtifact(%+v) returned an error: %s", message, err)
 		}
 
-		if diff := cmp.Diff(actual, expected); diff != "" {
+		if diff := cmp.Diff(expected, actual); diff != "" {
 			t.Errorf("GetArtifact(%q) returned unexpected diff: (-want +got):\n%s", message, diff)
 		}
 	}

cmd/registry/cmd/apply/testdata/registry.yaml

@@ -50,3 +50,18 @@ data:
         externalChannelURI: https://apigee.github.io/registry/
         intendedAudience: Public
         accessGuidance: See https://github.com/apigee/registry for tools and usage information.
+  artifacts:
+    - kind: ReferenceList
+      metadata:
+        name: apihub-related
+      data:
+        description: Defines a list of related resources
+        references:
+          - id: github
+            displayName: GitHub Repo
+            category: apihub-source-code
+            uri: https://github.com/apigee/registry
+          - id: docs
+            displayName: GitHub Documentation
+            category: apihub-other
+            uri: https://apigee.github.io/registry/

cmd/registry/cmd/upload/artifact.go

@@ -52,8 +52,7 @@ func artifactCommand() *cobra.Command {
 				log.FromContext(ctx).WithError(err).Fatal("Failed to get client")
 			}
 			log.Debugf(ctx, "Uploading %s", artifact.Name)
-			err = core.SetArtifact(ctx, client, artifact)
-			if err != nil {
+			if err = core.SetArtifact(ctx, client, artifact); err != nil {
 				log.FromContext(ctx).WithError(err).Fatal("Failed to save artifact")
 			}
 		},
@@ -75,21 +74,24 @@ func buildArtifact(ctx context.Context, parent string, filename string) (*rpc.Ar
 		Kind string `yaml:"kind"`
 	}
 	var header ArtifactHeader
-	err = yaml.Unmarshal(yamlBytes, &header)
-	if err != nil {
+	if err = yaml.Unmarshal(yamlBytes, &header); err != nil {
 		return nil, err
 	}
 
 	// read the specified kind of artifact
 	jsonBytes, _ := yaml.YAMLToJSON(yamlBytes) // to use protojson.Unmarshal()
 	var artifact *rpc.Artifact
 	switch header.Kind {
+	case "DisplaySettings", patch.DisplaySettingsMimeType:
+		artifact, err = buildDisplaySettingsArtifact(ctx, jsonBytes)
+	case "Lifecycle", patch.LifecycleMimeType:
+		artifact, err = buildLifecycleArtifact(ctx, jsonBytes)
 	case "Manifest", patch.ManifestMimeType:
 		artifact, err = buildManifestArtifact(ctx, parent, jsonBytes)
-	case "TaxonomyList", "google.cloud.apigeeregistry.v1.apihub.TaxonomyList":
+	case "ReferenceList", patch.ReferenceListMimeType:
+		artifact, err = buildReferenceListArtifact(ctx, jsonBytes)
+	case "TaxonomyList", patch.TaxonomyListMimeType:
 		artifact, err = buildTaxonomyListArtifact(ctx, jsonBytes)
-	case "Lifecycle", "google.cloud.apigeeregistry.v1.apihub.Lifecycle":
-		artifact, err = buildLifecycleArtifact(ctx, jsonBytes)
 	default:
 		err = fmt.Errorf("unsupported artifact type %s", header.Kind)
 	}
@@ -102,34 +104,61 @@ func buildArtifact(ctx context.Context, parent string, filename string) (*rpc.Ar
 	return artifact, nil
 }
 
+func buildDisplaySettingsArtifact(ctx context.Context, jsonBytes []byte) (*rpc.Artifact, error) {
+	m := &rpc.DisplaySettings{}
+	if err := protojson.Unmarshal(jsonBytes, m); err != nil {
+		return nil, err
+	}
+	artifactBytes, err := proto.Marshal(m)
+	if err != nil {
+		return nil, err
+	}
+	return &rpc.Artifact{
+		Contents: artifactBytes,
+		MimeType: patch.DisplaySettingsMimeType,
+	}, nil
+}
+
+func buildLifecycleArtifact(ctx context.Context, jsonBytes []byte) (*rpc.Artifact, error) {
+	m := &rpc.Lifecycle{}
+	if err := protojson.Unmarshal(jsonBytes, m); err != nil {
+		return nil, err
+	}
+	artifactBytes, err := proto.Marshal(m)
+	if err != nil {
+		return nil, err
+	}
+	return &rpc.Artifact{
+		Contents: artifactBytes,
+		MimeType: patch.LifecycleMimeType,
+	}, nil
+}
+
 func buildManifestArtifact(ctx context.Context, parent string, jsonBytes []byte) (*rpc.Artifact, error) {
 	m := &rpc.Manifest{}
-	err := protojson.Unmarshal(jsonBytes, m)
-	if err != nil {
+	if err := protojson.Unmarshal(jsonBytes, m); err != nil {
 		return nil, err
 	}
 	errs := controller.ValidateManifest(ctx, parent, m)
 	if count := len(errs); count > 0 {
-		for _, err = range errs {
+		for _, err := range errs {
 			log.FromContext(ctx).WithError(err).Error("Manifest error")
 		}
 		return nil, fmt.Errorf("manifest definition contains %d error(s): see logs for details", count)
 	}
-
 	artifactBytes, err := proto.Marshal(m)
 	if err != nil {
 		return nil, err
 	}
 	return &rpc.Artifact{
 		Contents: artifactBytes,
-		MimeType: core.MimeTypeForMessageType("google.cloud.apigeeregistry.v1.controller.Manifest"),
+		MimeType: patch.ManifestMimeType,
 	}, nil
 }
 
-func buildTaxonomyListArtifact(ctx context.Context, jsonBytes []byte) (*rpc.Artifact, error) {
-	m := &rpc.TaxonomyList{}
-	err := protojson.Unmarshal(jsonBytes, m)
-	if err != nil {
+func buildReferenceListArtifact(ctx context.Context, jsonBytes []byte) (*rpc.Artifact, error) {
+	m := &rpc.ReferenceList{}
+	if err := protojson.Unmarshal(jsonBytes, m); err != nil {
 		return nil, err
 	}
 	artifactBytes, err := proto.Marshal(m)
@@ -138,14 +167,13 @@ func buildTaxonomyListArtifact(ctx context.Context, jsonBytes []byte) (*rpc.Arti
 	}
 	return &rpc.Artifact{
 		Contents: artifactBytes,
-		MimeType: core.MimeTypeForMessageType("google.cloud.apigeeregistry.v1.controller.TaxonomyList"),
+		MimeType: patch.ReferenceListMimeType,
 	}, nil
 }
 
-func buildLifecycleArtifact(ctx context.Context, jsonBytes []byte) (*rpc.Artifact, error) {
-	m := &rpc.Lifecycle{}
-	err := protojson.Unmarshal(jsonBytes, m)
-	if err != nil {
+func buildTaxonomyListArtifact(ctx context.Context, jsonBytes []byte) (*rpc.Artifact, error) {
+	m := &rpc.TaxonomyList{}
+	if err := protojson.Unmarshal(jsonBytes, m); err != nil {
 		return nil, err
 	}
 	artifactBytes, err := proto.Marshal(m)
@@ -154,6 +182,6 @@ func buildLifecycleArtifact(ctx context.Context, jsonBytes []byte) (*rpc.Artifac
 	}
 	return &rpc.Artifact{
 		Contents: artifactBytes,
-		MimeType: core.MimeTypeForMessageType("google.cloud.apigeeregistry.v1.controller.Lifecycle"),
+		MimeType: patch.TaxonomyListMimeType,
 	}, nil
 }

cmd/registry/cmd/upload/bulk/protos.go

@@ -290,7 +290,7 @@ func (task *uploadProtoTask) fileName() string {
 
 func (task *uploadProtoTask) zipContents() ([]byte, error) {
 	prefix := task.directory + "/"
-	contents, err := core.ZipArchiveOfPath(task.path, prefix)
+	contents, err := core.ZipArchiveOfPath(task.path, prefix, true)
 	if err != nil {
 		return nil, err
 	}

cmd/registry/cmd/upload/spec.go

@@ -91,7 +91,7 @@ func uploadSpecDirectory(ctx context.Context, dirname string, client *gapic.Regi
 	prefix := dirname + "/"
 	// build a zip archive with the contents of the path
 	// https://golangcode.com/create-zip-files-in-go/
-	buf, err := core.ZipArchiveOfPath(dirname, prefix)
+	buf, err := core.ZipArchiveOfPath(dirname, prefix, true)
 	if err != nil {
 		return err
 	}

cmd/registry/core/print.go

@@ -121,8 +121,12 @@ func PrintArtifactContents(artifact *rpc.Artifact) error {
 		return unmarshalAndPrint(artifact.GetContents(), &rpc.Index{})
 	case "google.cloud.apigeeregistry.applications.v1alpha1.Lint":
 		return unmarshalAndPrint(artifact.GetContents(), &rpc.Lint{})
+	case "google.cloud.apigeeregistry.v1.apihub.DisplaySettings":
+		return unmarshalAndPrint(artifact.GetContents(), &rpc.DisplaySettings{})
 	case "google.cloud.apigeeregistry.v1.apihub.Lifecycle":
 		return unmarshalAndPrint(artifact.GetContents(), &rpc.Lifecycle{})
+	case "google.cloud.apigeeregistry.v1.apihub.ReferenceList":
+		return unmarshalAndPrint(artifact.GetContents(), &rpc.ReferenceList{})
 	case "google.cloud.apigeeregistry.v1.apihub.TaxonomyList":
 		return unmarshalAndPrint(artifact.GetContents(), &rpc.TaxonomyList{})
 	case "google.cloud.apigeeregistry.v1.controller.Manifest":

cmd/registry/core/zip.go

@@ -19,6 +19,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
@@ -101,23 +102,24 @@ func UnzipArchiveToMap(b []byte) (map[string][]byte, error) {
 // ZipArchiveOfPath reads the contents of a path into a zip archive.
 // The specified prefix is stripped from file names in the archive.
 // Based on an example published at https://golangcode.com/create-zip-files-in-go/
-func ZipArchiveOfPath(path, prefix string) (buf bytes.Buffer, err error) {
+func ZipArchiveOfPath(path, prefix string, recursive bool) (buf bytes.Buffer, err error) {
 	zipWriter := zip.NewWriter(&buf)
 	defer zipWriter.Close()
 
-	_ = filepath.Walk(path, func(p string, info os.FileInfo, err error) error {
+	err = filepath.WalkDir(path, func(p string, entry fs.DirEntry, err error) error {
 		if err != nil {
 			return err
-		}
-		if info.IsDir() {
-			return nil
+		} else if entry.IsDir() && p != path && !recursive {
+			return filepath.SkipDir // Skip the directory and contents.
+		} else if entry.IsDir() {
+			return nil // Do nothing for the directory, but still walk its contents.
 		}
 		if err = addFileToZip(zipWriter, p, prefix); err != nil {
 			return err
 		}
 		return nil
 	})
-	return buf, nil
+	return buf, err
 }
 
 func addFileToZip(zipWriter *zip.Writer, filename, prefix string) error {

cmd/registry/patch/api.go

@@ -47,61 +47,81 @@ func newApi(ctx context.Context, client *gapic.RegistryClient, message *rpc.Api)
 	if err != nil {
 		return nil, err
 	}
-	api := &Api{
-		Header: Header{
-			ApiVersion: RegistryV1,
-			Kind:       "API",
-			Metadata: Metadata{
-				Name:        apiName.ApiID,
-				Labels:      message.Labels,
-				Annotations: message.Annotations,
-			},
-		},
-		Data: ApiData{
-			DisplayName:  message.DisplayName,
-			Description:  message.Description,
-			Availability: message.Availability,
-		},
-	}
-	api.Data.RecommendedVersion, err = relativeVersionName(apiName, message.RecommendedVersion)
+	recommendedVersion, err := relativeVersionName(apiName, message.RecommendedVersion)
 	if err != nil {
 		return nil, err
 	}
-	api.Data.RecommendedDeployment, err = relativeDeploymentName(apiName, message.RecommendedDeployment)
+	recommendedDeployment, err := relativeDeploymentName(apiName, message.RecommendedDeployment)
 	if err != nil {
 		return nil, err
 	}
-
-	err = core.ListVersions(ctx, client, apiName.Version("-"), "", func(message *rpc.ApiVersion) error {
+	versions := make([]*ApiVersion, 0)
+	if err = core.ListVersions(ctx, client, apiName.Version("-"), "", func(message *rpc.ApiVersion) error {
 		var version *ApiVersion
 		version, err := newApiVersion(ctx, client, message)
 		if err != nil {
 			return err
 		}
-
 		// unset these because they can be inferred
 		version.ApiVersion = ""
 		version.Kind = ""
-		api.Data.ApiVersions = append(api.Data.ApiVersions, version)
+		versions = append(versions, version)
 		return nil
-	})
-	if err != nil {
+	}); err != nil {
 		return nil, err
 	}
-
-	return api, core.ListDeployments(ctx, client, apiName.Deployment("-"), "", func(message *rpc.ApiDeployment) error {
+	deployments := make([]*ApiDeployment, 0)
+	if err = core.ListDeployments(ctx, client, apiName.Deployment("-"), "", func(message *rpc.ApiDeployment) error {
 		var deployment *ApiDeployment
 		deployment, err = newApiDeployment(ctx, client, message)
 		if err != nil {
 			return err
 		}
-
 		// unset these because they can be inferred
 		deployment.ApiVersion = ""
 		deployment.Kind = ""
-		api.Data.ApiDeployments = append(api.Data.ApiDeployments, deployment)
+		deployments = append(deployments, deployment)
 		return nil
-	})
+	}); err != nil {
+		return nil, err
+	}
+	artifacts := make([]*Artifact, 0)
+	if err = core.ListArtifacts(ctx, client, apiName.Artifact("-"), "", true, func(message *rpc.Artifact) error {
+		var artifact *Artifact
+		artifact, err = newArtifact(message)
+		if err != nil {
+			return err
+		}
+		// skip unsupported artifact types, "Artifact" is the generic type
+		if artifact.Kind != "Artifact" {
+			artifact.ApiVersion = ""
+			artifacts = append(artifacts, artifact)
+		}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return &Api{
+		Header: Header{
+			ApiVersion: RegistryV1,
+			Kind:       "API",
+			Metadata: Metadata{
+				Name:        apiName.ApiID,
+				Labels:      message.Labels,
+				Annotations: message.Annotations,
+			},
+		},
+		Data: ApiData{
+			DisplayName:           message.DisplayName,
+			Description:           message.Description,
+			Availability:          message.Availability,
+			RecommendedVersion:    recommendedVersion,
+			RecommendedDeployment: recommendedDeployment,
+			ApiVersions:           versions,
+			ApiDeployments:        deployments,
+			Artifacts:             artifacts,
+		},
+	}, err
 }
 
 // ExportAPI allows an API to be individually exported as a YAML file.
@@ -217,5 +237,11 @@ func applyApiPatch(ctx context.Context, client connection.Client, bytes []byte,
 			return err
 		}
 	}
+	for _, artifactPatch := range api.Data.Artifacts {
+		err = applyArtifactPatch(ctx, client, artifactPatch, apiName.String())
+		if err != nil {
+			return err
+		}
+	}
 	return nil
 }