diff --git a/cmd/registry/cmd/compute/vocabulary.go b/cmd/registry/cmd/compute/vocabulary.go index 223cf9ac7..68340c48d 100644 --- a/cmd/registry/cmd/compute/vocabulary.go +++ b/cmd/registry/cmd/compute/vocabulary.go @@ -64,7 +64,7 @@ func vocabularyCommand() *cobra.Command { if err != nil { log.FromContext(ctx).WithError(err).Fatal("Failed to get jobs from flags") } - taskQueue, wait := core.WorkerPool(ctx, jobs) + taskQueue, wait := core.WorkerPoolWithWarnings(ctx, jobs) defer wait() parsed, err := names.ParseSpecRevision(path) diff --git a/cmd/registry/cmd/upload/bulk/discovery.go b/cmd/registry/cmd/upload/bulk/discovery.go index 78b20b896..30a0088c7 100644 --- a/cmd/registry/cmd/upload/bulk/discovery.go +++ b/cmd/registry/cmd/upload/bulk/discovery.go @@ -69,7 +69,7 @@ func discoveryCommand() *cobra.Command { parent: parent, apiID: sanitize(api.Name), versionID: sanitize(api.Version), - specID: "discovery.json", + specID: "discovery", } } return nil diff --git a/cmd/registry/core/print.go b/cmd/registry/core/print.go index 4b434597f..5fe75aac3 100644 --- a/cmd/registry/core/print.go +++ b/cmd/registry/core/print.go @@ -108,65 +108,70 @@ func PrintArtifactContents(artifact *rpc.Artifact) error { return nil } - messageType, err := MessageTypeForMimeType(artifact.GetMimeType()) + message, err := GetArtifactMessageContents(artifact) if err != nil { return err } + PrintMessage(message) + return nil +} + +func PrintMessage(message proto.Message) { + fmt.Println(protojson.Format(message)) +} +func GetArtifactMessageContents(artifact *rpc.Artifact) (proto.Message, error) { + messageType, err := MessageTypeForMimeType(artifact.GetMimeType()) + if err != nil { + return nil, err + } switch messageType { case "gnostic.metrics.Complexity": - return unmarshalAndPrint(artifact.GetContents(), &metrics.Complexity{}) + return unmarshal(artifact.GetContents(), &metrics.Complexity{}) case "gnostic.metrics.Vocabulary": - return unmarshalAndPrint(artifact.GetContents(), &metrics.Vocabulary{}) + return unmarshal(artifact.GetContents(), &metrics.Vocabulary{}) case "gnostic.metrics.VersionHistory": - return unmarshalAndPrint(artifact.GetContents(), &metrics.VersionHistory{}) - case "google.cloud.apigeeregistry.applications.v1alpha1.Lint": - return unmarshalAndPrint(artifact.GetContents(), &rpc.Lint{}) - case "google.cloud.apigeeregistry.applications.v1alpha1.LintStats": - return unmarshalAndPrint(artifact.GetContents(), &rpc.LintStats{}) + return unmarshal(artifact.GetContents(), &metrics.VersionHistory{}) case "google.cloud.apigeeregistry.v1.apihub.DisplaySettings": - return unmarshalAndPrint(artifact.GetContents(), &rpc.DisplaySettings{}) + return unmarshal(artifact.GetContents(), &rpc.DisplaySettings{}) case "google.cloud.apigeeregistry.v1.apihub.Lifecycle": - return unmarshalAndPrint(artifact.GetContents(), &rpc.Lifecycle{}) + return unmarshal(artifact.GetContents(), &rpc.Lifecycle{}) case "google.cloud.apigeeregistry.v1.apihub.ReferenceList": - return unmarshalAndPrint(artifact.GetContents(), &rpc.ReferenceList{}) + return unmarshal(artifact.GetContents(), &rpc.ReferenceList{}) case "google.cloud.apigeeregistry.v1.apihub.TaxonomyList": - return unmarshalAndPrint(artifact.GetContents(), &rpc.TaxonomyList{}) + return unmarshal(artifact.GetContents(), &rpc.TaxonomyList{}) case "google.cloud.apigeeregistry.v1.controller.Manifest": - return unmarshalAndPrint(artifact.GetContents(), &rpc.Manifest{}) + return unmarshal(artifact.GetContents(), &rpc.Manifest{}) case "google.cloud.apigeeregistry.v1.controller.Receipt": - return unmarshalAndPrint(artifact.GetContents(), &rpc.Receipt{}) + return unmarshal(artifact.GetContents(), &rpc.Receipt{}) case "google.cloud.apigeeregistry.v1.scoring.Score": - return unmarshalAndPrint(artifact.GetContents(), &rpc.Score{}) + return unmarshal(artifact.GetContents(), &rpc.Score{}) case "google.cloud.apigeeregistry.v1.scoring.ScoreCard": - return unmarshalAndPrint(artifact.GetContents(), &rpc.ScoreCard{}) + return unmarshal(artifact.GetContents(), &rpc.ScoreCard{}) case "google.cloud.apigeeregistry.v1.scoring.ScoreDefinition": - return unmarshalAndPrint(artifact.GetContents(), &rpc.ScoreDefinition{}) + return unmarshal(artifact.GetContents(), &rpc.ScoreDefinition{}) case "google.cloud.apigeeregistry.v1.scoring.ScoreCardDefinition": - return unmarshalAndPrint(artifact.GetContents(), &rpc.ScoreCardDefinition{}) + return unmarshal(artifact.GetContents(), &rpc.ScoreCardDefinition{}) case "google.cloud.apigeeregistry.v1.style.ConformanceReport": - return unmarshalAndPrint(artifact.GetContents(), &rpc.ConformanceReport{}) + return unmarshal(artifact.GetContents(), &rpc.ConformanceReport{}) + case "google.cloud.apigeeregistry.v1.style.Lint": + return unmarshal(artifact.GetContents(), &rpc.Lint{}) + case "google.cloud.apigeeregistry.v1.style.LintStats": + return unmarshal(artifact.GetContents(), &rpc.LintStats{}) case "google.cloud.apigeeregistry.v1.style.StyleGuide": - return unmarshalAndPrint(artifact.GetContents(), &rpc.StyleGuide{}) + return unmarshal(artifact.GetContents(), &rpc.StyleGuide{}) case "gnostic.openapiv2.Document": - return unmarshalAndPrint(artifact.GetContents(), &openapiv2.Document{}) + return unmarshal(artifact.GetContents(), &openapiv2.Document{}) case "gnostic.openapiv3.Document": - return unmarshalAndPrint(artifact.GetContents(), &openapiv3.Document{}) + return unmarshal(artifact.GetContents(), &openapiv3.Document{}) default: - // To get a parent context here would require changing the ArtifactHandler type in handlers.go - return fmt.Errorf("Unsupported message type: %s, please use --raw to get raw contents", messageType) + return nil, fmt.Errorf("unsupported message type: %s", messageType) } } -func PrintMessage(message proto.Message) { - fmt.Println(protojson.Format(message)) -} - -func unmarshalAndPrint(value []byte, message proto.Message) error { +func unmarshal(value []byte, message proto.Message) (proto.Message, error) { if err := proto.Unmarshal(value, message); err != nil { - return err + return nil, err } - - PrintMessage(message) - return nil + return message, nil } diff --git a/cmd/registry/patch/apply.go b/cmd/registry/patch/apply.go index 4d8578030..68d864f76 100644 --- a/cmd/registry/patch/apply.go +++ b/cmd/registry/patch/apply.go @@ -69,9 +69,6 @@ func (p *patchGroup) add(task *applyFileTask) error { return err } task.kind = header.Kind - if header.Metadata.Parent != "" { - task.parent = task.parent + "/" + header.Metadata.Parent - } switch task.kind { case "API": p.apiTasks = append(p.apiTasks, task) diff --git a/cmd/registry/patch/artifact.go b/cmd/registry/patch/artifact.go index befb96ea2..b0806ba98 100644 --- a/cmd/registry/patch/artifact.go +++ b/cmd/registry/patch/artifact.go @@ -29,6 +29,7 @@ import ( "github.com/apigee/registry/pkg/models" "github.com/apigee/registry/rpc" "github.com/apigee/registry/server/registry/names" + metrics "github.com/google/gnostic/metrics" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" ) @@ -164,8 +165,11 @@ func applyArtifactPatchBytes(ctx context.Context, client connection.RegistryClie return applyArtifactPatch(ctx, client, &artifact, parent) } -func artifactName(parent, artifactID string) (names.Artifact, error) { - return names.ParseArtifact(parent + "/artifacts/" + artifactID) +func artifactName(parent string, metadata models.Metadata) (names.Artifact, error) { + if metadata.Parent != "" { + parent = parent + "/" + metadata.Parent + } + return names.ParseArtifact(parent + "/artifacts/" + metadata.Name) } func applyArtifactPatch(ctx context.Context, client connection.RegistryClient, content *models.Artifact, parent string) error { @@ -177,7 +181,7 @@ func applyArtifactPatch(ctx context.Context, client connection.RegistryClient, c return err } // Populate Id and Kind fields in the contents of the artifact - j, err = populateIdAndKind(j, content.Kind, content.Metadata.Name) + jWithIdAndKind, err := populateIdAndKind(j, content.Kind, content.Metadata.Name) if err != nil { return err } @@ -187,16 +191,22 @@ func applyArtifactPatch(ctx context.Context, client connection.RegistryClient, c if err != nil { return err } - err = protojson.Unmarshal(j, m) + err = protojson.Unmarshal(jWithIdAndKind, m) if err != nil { - return err + if strings.Contains(err.Error(), "unknown field") { + // Try unmarshaling the original YAML (without the additional Id and Kind fields). + err = protojson.Unmarshal(j, m) + if err != nil { + return err + } + } } // Marshal the message struct to bytes. bytes, err := proto.Marshal(m) if err != nil { return err } - name, err := artifactName(parent, content.Header.Metadata.Name) + name, err := artifactName(parent, content.Header.Metadata) if err != nil { return err } @@ -295,4 +305,6 @@ var artifactMessageTypes map[string]messageFactory = map[string]messageFactory{ "google.cloud.apigeeregistry.v1.style.StyleGuide": func() proto.Message { return new(rpc.StyleGuide) }, "google.cloud.apigeeregistry.v1.style.ConformanceReport": func() proto.Message { return new(rpc.ConformanceReport) }, "google.cloud.apigeeregistry.v1.style.Lint": func() proto.Message { return new(rpc.Lint) }, + "gnostic.metrics.Complexity": func() proto.Message { return new(metrics.Complexity) }, + "gnostic.metrics.Vocabulary": func() proto.Message { return new(metrics.Vocabulary) }, } diff --git a/cmd/registry/patch/deployment.go b/cmd/registry/patch/deployment.go index 2ad94d8b7..44343cde8 100644 --- a/cmd/registry/patch/deployment.go +++ b/cmd/registry/patch/deployment.go @@ -106,12 +106,15 @@ func applyApiDeploymentPatchBytes(ctx context.Context, client connection.Registr return applyApiDeploymentPatch(ctx, client, &deployment, parent) } -func deploymentName(parent, deploymentID string) (names.Deployment, error) { +func deploymentName(parent string, metadata models.Metadata) (names.Deployment, error) { + if metadata.Parent != "" { + parent = parent + "/" + metadata.Parent + } api, err := names.ParseApi(parent) if err != nil { return names.Deployment{}, err } - return api.Deployment(deploymentID), nil + return api.Deployment(metadata.Name), nil } func applyApiDeploymentPatch( @@ -119,7 +122,7 @@ func applyApiDeploymentPatch( client connection.RegistryClient, deployment *models.ApiDeployment, parent string) error { - name, err := deploymentName(parent, deployment.Metadata.Name) + name, err := deploymentName(parent, deployment.Metadata) if err != nil { return err } diff --git a/cmd/registry/patch/patch_test.go b/cmd/registry/patch/patch_test.go new file mode 100644 index 000000000..ac1bf3413 --- /dev/null +++ b/cmd/registry/patch/patch_test.go @@ -0,0 +1,125 @@ +package patch + +import ( + "context" + "os" + "testing" + + "github.com/apigee/registry/cmd/registry/core" + "github.com/apigee/registry/pkg/connection" + "github.com/apigee/registry/pkg/connection/grpctest" + "github.com/apigee/registry/rpc" + "github.com/apigee/registry/server/registry" + "github.com/apigee/registry/server/registry/names" + "github.com/apigee/registry/server/registry/test/seeder" + metrics "github.com/google/gnostic/metrics" + "github.com/google/go-cmp/cmp" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" +) + +// TestMain will set up a local RegistryServer and grpc.Server for all +// tests in this package if APG_REGISTRY_ADDRESS env var is not set +// for the client. +func TestMain(m *testing.M) { + grpctest.TestMain(m, registry.Config{}) +} + +func TestSpecArtifactPatches(t *testing.T) { + tests := []struct { + artifactName string + yamlFileName string + message proto.Message + }{ + { + artifactName: "complexity", + yamlFileName: "testdata/artifacts/complexity.yaml", + message: &metrics.Complexity{ + PathCount: 76, + GetCount: 25, + PostCount: 27, + PutCount: 11, + DeleteCount: 13, + SchemaCount: 1150, + SchemaPropertyCount: 964, + }, + }, + { + artifactName: "vocabulary", + yamlFileName: "testdata/artifacts/vocabulary.yaml", + message: &metrics.Vocabulary{ + Name: "sample-name", + Schemas: []*metrics.WordCount{{Word: "sample-schema", Count: 1}}, + Properties: []*metrics.WordCount{{Word: "sample-property", Count: 2}}, + Operations: []*metrics.WordCount{{Word: "sample-operation", Count: 3}}, + Parameters: []*metrics.WordCount{{Word: "sample-parameter", Count: 4}}, + }, + }, + } + ctx := context.Background() + adminClient, err := connection.NewAdminClient(ctx) + if err != nil { + t.Fatalf("Setup: failed to create client: %+v", err) + } + defer adminClient.Close() + registryClient, err := connection.NewRegistryClient(ctx) + if err != nil { + t.Fatalf("Setup: Failed to create registry client: %s", err) + } + defer registryClient.Close() + client := seeder.Client{ + RegistryClient: registryClient, + AdminClient: adminClient, + } + spec := &rpc.ApiSpec{ + Name: "projects/patch-project-test/locations/global/apis/a/versions/v/specs/s", + } + if err := seeder.SeedSpecs(ctx, client, spec); err != nil { + t.Fatalf("Setup/Seeding: Failed to seed registry: %s", err) + } + for _, test := range tests { + t.Run(test.artifactName, func(t *testing.T) { + b, err := os.ReadFile(test.yamlFileName) + if err != nil { + t.Fatalf("%s", err) + } + err = applyArtifactPatchBytes(ctx, registryClient, b, "projects/patch-project-test/locations/global") + if err != nil { + t.Fatalf("%s", err) + } + artifactName, err := names.ParseArtifact("projects/patch-project-test/locations/global/apis/a/versions/v/specs/s/artifacts/" + test.artifactName) + if err != nil { + t.Fatalf("%s", err) + } + err = core.GetArtifact(ctx, registryClient, artifactName, true, + func(artifact *rpc.Artifact) error { + contents, err := core.GetArtifactMessageContents(artifact) + if err != nil { + t.Fatalf("%s", err) + } + opts := cmp.Options{protocmp.Transform()} + if !cmp.Equal(test.message, contents, opts) { + t.Errorf("GetDiff returned unexpected diff (-want +got):\n%s", cmp.Diff(test.message, contents, opts)) + } + out, header, err := ExportArtifact(ctx, registryClient, artifact) + if err != nil { + t.Fatalf("%s", err) + } + wantedParent := "apis/a/versions/v/specs/s" + if header.Metadata.Parent != wantedParent { + t.Errorf("Incorrect export parent. Wanted %s, got %s", wantedParent, header.Metadata.Parent) + } + if header.Metadata.Name != test.artifactName { + t.Errorf("Incorrect export name. Wanted %s, got %s", test.artifactName, header.Metadata.Name) + } + if !cmp.Equal(b, out, opts) { + t.Errorf("GetDiff returned unexpected diff (-want +got):\n%s", cmp.Diff(b, out, opts)) + } + return nil + }) + if err != nil { + t.Fatalf("%s", err) + } + }) + } +} diff --git a/cmd/registry/patch/spec.go b/cmd/registry/patch/spec.go index 846009fa2..98b2686a9 100644 --- a/cmd/registry/patch/spec.go +++ b/cmd/registry/patch/spec.go @@ -93,12 +93,15 @@ func applyApiSpecPatchBytes( return applyApiSpecPatch(ctx, client, &spec, parent) } -func specName(parent, specID string) (names.Spec, error) { +func specName(parent string, metadata models.Metadata) (names.Spec, error) { + if metadata.Parent != "" { + parent = parent + "/" + metadata.Parent + } version, err := names.ParseVersion(parent) if err != nil { return names.Spec{}, err } - return version.Spec(specID), nil + return version.Spec(metadata.Name), nil } func applyApiSpecPatch( @@ -106,7 +109,7 @@ func applyApiSpecPatch( client connection.RegistryClient, spec *models.ApiSpec, parent string) error { - name, err := specName(parent, spec.Metadata.Name) + name, err := specName(parent, spec.Metadata) if err != nil { return err } diff --git a/cmd/registry/patch/testdata/artifacts/complexity.yaml b/cmd/registry/patch/testdata/artifacts/complexity.yaml new file mode 100644 index 000000000..277c71b9b --- /dev/null +++ b/cmd/registry/patch/testdata/artifacts/complexity.yaml @@ -0,0 +1,13 @@ +apiVersion: apigeeregistry/v1 +kind: Complexity +metadata: + name: complexity + parent: apis/a/versions/v/specs/s +data: + pathCount: 76 + getCount: 25 + postCount: 27 + putCount: 11 + deleteCount: 13 + schemaCount: 1150 + schemaPropertyCount: 964 diff --git a/cmd/registry/patch/testdata/artifacts/vocabulary.yaml b/cmd/registry/patch/testdata/artifacts/vocabulary.yaml new file mode 100755 index 000000000..cbdb54d6e --- /dev/null +++ b/cmd/registry/patch/testdata/artifacts/vocabulary.yaml @@ -0,0 +1,19 @@ +apiVersion: apigeeregistry/v1 +kind: Vocabulary +metadata: + name: vocabulary + parent: apis/a/versions/v/specs/s +data: + name: sample-name + schemas: + - word: sample-schema + count: 1 + properties: + - word: sample-property + count: 2 + operations: + - word: sample-operation + count: 3 + parameters: + - word: sample-parameter + count: 4 diff --git a/cmd/registry/patch/version.go b/cmd/registry/patch/version.go index 071f8b06f..ee213dfff 100644 --- a/cmd/registry/patch/version.go +++ b/cmd/registry/patch/version.go @@ -104,12 +104,15 @@ func applyApiVersionPatchBytes( return applyApiVersionPatch(ctx, client, &version, parent) } -func versionName(parent, versionID string) (names.Version, error) { +func versionName(parent string, metadata models.Metadata) (names.Version, error) { + if metadata.Parent != "" { + parent = parent + "/" + metadata.Parent + } api, err := names.ParseApi(parent) if err != nil { return names.Version{}, err } - return api.Version(versionID), nil + return api.Version(metadata.Name), nil } func applyApiVersionPatch( @@ -117,7 +120,7 @@ func applyApiVersionPatch( client connection.RegistryClient, version *models.ApiVersion, parent string) error { - name, err := versionName(parent, version.Metadata.Name) + name, err := versionName(parent, version.Metadata) if err != nil { return err } diff --git a/demos/disco.sh b/demos/disco.sh index 96248c9ac..603654dbc 100755 --- a/demos/disco.sh +++ b/demos/disco.sh @@ -45,27 +45,27 @@ registry list projects/$PROJECT/locations/global/apis | wc -l registry list projects/$PROJECT/locations/global/apis/-/versions # Similarly, we can use wildcards for the version ids and list all of the specs. -# Here you'll see that the spec IDs are "discovery.json". This was set in the registry +# Here you'll see that the spec IDs are "discovery". This was set in the registry # tool, which uploaded each API description as gzipped JSON. registry list projects/$PROJECT/locations/global/apis/-/versions/-/specs # To see more about an individual spec, use the `registry get` command: -registry get projects/$PROJECT/locations/global/apis/translate/versions/v3/specs/discovery.json +registry get projects/$PROJECT/locations/global/apis/translate/versions/v3/specs/discovery # You can also get this with direct calls to the registry rpc service: registry rpc get-api-spec \ - --name projects/$PROJECT/locations/global/apis/translate/versions/v3/specs/discovery.json + --name projects/$PROJECT/locations/global/apis/translate/versions/v3/specs/discovery # Add the `--json` flag to get this as JSON: registry rpc get-api-spec --json \ - --name projects/$PROJECT/locations/global/apis/translate/versions/v3/specs/discovery.json + --name projects/$PROJECT/locations/global/apis/translate/versions/v3/specs/discovery # You might notice that didn't return the actual spec. That's because the spec contents # are accessed through a separate method that (when transcoded to HTTP) allows direct download # of spec contents. registry rpc get-api-spec-contents \ - --name projects/$PROJECT/locations/global/apis/translate/versions/v3/specs/discovery.json + --name projects/$PROJECT/locations/global/apis/translate/versions/v3/specs/discovery # Another way to get the bytes of the spec is to use `registry get` with the `--contents` flag. -registry get projects/$PROJECT/locations/global/apis/translate/versions/v3/specs/discovery.json \ +registry get projects/$PROJECT/locations/global/apis/translate/versions/v3/specs/discovery \ --contents