Skip to content

Commit

Permalink
Update 'compute lint' to use linter plugins. (#1125)
Browse files Browse the repository at this point in the history
  • Loading branch information
@timburks
timburks authored Mar 30, 2023
1 parent 8155257 commit a38ff58
Show file tree
Hide file tree
Showing 8 changed files with 565 additions and 126 deletions.
88 changes: 39 additions & 49 deletions cmd/registry/cmd/compute/lint/lint.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,14 @@ package lint

import (
"context"
"errors"
"fmt"
"os"
"os/exec"

"github.com/apigee/registry/cmd/registry/conformance"
"github.com/apigee/registry/cmd/registry/tasks"
"github.com/apigee/registry/pkg/application/style"
"github.com/apigee/registry/pkg/connection"
"github.com/apigee/registry/pkg/log"
"github.com/apigee/registry/pkg/mime"
"github.com/apigee/registry/pkg/names"
"github.com/apigee/registry/pkg/visitor"
Expand All @@ -32,47 +34,57 @@ import (
)

func Command() *cobra.Command {
var linter string
cmd := &cobra.Command{
Use: "lint SPEC",
Short: "Compute lint results for API specs",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
c, err := connection.ActiveConfig()
if err != nil {
log.FromContext(ctx).WithError(err).Fatal("Failed to get config")
return err
}
args[0] = c.FQName(args[0])

linter, err := cmd.Flags().GetString("linter")
if err != nil {
return fmt.Errorf("failed to get linter from flags: %s", err)
}
if linter == "" {
return errors.New("--linter argument cannot be empty")
}
if _, err = exec.LookPath(fmt.Sprintf("registry-lint-%s", linter)); err != nil {
return err
}

filter, err := cmd.Flags().GetString("filter")
if err != nil {
log.FromContext(ctx).WithError(err).Fatal("Failed to get filter from flags")
return fmt.Errorf("failed to get filter from flags: %s", err)
}
dryRun, err := cmd.Flags().GetBool("dry-run")
if err != nil {
log.FromContext(ctx).WithError(err).Fatal("Failed to get fry-run from flags")
return fmt.Errorf("failed to get dry-run from flags: %s", err)
}

client, err := connection.NewRegistryClientWithSettings(ctx, c)
if err != nil {
log.FromContext(ctx).WithError(err).Fatal("Failed to get client")
return err
}
// Initialize task queue.
jobs, err := cmd.Flags().GetInt("jobs")
if err != nil {
log.FromContext(ctx).WithError(err).Fatal("Failed to get jobs from flags")
return fmt.Errorf("failed to get jobs from flags: %s", err)
}
taskQueue, wait := tasks.WorkerPoolIgnoreError(ctx, jobs)
defer wait()

spec, err := names.ParseSpec(args[0])
if err != nil {
log.FromContext(ctx).WithError(err).Fatal("Failed parse")
return fmt.Errorf("invalid spec pattern %s", args[0])
}

// Iterate through a collection of specs and evaluate each.
err = visitor.ListSpecs(ctx, client, spec, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
return visitor.ListSpecs(ctx, client, spec, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
taskQueue <- &computeLintTask{
client: client,
spec: spec,
Expand All @@ -81,13 +93,14 @@ func Command() *cobra.Command {
}
return nil
})
if err != nil {
log.FromContext(ctx).WithError(err).Fatal("Failed to list specs")
}
},
}

cmd.Flags().StringVar(&linter, "linter", "", "the linter to use (aip|spectral|gnostic)")
cmd.Flags().String("linter", "", "the linter to use")
_ = cmd.MarkFlagRequired("linter")
cmd.Flags().String("filter", "", "Filter selected resources")
cmd.Flags().Bool("dry-run", false, "if set, computation results will only be printed and will not stored in the registry")
cmd.Flags().Int("jobs", 10, "Number of actions to perform concurrently")
return cmd
}

Expand All @@ -107,50 +120,27 @@ func lintRelation(linter string) string {
}

func (task *computeLintTask) Run(ctx context.Context) error {
spec := task.spec
err := visitor.FetchSpecContents(ctx, task.client, spec)
root, err := conformance.WriteSpecForLinting(ctx, task.client, task.spec)
if root != "" {
defer os.RemoveAll(root)
}
if err != nil {
return err
}
var relation string
var lint *style.Lint
if mime.IsOpenAPIv2(spec.GetMimeType()) || mime.IsOpenAPIv3(spec.GetMimeType()) {
// the default openapi linter is gnostic
if task.linter == "" {
task.linter = "gnostic"
}
relation = lintRelation(task.linter)
log.Debugf(ctx, "Computing %s/artifacts/%s", spec.Name, relation)
lint, err = NewLintFromOpenAPI(spec.Name, spec.Contents, task.linter)
if err != nil {
return fmt.Errorf("error processing OpenAPI: %s (%s)", spec.Name, err.Error())
}
} else if mime.IsDiscovery(spec.GetMimeType()) {
return fmt.Errorf("unsupported Discovery document: %s", spec.Name)
} else if mime.IsProto(spec.GetMimeType()) && mime.IsZipArchive(spec.GetMimeType()) {
// the default proto linter is the aip linter
if task.linter == "" {
task.linter = "aip"
}
relation = lintRelation(task.linter)
log.Debugf(ctx, "Computing %s/artifacts/%s", spec.Name, relation)
lint, err = NewLintFromZippedProtos(spec.Name, spec.Contents)
if err != nil {
return fmt.Errorf("error processing protos: %s (%s)", spec.Name, err.Error())
}
} else {
return fmt.Errorf("we don't know how to lint %s", spec.Name)
linterMetadata := conformance.SimpleLinterMetadata(task.linter)
response, err := conformance.RunLinter(ctx, root, linterMetadata)
if err != nil {
return err
}

lint := response.Lint
if task.dryRun {
fmt.Println(protojson.Format((lint)))
return nil
}

subject := spec.GetName()
subject := task.spec.GetName()
messageData, _ := proto.Marshal(lint)
artifact := &rpc.Artifact{
Name: subject + "/artifacts/" + relation,
Name: subject + "/artifacts/" + lintRelation(task.linter),
MimeType: mime.MimeTypeForMessageType("google.cloud.apigeeregistry.v1.style.Lint"),
Contents: messageData,
}
Expand Down
117 changes: 117 additions & 0 deletions cmd/registry/cmd/compute/lint/lint_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,18 @@
package lint

import (
"context"
"testing"

"github.com/apigee/registry/cmd/registry/cmd/apply"
"github.com/apigee/registry/pkg/application/style"
"github.com/apigee/registry/pkg/connection"
"github.com/apigee/registry/pkg/connection/grpctest"
"github.com/apigee/registry/pkg/names"
"github.com/apigee/registry/pkg/visitor"
"github.com/apigee/registry/rpc"
"github.com/apigee/registry/server/registry"
"google.golang.org/protobuf/proto"
)

// TestMain will set up a local RegistryServer and grpc.Server for all
Expand All @@ -30,7 +38,116 @@ func TestMain(m *testing.M) {

func TestLint(t *testing.T) {
command := Command()
command.SilenceErrors = true
command.SilenceUsage = true
if err := command.Execute(); err == nil {
t.Fatalf("Execute() with no args succeeded and should have failed")
}
}

func TestInvalidComputeLint(t *testing.T) {
tests := []struct {
desc string
args []string
}{
{
desc: "no-linter-specified",
args: []string{"spec"},
},
{
desc: "missing-linter-specified",
args: []string{"spec", "--linter", "nonexistent"},
},
{
desc: "empty-linter-specified",
args: []string{"spec", "--linter", ""},
},
}
for _, test := range tests {
t.Run(test.desc, func(t *testing.T) {
command := Command()
command.SilenceErrors = true
command.SilenceUsage = true
command.SetArgs(test.args)
if err := command.Execute(); err == nil {
t.Fatalf("Execute() with no args succeeded and should have failed")
}
})
}
}

func TestComputeLint(t *testing.T) {
project := names.Project{ProjectID: "lint-test"}
ctx := context.Background()
registryClient, _ := grpctest.SetupRegistry(ctx, t, project.ProjectID, nil)

config, err := connection.ActiveConfig()
if err != nil {
t.Fatalf("Setup: Failed to get registry configuration: %s", err)
}
config.Project = project.ProjectID
connection.SetConfig(config)

applyCmd := apply.Command()
applyCmd.SetArgs([]string{"-f", "../complexity/testdata/apigeeregistry", "-R"})
if err := applyCmd.Execute(); err != nil {
t.Fatalf("Failed to apply test API")
}

t.Run("protos", func(t *testing.T) {
specName := project.Api("apigeeregistry").Version("v1").Spec("protos")
lintCmd := Command()
lintCmd.SetArgs([]string{specName.String(), "--linter", "test"})
if err := lintCmd.Execute(); err != nil {
t.Fatalf("Compute lint failed: %s", err)
}
artifactName := specName.Artifact("lint-test")
if err = visitor.GetArtifact(ctx, registryClient, artifactName, true, func(ctx context.Context, message *rpc.Artifact) error {
var lint style.Lint
if err = proto.Unmarshal(message.Contents, &lint); err != nil {
return err
}
return nil
}); err != nil {
t.Fatalf("Error getting artifact: %s", err)
}
})

t.Run("openapi", func(t *testing.T) {
specName := project.Api("apigeeregistry").Version("v1").Spec("openapi")
lintCmd := Command()
lintCmd.SetArgs([]string{specName.String(), "--linter", "test"})
if err := lintCmd.Execute(); err != nil {
t.Fatalf("Compute lint failed: %s", err)
}
artifactName := specName.Artifact("lint-test")
if err = visitor.GetArtifact(ctx, registryClient, artifactName, true, func(ctx context.Context, message *rpc.Artifact) error {
var lint style.Lint
if err = proto.Unmarshal(message.Contents, &lint); err != nil {
return err
}
return nil
}); err != nil {
t.Fatalf("Error getting artifact: %s", err)
}
})

t.Run("discovery", func(t *testing.T) {
specName := project.Api("apigeeregistry").Version("v1").Spec("discovery")
lintCmd := Command()
lintCmd.SetArgs([]string{specName.String(), "--linter", "test"})
if err := lintCmd.Execute(); err != nil {
t.Fatalf("Compute lint failed: %s", err)
}
artifactName := specName.Artifact("lint-test")
if err = visitor.GetArtifact(ctx, registryClient, artifactName, true, func(ctx context.Context, message *rpc.Artifact) error {
var lint style.Lint
if err = proto.Unmarshal(message.Contents, &lint); err != nil {
return err
}
return nil
}); err != nil {
t.Fatalf("Error getting artifact: %s", err)
}
})
}
Loading

0 comments on commit a38ff58

Please sign in to comment.