Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Prefactor: move BlockLog and LogEntry into api package #70

Merged
merged 1 commit into from
Apr 22, 2024
Merged

Prefactor: move BlockLog and LogEntry into api package #70

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion app/pkg/analyze/types.go → app/api/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package analyze
package api

import (
"encoding/json"
Expand Down
2 changes: 1 addition & 1 deletion app/pkg/analyze/types_test.go → app/api/types_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package analyze
package api

import (
"encoding/json"
Expand Down
42 changes: 22 additions & 20 deletions app/pkg/analyze/analyzer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import (
"strings"
"time"

"github.com/jlewi/foyle/app/api"

"github.com/jlewi/foyle/app/pkg/docs"
"github.com/jlewi/foyle/app/pkg/logs"
"github.com/jlewi/foyle/protos/go/foyle/v1alpha1"
Expand Down Expand Up @@ -71,10 +73,10 @@ func (a *Analyzer) Analyze(ctx context.Context, logsDir string, outDir string) (
}

// buildTraces creates a map of all the traces and initializes the blocks.
func buildTraces(ctx context.Context, jsonFiles []string, resultFiles ResultFiles) (map[string]Trace, map[string]*BlockLog, error) {
func buildTraces(ctx context.Context, jsonFiles []string, resultFiles ResultFiles) (map[string]api.Trace, map[string]*api.BlockLog, error) {
log := logs.FromContext(ctx)
// Entries is a mapping from a traceId to a list of logEntries associated with that entry.
traceEntries := make(map[string][]*LogEntry)
traceEntries := make(map[string][]*api.LogEntry)

for _, p := range jsonFiles {
log.Info("Reading file", "path", p)
Expand All @@ -86,7 +88,7 @@ func buildTraces(ctx context.Context, jsonFiles []string, resultFiles ResultFile
d := json.NewDecoder(f)

for {
entry := &LogEntry{}
entry := &api.LogEntry{}
err := d.Decode(entry)

if err != nil {
Expand All @@ -104,18 +106,18 @@ func buildTraces(ctx context.Context, jsonFiles []string, resultFiles ResultFile

items, ok := traceEntries[entry.TraceID()]
if !ok {
items = make([]*LogEntry, 0, 10)
items = make([]*api.LogEntry, 0, 10)
}
items = append(items, entry)
traceEntries[entry.TraceID()] = items
}
}

// Store a map of all traces
traces := make(map[string]Trace)
traces := make(map[string]api.Trace)

// Build a map of the blocks
blocks := make(map[string]*BlockLog)
blocks := make(map[string]*api.BlockLog)

// Create encoders to write the traces
genFile, err := os.Create(resultFiles.GenerateTraces[0])
Expand Down Expand Up @@ -147,30 +149,30 @@ func buildTraces(ctx context.Context, jsonFiles []string, resultFiles ResultFile

// Update the blocks associated with this trace
switch t := trace.(type) {
case *GenerateTrace:
case *api.GenerateTrace:
for _, oBlock := range t.Response.GetBlocks() {
bid := oBlock.GetId()
if bid == "" {
continue
}
block, ok := blocks[bid]
if !ok {
block = &BlockLog{
block = &api.BlockLog{
ID: bid,
}
blocks[bid] = block
}
block.GenTraceID = tid
}
enc = genEnc
case *ExecuteTrace:
case *api.ExecuteTrace:
bid := t.Request.GetBlock().GetId()
if bid == "" {
continue
}
block, ok := blocks[bid]
if !ok {
block = &BlockLog{
block = &api.BlockLog{
ID: bid,
}
blocks[bid] = block
Expand Down Expand Up @@ -243,7 +245,7 @@ func initResultFiles(outDir string) ResultFiles {
}
}

func buildBlockLogs(ctx context.Context, traces map[string]Trace, blocks map[string]*BlockLog, outFile string) error {
func buildBlockLogs(ctx context.Context, traces map[string]api.Trace, blocks map[string]*api.BlockLog, outFile string) error {
log := logs.FromContext(ctx)

oDir := filepath.Dir(outFile)
Expand Down Expand Up @@ -277,7 +279,7 @@ func buildBlockLogs(ctx context.Context, traces map[string]Trace, blocks map[str
return nil
}

func buildBlockLog(ctx context.Context, block *BlockLog, traces map[string]Trace) error {
func buildBlockLog(ctx context.Context, block *api.BlockLog, traces map[string]api.Trace) error {
log := logs.FromContext(ctx)
log = log.WithValues("blockId", block.ID)
log.Info("Building block log", "block", block)
Expand All @@ -287,7 +289,7 @@ func buildBlockLog(ctx context.Context, block *BlockLog, traces map[string]Trace
}

if block.GenTraceID != "" {
genTrace, ok := traces[block.GenTraceID].(*GenerateTrace)
genTrace, ok := traces[block.GenTraceID].(*api.GenerateTrace)
if !ok {
log.Error(errors.New("Missing GenerateTrace for traceId"), "Error getting generate trace", "genTraceId", block.GenTraceID)
} else {
Expand All @@ -306,10 +308,10 @@ func buildBlockLog(ctx context.Context, block *BlockLog, traces map[string]Trace
}
}

var lastTrace *ExecuteTrace
var lastTrace *api.ExecuteTrace
// Get the last execution trace
for _, tid := range block.ExecTraceIDs {
trace, ok := traces[tid].(*ExecuteTrace)
trace, ok := traces[tid].(*api.ExecuteTrace)
if !ok {
log.Error(errors.New("Missing ExecuteTrace for traceId"), "Error getting execute trace", "execTraceId", tid)
continue
Expand Down Expand Up @@ -339,7 +341,7 @@ func buildBlockLog(ctx context.Context, block *BlockLog, traces map[string]Trace
return nil
}

func combineEntriesForTrace(ctx context.Context, entries []*LogEntry) (Trace, error) {
func combineEntriesForTrace(ctx context.Context, entries []*api.LogEntry) (api.Trace, error) {
// First sort the entries by timestamp.
sort.Slice(entries, func(i, j int) bool {
return entries[i].Time().Before(entries[j].Time())
Expand All @@ -360,8 +362,8 @@ func combineEntriesForTrace(ctx context.Context, entries []*LogEntry) (Trace, er
return nil, errors.New("Failed to identify trace type")
}

func combineGenerateTrace(ctx context.Context, entries []*LogEntry) (*GenerateTrace, error) {
trace := &GenerateTrace{}
func combineGenerateTrace(ctx context.Context, entries []*api.LogEntry) (*api.GenerateTrace, error) {
trace := &api.GenerateTrace{}
for _, e := range entries {
if trace.TraceID == "" {
trace.TraceID = e.TraceID()
Expand Down Expand Up @@ -394,8 +396,8 @@ func combineGenerateTrace(ctx context.Context, entries []*LogEntry) (*GenerateTr
return trace, nil
}

func combineExecuteTrace(ctx context.Context, entries []*LogEntry) (*ExecuteTrace, error) {
trace := &ExecuteTrace{}
func combineExecuteTrace(ctx context.Context, entries []*api.LogEntry) (*api.ExecuteTrace, error) {
trace := &api.ExecuteTrace{}
for _, e := range entries {
if trace.TraceID == "" {
trace.TraceID = e.TraceID()
Expand Down
40 changes: 21 additions & 19 deletions app/pkg/analyze/analyzer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import (
"testing"
"time"

"github.com/jlewi/foyle/app/api"

"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/jlewi/foyle/app/pkg/testutil"
Expand All @@ -33,15 +35,15 @@ func shuffle(in []string) []string {
func Test_BuildBlockLog(t *testing.T) {
type testCase struct {
name string
block *BlockLog
traces map[string]Trace
expected *BlockLog
block *api.BlockLog
traces map[string]api.Trace
expected *api.BlockLog
}

traces := make(map[string]Trace)
traces := make(map[string]api.Trace)

const bid1 = "g123output1"
genTrace := &GenerateTrace{
genTrace := &api.GenerateTrace{
TraceID: "g123",
StartTime: timeMustParse(time.RFC3339, "2021-01-01T00:00:00Z"),
EndTime: timeMustParse(time.RFC3339, "2021-01-01T00:01:00Z"),
Expand All @@ -64,7 +66,7 @@ func Test_BuildBlockLog(t *testing.T) {
},
}

execTrace1 := &ExecuteTrace{
execTrace1 := &api.ExecuteTrace{
TraceID: "e456",
StartTime: timeMustParse(time.RFC3339, "2021-01-02T00:00:00Z"),
EndTime: timeMustParse(time.RFC3339, "2021-01-02T00:01:00Z"),
Expand All @@ -87,7 +89,7 @@ func Test_BuildBlockLog(t *testing.T) {
},
}

execTrace2 := &ExecuteTrace{
execTrace2 := &api.ExecuteTrace{
TraceID: "e789",
StartTime: timeMustParse(time.RFC3339, "2021-01-03T00:00:00Z"),
EndTime: timeMustParse(time.RFC3339, "2021-01-03T00:01:00Z"),
Expand Down Expand Up @@ -119,13 +121,13 @@ func Test_BuildBlockLog(t *testing.T) {
cases := []testCase{
{
name: "basic",
block: &BlockLog{
block: &api.BlockLog{
ID: bid1,
GenTraceID: genTrace.TraceID,

ExecTraceIDs: execTraceIds,
},
expected: &BlockLog{
expected: &api.BlockLog{
ID: bid1,
GenTraceID: genTrace.TraceID,
ExecTraceIDs: execTraceIds,
Expand Down Expand Up @@ -189,9 +191,9 @@ func Test_Analyzer(t *testing.T) {
}
d := json.NewDecoder(f)

actual := map[string]*BlockLog{}
actual := map[string]*api.BlockLog{}
for {
var b BlockLog
var b api.BlockLog
if err := d.Decode(&b); err != nil {
if err == io.EOF {
break
Expand Down Expand Up @@ -249,10 +251,10 @@ func checkGenTracesFiles(t *testing.T, path string) {
t.Errorf("Failed to open output file: %v", err)
return
}
traces := make([]*GenerateTrace, 0, 10)
traces := make([]*api.GenerateTrace, 0, 10)
d := json.NewDecoder(genFile)
for {
trace := &GenerateTrace{}
trace := &api.GenerateTrace{}
if err := d.Decode(trace); err != nil {
if err == io.EOF {
break
Expand All @@ -274,10 +276,10 @@ func checkExecuteTracesFiles(t *testing.T, path string) {
t.Errorf("Failed to open output file: %v", err)
return
}
traces := make([]*ExecuteTrace, 0, 10)
traces := make([]*api.ExecuteTrace, 0, 10)
d := json.NewDecoder(genFile)
for {
trace := &ExecuteTrace{}
trace := &api.ExecuteTrace{}
if err := d.Decode(trace); err != nil {
if err == io.EOF {
break
Expand Down Expand Up @@ -311,14 +313,14 @@ func Test_CombineGenerateEntries(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
entries := make([]*LogEntry, 0, 10)
entries := make([]*api.LogEntry, 0, 10)
testFile, err := os.Open(filepath.Join(cwd, "test_data", c.linesFile))
if err != nil {
t.Fatalf("Failed to open test file: %v", err)
}
d := json.NewDecoder(testFile)
for {
e := &LogEntry{}
e := &api.LogEntry{}
err := d.Decode(e)
if err != nil {
if err == io.EOF {
Expand Down Expand Up @@ -366,14 +368,14 @@ func Test_CombineExecuteEntries(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
entries := make([]*LogEntry, 0, 10)
entries := make([]*api.LogEntry, 0, 10)
testFile, err := os.Open(filepath.Join(cwd, "test_data", c.linesFile))
if err != nil {
t.Fatalf("Failed to open test file: %v", err)
}
d := json.NewDecoder(testFile)
for {
e := &LogEntry{}
e := &api.LogEntry{}
err := d.Decode(e)
if err != nil {
if err == io.EOF {
Expand Down
Loading