add(count) added counters

pkg/count/counters.go

@@ -0,0 +1,108 @@
+// Copyright 2019 ScyllaDB
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package count
+
+import (
+	"fmt"
+	"sync/atomic"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type SimpleCounters []*SimpleCounter
+
+func (l SimpleCounters) Add(idx, in int) {
+	l[idx].Add(in)
+}
+
+func (l SimpleCounters) Inc(idx int) {
+	l[idx].Inc()
+}
+
+func (l SimpleCounters) Get(idx int) uint64 {
+	return l[idx].Get()
+}
+
+func (l SimpleCounters) GetCounter(idx int) *SimpleCounter {
+	return l[idx]
+}
+
+func (l SimpleCounters) printFull() printRows {
+	out := make(printRows, 0, len(l))
+	for idx := range l {
+		if l.Get(idx) == 0 {
+			continue
+		}
+		out = append(out, l[idx].printFull())
+	}
+	out.alignRows()
+	return out
+}
+
+type SimpleCounter struct {
+	group        *Group
+	name         string
+	prometheus   prometheus.Counter
+	unit         string
+	description  string
+	val          atomic.Uint64
+	inPrometheus bool
+	_            noCopy
+}
+
+func (c *SimpleCounter) Add(in int) {
+	if in > 0 {
+		c.val.Add(uint64(in))
+		if c.inPrometheus {
+			c.prometheus.Add(float64(in))
+		}
+	}
+	if in < 0 {
+		panic("add value should be >0")
+	}
+}
+
+func (c *SimpleCounter) Inc() {
+	c.val.Add(1)
+	if c.inPrometheus {
+		c.prometheus.Inc()
+	}
+}
+
+func (c *SimpleCounter) Get() uint64 {
+	return c.val.Load()
+}
+
+func (c *SimpleCounter) printFull() printRow {
+	prometh := "prometheus:no "
+	if c.inPrometheus {
+		prometh = "prometheus:yes"
+	}
+	return printRow{
+		"",
+		simpleCounterName,
+		c.name + ":",
+		fmt.Sprintf("%d", c.val.Load()),
+		separator + c.unit,
+		separator + prometh,
+		separator + "description:" + c.description,
+	}
+}
+
+type noCopy struct{}
+
+func (*noCopy) Lock() {}
+
+func (*noCopy) Unlock() {}

pkg/count/counters_test.go

@@ -0,0 +1,143 @@
+// Copyright 2019 ScyllaDB
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package count_test
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"golang.org/x/exp/rand"
+
+	"github.com/scylladb/gemini/pkg/count"
+)
+
+func TestSimpleCounters(t *testing.T) {
+	t.Parallel()
+	countersInfo := []count.Info{{
+		Name:                  "test simple counter 1",
+		Unit:                  "ms",
+		PrometheusIntegration: false,
+		Description:           "test counts",
+	}, {
+		Name:                  "test simple counter 123123",
+		Unit:                  "ssm",
+		PrometheusIntegration: true,
+		Description:           "test counts",
+	}, {
+		Name:                  "test simple counter 5656565",
+		Unit:                  "s",
+		PrometheusIntegration: true,
+		Description:           "test counts",
+	}}
+	group := count.InitGroup("test group", "testing", true)
+	sCounters := group.AddSimpleCounters(countersInfo)
+	workers := 10
+	adds := 100000
+	var wg sync.WaitGroup
+	for i := 0; i < workers; i++ {
+		wg.Add(1)
+		go func() {
+			scOperations(sCounters, adds)
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+	sum := getSimpleCounterSum(sCounters)
+	if sum != workers*adds {
+		t.Errorf("wrong simple counters work. expected sum:%d, received sum:%d", workers*adds, sum)
+	}
+	count.PrintAllGroups()
+}
+
+func TestTotalCounters(t *testing.T) {
+	t.Parallel()
+	countersNames := []string{
+		"sub counter1",
+		"sub counter200",
+		"sub counter3",
+		"sub counter4",
+		"sub counter5000",
+		"sub counter6",
+		"sub counter7",
+		"sub counter800000",
+		"sub counter9",
+		"sub counter10",
+	}
+	group := count.InitGroup("test group", "testing", true)
+	group2 := group.AddGroup("test group222", "testing222", true)
+	tCounter1 := group.AddTotalCounter(count.Info{Name: "total counter 1", Unit: "qty", PrometheusIntegration: false, Description: "count qty"}, "count", countersNames)
+	tCounter2 := group2.AddTotalCounter(count.Info{Name: "total counter 2", Unit: "ps", PrometheusIntegration: true, Description: "count ps"}, "count", countersNames)
+	tCounters := count.TotalCounters{tCounter1, tCounter2}
+	workers := 10
+	adds := 100000
+	var wg sync.WaitGroup
+	for i := 0; i < workers; i++ {
+		wg.Add(1)
+		go func() {
+			totalCounterOperations(tCounters, adds)
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+	tSum, sum := getTotalCounterSum(tCounters)
+	if sum != workers*adds {
+		t.Errorf("wrong simple counters work. expected sum:%d, received sum:%d", workers*adds, sum)
+	}
+	if tSum != workers*adds {
+		t.Errorf("wrong simple counters work. expected sum:%d, received sum:%d", workers*adds, sum)
+	}
+	count.PrintAllGroups()
+}
+
+func scOperations(counters count.SimpleCounters, adds int) {
+	cl := len(counters)
+	rnd := rand.New(rand.NewSource(uint64(time.Now().Unix())))
+	for c := 0; c < adds; c++ {
+		counters[rnd.Intn(cl)].Inc()
+		counters[rnd.Intn(cl)].Get()
+	}
+}
+
+func getSimpleCounterSum(counters count.SimpleCounters) int {
+	sum := 0
+	for idx := range counters {
+		sum += int(counters[idx].Get())
+	}
+	return sum
+}
+
+func totalCounterOperations(counters count.TotalCounters, adds int) {
+	cl := len(counters)
+	rnd := rand.New(rand.NewSource(uint64(time.Now().Unix())))
+	for c := 0; c < adds; c++ {
+		n := rnd.Intn(10)
+		counters[rnd.Intn(cl)].Inc(n)
+		counters[rnd.Intn(cl)].Get(n)
+	}
+}
+
+func getTotalCounterSum(counters count.TotalCounters) (int, int) {
+	sumTotal := 0
+	sum := 0
+	for idx := range counters {
+		sumTotal += int(counters[idx].GetTotal())
+		subCounters := counters[idx].GetSubCounters()
+		for _, sub := range subCounters {
+			sum += int(sub.Get())
+		}
+	}
+	return sumTotal, sum
+}

pkg/count/groups.go

@@ -0,0 +1,136 @@
+// Copyright 2019 ScyllaDB
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package count
+
+import (
+	"sync"
+	"sync/atomic"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	simpleCounterName = "sc."
+	totalCounterName  = "tc."
+	subCounterName    = "uc."
+	groupName         = "gr."
+)
+
+var (
+	allGroups     = make(Groups, 0)
+	allGroupsMute sync.RWMutex
+)
+
+var StmtsCounters = InitGroup("generated stmt`s", "count of all generated stmt`s", true)
+
+type Group struct {
+	parentGroup    *Group
+	description    string
+	name           string
+	groups         Groups
+	simpleCounters SimpleCounters
+	totalCounters  TotalCounters
+	active         bool
+	mut            sync.RWMutex
+}
+
+type Groups []*Group
+
+type Info struct {
+	Name                  string
+	Unit                  string
+	Description           string
+	PrometheusIntegration bool
+}
+
+func InitGroup(name, description string, active bool) *Group {
+	group := Group{
+		parentGroup: nil,
+		name:        name,
+		description: description,
+		active:      active,
+	}
+	allGroupsMute.Lock()
+	allGroups = append(allGroups, &group)
+	allGroupsMute.Unlock()
+	return &group
+}
+
+func (g *Group) AddGroup(name, description string, active bool) *Group {
+	group := Group{
+		parentGroup: g,
+		name:        name,
+		description: description,
+		active:      active,
+	}
+	if !g.active {
+		group.active = false
+	}
+
+	g.mut.Lock()
+	defer g.mut.Unlock()
+	g.groups = append(g.groups, &group)
+
+	return &group
+}
+
+func (g *Group) AddSimpleCounters(counters []Info) SimpleCounters {
+	sCounters := make(SimpleCounters, len(counters))
+	for idx := range sCounters {
+		sCounters[idx] = g.initCounter(counters[idx])
+	}
+
+	g.mut.Lock()
+	defer g.mut.Unlock()
+	g.simpleCounters = sCounters
+	return sCounters
+}
+
+func (g *Group) AddSimpleCounter(counter Info) *SimpleCounter {
+	g.mut.Lock()
+	defer g.mut.Unlock()
+
+	sCounter := g.initCounter(counter)
+	g.simpleCounters = append(g.simpleCounters, sCounter)
+	return sCounter
+}
+
+func (g *Group) initCounter(counter Info) *SimpleCounter {
+	newCounter := &SimpleCounter{
+		name:         counter.Name,
+		unit:         counter.Unit,
+		inPrometheus: counter.PrometheusIntegration,
+		description:  counter.Description,
+		group:        g,
+		val:          atomic.Uint64{},
+	}
+	if counter.PrometheusIntegration {
+		newCounter.prometheus = prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: g.getParentGroupName(),
+			Subsystem: g.name,
+			Name:      counter.Name,
+			Help:      counter.Description,
+		})
+	}
+	return newCounter
+}
+
+func (g *Group) getParentGroupName() string {
+	pgName := ""
+	if g.parentGroup != nil {
+		pgName = g.parentGroup.name
+	}
+	return pgName
+}