Merge pull request #12 from sjtug/prometheus

Prometheus Integration

config.example.yaml

@@ -1,5 +1,6 @@
 interval: 3 # Interval between pollings
 loglevel: 5 # 1-5
+exporter_address: :8081
 repos:
     - type: rsync
       source: rsync://rsync.chiark.greenend.org.uk/ftp/users/sgtatham/putty-website-mirror/

config/config.go

@@ -19,6 +19,8 @@ type Config struct {
 	LogLevel log.Level
 	// LogStashAddr represents the address of logstash
 	LogStashAddr string
+	// ExporterAddr is the address to expose metrics, :8080 for default
+	ExporterAddr string `yaml:"exporter_address"`
 	// Config for each repo is represented as an array of RepoConfig
 	Repos []RepoConfig
 }
@@ -33,6 +35,9 @@ func (c *Config) Parse(in []byte) (err error) {
 		if c.LogLevel < 0 || c.LogLevel > 5 {
 			return errors.New("loglevel must be 0-5")
 		}
+		if c.ExporterAddr == "" {
+			c.ExporterAddr = ":8080"
+		}
 	}
 	return err
 }

exporter/exporter.go

@@ -0,0 +1,87 @@
+// Package exporter provides definition of exporter
+package exporter
+
+import (
+	log "github.com/Sirupsen/logrus"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/sjtug/lug/helper"
+	"net/http"
+)
+
+// Exporter exports lug metrics to Prometheus
+type Exporter struct {
+	successCounter *prometheus.CounterVec
+	failCounter    *prometheus.CounterVec
+	diskUsage      *prometheus.GaugeVec
+}
+
+var instance *Exporter
+
+// newExporter creates a new exporter
+func newExporter() *Exporter {
+	newExporter := Exporter{
+		successCounter: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "success_sync",
+				Help: "How many successful synchronizations processed, partitioned by workers.",
+			},
+			[]string{"worker"},
+		),
+		failCounter: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "fail_sync",
+				Help: "How many failed synchronizations processed, partitioned by workers.",
+			},
+			[]string{"worker"},
+		),
+		diskUsage: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: "lug",
+				Subsystem: "storage",
+				Name:      "disk_usage",
+				Help:      "Disk usage in bytes, partitioned by workers.",
+			},
+			[]string{"worker"},
+		),
+	}
+	prometheus.MustRegister(newExporter.successCounter)
+	prometheus.MustRegister(newExporter.failCounter)
+	prometheus.MustRegister(newExporter.diskUsage)
+	log.Info("Exporter initialized")
+	return &newExporter
+}
+
+// GetInstance gets the exporter
+func GetInstance() *Exporter {
+	if instance == nil {
+		instance = newExporter()
+	}
+	return instance
+}
+
+// Expose the registered metrics via HTTP.
+func Expose(addr string) {
+	GetInstance() // ensure init
+	http.Handle("/metrics", promhttp.Handler())
+	log.Info("Metrics exposed at " + addr + "/metrics")
+	log.Fatal(http.ListenAndServe(addr, nil))
+}
+
+// SyncSuccess will report a successful synchronization
+func (e *Exporter) SyncSuccess(worker string) {
+	e.successCounter.With(prometheus.Labels{"worker": worker}).Inc()
+}
+
+// SyncFail will report a failed synchronization
+func (e *Exporter) SyncFail(worker string) {
+	e.failCounter.With(prometheus.Labels{"worker": worker}).Inc()
+}
+
+// UpdateDiskUsage will update the disk usage of a directory
+func (e *Exporter) UpdateDiskUsage(worker string, path string) {
+	size, err := helper.DiskUsage(path)
+	if err == nil {
+		e.diskUsage.With(prometheus.Labels{"worker": worker}).Set(float64(size))
+	}
+}

glide.yaml

@@ -31,3 +31,7 @@ import:
   - unix
 - package: gopkg.in/yaml.v2
   version: e4d366fc3c7938e2958e662b4258c7a89e1f0e3e
+- package: github.com/prometheus/client_golang
+  version: ^0.9.0-pre1
+  subpackages:
+  - prometheus/promhttp

helper/disk_usage.go

@@ -0,0 +1,35 @@
+package helper
+
+import (
+	"os"
+	"path"
+)
+
+// DiskUsage counts the disk usage of a directory
+func DiskUsage(curPath string) (int64, error) {
+	var size int64
+
+	dir, err := os.Open(curPath)
+	if err != nil {
+		return size, err
+	}
+	defer dir.Close()
+
+	files, err := dir.Readdir(-1)
+	if err != nil {
+		return size, err
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			s, err := DiskUsage(path.Join(curPath, file.Name()))
+			if err != nil {
+				return size, err
+			}
+			size += s
+		} else {
+			size += file.Size()
+		}
+	}
+	return size, nil
+}

helper/helper_test.go

@@ -53,3 +53,10 @@ func TestMaxLengthSlice(t *testing.T) {
 	assert.True(reflect.DeepEqual([]string{"bar", "2", "3"}, mlss.GetAll()))
 	assert.True(reflect.DeepEqual([]string{"foo", "foobar"}, raw))
 }
+
+func TestDiskUsage(t *testing.T) {
+	assert := assert.New(t)
+	size, err := DiskUsage(".")
+	assert.True(err == nil)
+	assert.True(size > 0)
+}