Skip to content

Commit 8954e18

Browse files
ajnavarroajnavarro
authored
Expose prometheus metrics over http (#815). (#919)
Expose prometheus metrics over http (#815).
2 parents b66dc53 + 4061625 commit 8954e18

File tree

5 files changed

+211
-26
lines changed

5 files changed

+211
-26
lines changed

‎CHANGELOG.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -141,8 +141,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
141141
- Added `json_unquote` function.
142142
- Added `commit_file_stats` function.
143143
- Added documentation about `commit_stats`.
144-
- Add metrics (engine, analyzer, regex, pilosa) based on go-kit interface. ([#744](https://github.com/src-d/go-mysql-server/pull/744))
145144
- `commit_files` is now squashable with `blobs`.
145+
- Add metrics (engine, analyzer, uast, pilosa) based on go-kit interface. ([#744](https://github.com/src-d/go-mysql-server/pull/744)).
146+
- Expose (if enabled) prometheus metrics over http (#815).
146147

147148
### Changed
148149

‎cmd/gitbase/command/server.go

+146-23
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
package command
22

33
import (
4+
"context"
45
"fmt"
56
"net"
7+
"net/http"
68
"net/url"
79
"os"
810
"path/filepath"
@@ -12,6 +14,10 @@ import (
1214
"strings"
1315
"time"
1416

17+
"github.com/go-kit/kit/metrics/prometheus"
18+
promopts "github.com/prometheus/client_golang/prometheus"
19+
"github.com/prometheus/client_golang/prometheus/promhttp"
20+
1521
"github.com/src-d/gitbase"
1622
"github.com/src-d/gitbase/internal/function"
1723
"github.com/src-d/gitbase/internal/rule"
@@ -54,29 +60,31 @@ type Server struct {
5460
plainLibrary *plain.Library
5561
sharedCache cache.Object
5662

57-
Name string `long:"db" default:"gitbase" description:"Database name"`
58-
Version string // Version of the application.
59-
Directories []string `short:"d" long:"directories" description:"Path where standard git repositories are located, multiple directories can be defined."`
60-
Format string `long:"format" default:"git" choice:"git" choice:"siva" description:"Library format"`
61-
Bucket int `long:"bucket" default:"2" description:"Bucketing level to use with siva libraries"`
62-
Bare bool `long:"bare" description:"Sets the library to use bare git repositories, used only with git format libraries"`
63-
NonBare bool `long:"non-bare" description:"Sets the library to use non bare git repositories, used only with git format libraries"`
64-
NonRooted bool `long:"non-rooted" description:"Disables treating siva files as rooted repositories"`
65-
Host string `long:"host" default:"localhost" description:"Host where the server is going to listen"`
66-
Port int `short:"p" long:"port" default:"3306" description:"Port where the server is going to listen"`
67-
User string `short:"u" long:"user" default:"root" description:"User name used for connection"`
68-
Password string `short:"P" long:"password" default:"" description:"Password used for connection"`
69-
UserFile string `short:"U" long:"user-file" env:"GITBASE_USER_FILE" default:"" description:"JSON file with credentials list"`
70-
ConnTimeout int `short:"t" long:"timeout" env:"GITBASE_CONNECTION_TIMEOUT" description:"Timeout in seconds used for connections"`
71-
IndexDir string `short:"i" long:"index" default:"/var/lib/gitbase/index" description:"Directory where the gitbase indexes information will be persisted." env:"GITBASE_INDEX_DIR"`
72-
CacheSize cache.FileSize `long:"cache" default:"512" description:"Object cache size in megabytes" env:"GITBASE_CACHESIZE_MB"`
73-
Parallelism uint `long:"parallelism" description:"Maximum number of parallel threads per table. By default, it's the number of CPU cores. 0 means default, 1 means disabled."`
74-
DisableSquash bool `long:"no-squash" description:"Disables the table squashing."`
75-
TraceEnabled bool `long:"trace" env:"GITBASE_TRACE" description:"Enables jaeger tracing"`
76-
ReadOnly bool `short:"r" long:"readonly" description:"Only allow read queries. This disables creating and deleting indexes as well. Cannot be used with --user-file." env:"GITBASE_READONLY"`
77-
SkipGitErrors bool // SkipGitErrors disables failing when Git errors are found.
78-
Verbose bool `short:"v" description:"Activates the verbose mode"`
79-
LogLevel string `long:"log-level" env:"GITBASE_LOG_LEVEL" choice:"info" choice:"debug" choice:"warning" choice:"error" choice:"fatal" default:"info" description:"logging level"`
63+
Name string `long:"db" default:"gitbase" description:"Database name"`
64+
Version string // Version of the application.
65+
Directories []string `short:"d" long:"directories" description:"Path where standard git repositories are located, multiple directories can be defined."`
66+
Format string `long:"format" default:"git" choice:"git" choice:"siva" description:"Library format"`
67+
Bucket int `long:"bucket" default:"2" description:"Bucketing level to use with siva libraries"`
68+
Bare bool `long:"bare" description:"Sets the library to use bare git repositories, used only with git format libraries"`
69+
NonBare bool `long:"non-bare" description:"Sets the library to use non bare git repositories, used only with git format libraries"`
70+
NonRooted bool `long:"non-rooted" description:"Disables treating siva files as rooted repositories"`
71+
Host string `long:"host" default:"localhost" description:"Host where the server is going to listen"`
72+
Port int `short:"p" long:"port" default:"3306" description:"Port where the server is going to listen"`
73+
User string `short:"u" long:"user" default:"root" description:"User name used for connection"`
74+
Password string `short:"P" long:"password" default:"" description:"Password used for connection"`
75+
UserFile string `short:"U" long:"user-file" env:"GITBASE_USER_FILE" default:"" description:"JSON file with credentials list"`
76+
ConnTimeout int `short:"t" long:"timeout" env:"GITBASE_CONNECTION_TIMEOUT" description:"Timeout in seconds used for connections"`
77+
IndexDir string `short:"i" long:"index" default:"/var/lib/gitbase/index" description:"Directory where the gitbase indexes information will be persisted." env:"GITBASE_INDEX_DIR"`
78+
CacheSize cache.FileSize `long:"cache" default:"512" description:"Object cache size in megabytes" env:"GITBASE_CACHESIZE_MB"`
79+
Parallelism uint `long:"parallelism" description:"Maximum number of parallel threads per table. By default, it's the number of CPU cores. 0 means default, 1 means disabled."`
80+
DisableSquash bool `long:"no-squash" description:"Disables the table squashing."`
81+
TraceEnabled bool `long:"trace" env:"GITBASE_TRACE" description:"Enables jaeger tracing"`
82+
MetricsEnabled bool `long:"metrics" env:"GITBASE_METRICS" description:"Enables prometheus metrics"`
83+
MetricsPort int `long:"metrics-port" env:"GITBASE_METRICS_PORT" default:"2112" description:"Port where the server is going to expose prometheus metrics"`
84+
ReadOnly bool `short:"r" long:"readonly" description:"Only allow read queries. This disables creating and deleting indexes as well. Cannot be used with --user-file." env:"GITBASE_READONLY"`
85+
SkipGitErrors bool // SkipGitErrors disables failing when Git errors are found.
86+
Verbose bool `short:"v" description:"Activates the verbose mode"`
87+
LogLevel string `long:"log-level" env:"GITBASE_LOG_LEVEL" choice:"info" choice:"debug" choice:"warning" choice:"error" choice:"fatal" default:"info" description:"logging level"`
8088
}
8189

8290
type jaegerLogrus struct {
@@ -212,6 +220,19 @@ func (c *Server) Execute(args []string) error {
212220
return err
213221
}
214222

223+
if c.MetricsEnabled {
224+
metricsSrv := enableMetrics(c.Host, c.MetricsPort)
225+
defer func() {
226+
if err := metricsSrv.Shutdown(context.Background()); err != nil {
227+
logrus.Errorln(err)
228+
}
229+
}()
230+
go func() {
231+
logrus.Infof("metrics server started and listening on %s", metricsSrv.Addr)
232+
logrus.Errorln(metricsSrv.ListenAndServe())
233+
}()
234+
}
235+
215236
logrus.Infof("server started and listening on %s:%d", c.Host, c.Port)
216237
return s.Start()
217238
}
@@ -506,3 +527,105 @@ func discoverBare(d directory) (bool, error) {
506527

507528
return bare, nil
508529
}
530+
531+
func enableMetrics(host string, port int) *http.Server {
532+
// Engine metrics
533+
sqle.QueryCounter = prometheus.NewCounterFrom(promopts.CounterOpts{
534+
Namespace: "go_mysql_server",
535+
Subsystem: "engine",
536+
Name: "query_counter",
537+
}, []string{
538+
"query",
539+
})
540+
sqle.QueryErrorCounter = prometheus.NewCounterFrom(promopts.CounterOpts{
541+
Namespace: "go_mysql_server",
542+
Subsystem: "engine",
543+
Name: "query_error_counter",
544+
}, []string{
545+
"query",
546+
"error",
547+
})
548+
sqle.QueryHistogram = prometheus.NewHistogramFrom(promopts.HistogramOpts{
549+
Namespace: "go_mysql_server",
550+
Subsystem: "engine",
551+
Name: "query_histogram",
552+
}, []string{
553+
"query",
554+
"duration",
555+
})
556+
557+
// Analyzer metrics
558+
analyzer.ParallelQueryCounter = prometheus.NewCounterFrom(promopts.CounterOpts{
559+
Namespace: "go_mysql_server",
560+
Subsystem: "analyzer",
561+
Name: "parallel_query_counter",
562+
}, []string{
563+
"parallelism",
564+
})
565+
566+
// Pilosa index driver metrics
567+
pilosa.RowsGauge = prometheus.NewGaugeFrom(promopts.GaugeOpts{
568+
Namespace: "go_mysql_server",
569+
Subsystem: "index",
570+
Name: "indexed_rows_gauge",
571+
}, []string{
572+
"driver",
573+
})
574+
pilosa.TotalHistogram = prometheus.NewHistogramFrom(promopts.HistogramOpts{
575+
Namespace: "go_mysql_server",
576+
Subsystem: "index",
577+
Name: "index_created_total_histogram",
578+
}, []string{
579+
"driver",
580+
"duration",
581+
})
582+
pilosa.MappingHistogram = prometheus.NewHistogramFrom(promopts.HistogramOpts{
583+
Namespace: "go_mysql_server",
584+
Subsystem: "index",
585+
Name: "index_created_mapping_histogram",
586+
}, []string{
587+
"driver",
588+
"duration",
589+
})
590+
pilosa.BitmapHistogram = prometheus.NewHistogramFrom(promopts.HistogramOpts{
591+
Namespace: "go_mysql_server",
592+
Subsystem: "index",
593+
Name: "index_created_bitmap_histogram",
594+
}, []string{
595+
"driver",
596+
"duration",
597+
})
598+
599+
//Uast metrics
600+
function.UastHitCacheCounter = prometheus.NewCounterFrom(promopts.CounterOpts{
601+
Namespace: "gitbase",
602+
Subsystem: "uast",
603+
Name: "hit_cache_counter",
604+
}, []string{
605+
"lang",
606+
"xpath",
607+
})
608+
function.UastMissCacheCounter = prometheus.NewCounterFrom(promopts.CounterOpts{
609+
Namespace: "gitbase",
610+
Subsystem: "uast",
611+
Name: "miss_cache_counter",
612+
}, []string{
613+
"lang",
614+
"xpath",
615+
})
616+
function.UastQueryHistogram = prometheus.NewHistogramFrom(promopts.HistogramOpts{
617+
Namespace: "gitbase",
618+
Subsystem: "uast",
619+
Name: "query_histogram",
620+
}, []string{
621+
"lang",
622+
"xpath",
623+
"duration",
624+
})
625+
626+
// metrics http server
627+
return &http.Server{
628+
Addr: net.JoinHostPort(host, strconv.Itoa(port)),
629+
Handler: promhttp.Handler(),
630+
}
631+
}

‎go.mod

+2
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,14 @@ require (
66
github.com/bblfsh/go-client/v4 v4.1.0
77
github.com/bblfsh/sdk/v3 v3.1.0
88
github.com/gliderlabs/ssh v0.2.0 // indirect
9+
github.com/go-kit/kit v0.8.0
910
github.com/go-sql-driver/mysql v1.4.1
1011
github.com/hashicorp/golang-lru v0.5.1
1112
github.com/hhatto/gocloc v0.3.0
1213
github.com/jessevdk/go-flags v1.4.0
1314
github.com/miekg/dns v1.1.1 // indirect
1415
github.com/opentracing/opentracing-go v1.1.0
16+
github.com/prometheus/client_golang v1.0.0
1517
github.com/sirupsen/logrus v1.3.0
1618
github.com/src-d/enry/v2 v2.0.0
1719
github.com/src-d/go-borges v0.0.0-20190628121335-da12a84d60fd

0 commit comments

Comments
 (0)