feat: Track git provider API usage metrics

Every time we access a Git provider's API we may use some of that
provider's API limits. Medium-sized providers may have many repositories
using PaC but may not have the computer overhead to support permissive
API rate limits. In some cases we have seen pipelines delayed due to API
rate limiting.

By tracking the Git providers' API load by PaC we can gain insight into
potential redundancies, hot spots, and optimizations to the Git provider
API use.

Since the Git provider API clients do not expose these metrics and are
not designed in a way that is easy to decorate, the closest proxy to
tracking actual API calls is tracking accesses to the API client itself.
Since one client-method call equates to one API call, this metric should
track the API usage well. However if a developer saves a reference to the
API client then they will be able to make API calls without the metric
incrementing; we will have to watch out for this if we want the metric
to be accurate.

Author: aThorp96

docs/content/docs/install/metrics.md

@@ -12,6 +12,7 @@ You can configure these exporters by referring to the [observability configurati
 
 | Name                                                 | Type    | Description                                                        |
 |------------------------------------------------------|---------|--------------------------------------------------------------------|
+| `pipelines_as_code_git_provider_api_request_count`   | Counter | Number of API requests submitted to git providers                  |
 | `pipelines_as_code_pipelinerun_count`                | Counter | Number of pipelineruns created by pipelines-as-code                |
 | `pipelines_as_code_pipelinerun_duration_seconds_sum` | Counter | Number of seconds all pipelineruns have taken in pipelines-as-code |
 | `pipelines_as_code_running_pipelineruns_count`       | Gauge   | Number of running pipelineruns in pipelines-as-code                |

pkg/metrics/metrics.go

@@ -30,6 +30,12 @@ var runningPRCount = stats.Float64("pipelines_as_code_running_pipelineruns_count
 	"number of running pipelineruns by pipelines as code",
 	stats.UnitDimensionless)
 
+var gitProviderAPIRequestCount = stats.Int64(
+	"pipelines_as_code_git_provider_api_request_count",
+	"number of API requests from pipelines as code to git providers",
+	stats.UnitDimensionless,
+)
+
 // Recorder holds keys for metrics.
 type Recorder struct {
 	initialized     bool
@@ -61,36 +67,42 @@ func NewRecorder() (*Recorder, error) {
 
 		provider, errRegistering := tag.NewKey("provider")
 		if errRegistering != nil {
+			ErrRegistering = errRegistering
 			return
 		}
 		R.provider = provider
 
 		eventType, errRegistering := tag.NewKey("event-type")
 		if errRegistering != nil {
+			ErrRegistering = errRegistering
 			return
 		}
 		R.eventType = eventType
 
 		namespace, errRegistering := tag.NewKey("namespace")
 		if errRegistering != nil {
+			ErrRegistering = errRegistering
 			return
 		}
 		R.namespace = namespace
 
 		repository, errRegistering := tag.NewKey("repository")
 		if errRegistering != nil {
+			ErrRegistering = errRegistering
 			return
 		}
 		R.repository = repository
 
 		status, errRegistering := tag.NewKey("status")
 		if errRegistering != nil {
+			ErrRegistering = errRegistering
 			return
 		}
 		R.status = status
 
 		reason, errRegistering := tag.NewKey("reason")
 		if errRegistering != nil {
+			ErrRegistering = errRegistering
 			return
 		}
 		R.reason = reason
@@ -116,11 +128,18 @@ func NewRecorder() (*Recorder, error) {
 				Aggregation: view.LastValue(),
 				TagKeys:     []tag.Key{R.namespace, R.repository},
 			}
+			gitProviderAPIRequestView = &view.View{
+				Description: gitProviderAPIRequestCount.Description(),
+				Measure:     gitProviderAPIRequestCount,
+				Aggregation: view.Count(),
+				TagKeys:     []tag.Key{R.provider, R.eventType, R.namespace, R.repository},
+			}
 		)
 
-		view.Unregister(prCountView, prDurationView, runningPRView)
-		errRegistering = view.Register(prCountView, prDurationView, runningPRView)
+		view.Unregister(prCountView, prDurationView, runningPRView, gitProviderAPIRequestView)
+		errRegistering = view.Register(prCountView, prDurationView, runningPRView, gitProviderAPIRequestView)
 		if errRegistering != nil {
+			ErrRegistering = errRegistering
 			R.initialized = false
 			return
 		}
@@ -129,12 +148,19 @@ func NewRecorder() (*Recorder, error) {
 	return R, ErrRegistering
 }
 
-// Count logs number of times a pipelinerun is ran for a provider.
-func (r *Recorder) Count(provider, event, namespace, repository string) error {
+func (r Recorder) assertInitialized() error {
 	if !r.initialized {
 		return fmt.Errorf(
 			"ignoring the metrics recording for pipelineruns, failed to initialize the metrics recorder")
 	}
+	return nil
+}
+
+// Count logs number of times a pipelinerun is ran for a provider.
+func (r *Recorder) Count(provider, event, namespace, repository string) error {
+	if err := r.assertInitialized(); err != nil {
+		return err
+	}
 
 	ctx, err := tag.New(
 		context.Background(),
@@ -153,9 +179,8 @@ func (r *Recorder) Count(provider, event, namespace, repository string) error {
 
 // CountPRDuration collects duration taken by a pipelinerun in seconds accumulate them in prDurationCount.
 func (r *Recorder) CountPRDuration(namespace, repository, status, reason string, duration time.Duration) error {
-	if !r.initialized {
-		return fmt.Errorf(
-			"ignoring the metrics recording for pipelineruns, failed to initialize the metrics recorder")
+	if err := r.assertInitialized(); err != nil {
+		return err
 	}
 
 	ctx, err := tag.New(
@@ -175,9 +200,8 @@ func (r *Recorder) CountPRDuration(namespace, repository, status, reason string,
 
 // RunningPipelineRuns emits the number of running PipelineRuns for a repository and namespace.
 func (r *Recorder) RunningPipelineRuns(namespace, repository string, runningPRs float64) error {
-	if !r.initialized {
-		return fmt.Errorf(
-			"ignoring the metrics recording for pipelineruns, failed to initialize the metrics recorder")
+	if err := r.assertInitialized(); err != nil {
+		return err
 	}
 
 	ctx, err := tag.New(
@@ -266,6 +290,26 @@ func (r *Recorder) ReportRunningPipelineRuns(ctx context.Context, lister listers
 	}
 }
 
+func (r *Recorder) ReportGitProviderAPIUsage(provider, event, namespace, repository string) error {
+	if err := r.assertInitialized(); err != nil {
+		return err
+	}
+
+	ctx, err := tag.New(
+		context.Background(),
+		tag.Insert(r.provider, provider),
+		tag.Insert(r.eventType, event),
+		tag.Insert(r.namespace, namespace),
+		tag.Insert(r.repository, repository),
+	)
+	if err != nil {
+		return err
+	}
+
+	metrics.Record(ctx, gitProviderAPIRequestCount.M(1))
+	return nil
+}
+
 func ResetRecorder() {
 	Once = sync.Once{}
 	R = nil

pkg/provider/bitbucketcloud/bitbucket.go

@@ -11,6 +11,7 @@ import (
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/apis/pipelinesascode/v1alpha1"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/changedfiles"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/events"
+	"github.com/openshift-pipelines/pipelines-as-code/pkg/metrics"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/params"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/params/info"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/params/triggertype"
@@ -24,17 +25,43 @@ var _ provider.Interface = (*Provider)(nil)
 type Provider struct {
 	bbClient      *bitbucket.Client
 	Logger        *zap.SugaredLogger
+	metrics       *metrics.Recorder
 	run           *params.Run
 	pacInfo       *info.PacOpts
 	Token, APIURL *string
 	Username      *string
 	provenance    string
+	repo          *v1alpha1.Repository
+	triggerEvent  string
 }
 
 func (v Provider) Client() *bitbucket.Client {
+	v.recordAPIUsageMetrics()
 	return v.bbClient
 }
 
+func (v *Provider) recordAPIUsageMetrics() {
+	if v.metrics == nil {
+		m, err := metrics.NewRecorder()
+		if err != nil {
+			v.Logger.Errorf("Error initializing bitbucketcloud metrics recorder: %v", err)
+			return
+		}
+		v.metrics = m
+	}
+
+	name := ""
+	namespace := ""
+	if v.repo != nil {
+		name = v.repo.Name
+		namespace = v.repo.Namespace
+	}
+
+	if err := v.metrics.ReportGitProviderAPIUsage("bitbucketcloud", v.triggerEvent, namespace, name); err != nil {
+		v.Logger.Errorf("Error reporting git API usage metrics: %v", err)
+	}
+}
+
 // CheckPolicyAllowing TODO: Implement ME.
 func (v *Provider) CheckPolicyAllowing(_ context.Context, _ *info.Event, _ []string) (bool, string) {
 	return false, ""
@@ -180,7 +207,7 @@ func (v *Provider) GetFileInsideRepo(_ context.Context, event *info.Event, path,
 	return v.getBlob(event, revision, path)
 }
 
-func (v *Provider) SetClient(_ context.Context, run *params.Run, event *info.Event, _ *v1alpha1.Repository, _ *events.EventEmitter) error {
+func (v *Provider) SetClient(_ context.Context, run *params.Run, event *info.Event, repo *v1alpha1.Repository, _ *events.EventEmitter) error {
 	if event.Provider.Token == "" {
 		return fmt.Errorf("no git_provider.secret has been set in the repo crd")
 	}
@@ -191,6 +218,8 @@ func (v *Provider) SetClient(_ context.Context, run *params.Run, event *info.Eve
 	v.Token = &event.Provider.Token
 	v.Username = &event.Provider.User
 	v.run = run
+	v.repo = repo
+	v.triggerEvent = event.EventType
 	return nil
 }
 

pkg/provider/bitbucketdatacenter/bitbucketdatacenter.go

@@ -16,6 +16,7 @@ import (
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/apis/pipelinesascode/v1alpha1"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/changedfiles"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/events"
+	"github.com/openshift-pipelines/pipelines-as-code/pkg/metrics"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/params"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/params/info"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/params/triggertype"
@@ -34,6 +35,7 @@ type Provider struct {
 	bbClient                  *bbv1.APIClient // temporarily keeping it after the refactor finishes, will be removed
 	scmClient                 *scm.Client
 	Logger                    *zap.SugaredLogger
+	metrics                   *metrics.Recorder
 	run                       *params.Run
 	pacInfo                   *info.PacOpts
 	baseURL                   string
@@ -42,9 +44,12 @@ type Provider struct {
 	apiURL                    string
 	provenance                string
 	projectKey                string
+	repo                      *v1alpha1.Repository
+	triggerEvent              string
 }
 
 func (v Provider) Client() *bbv1.APIClient {
+	v.recordAPIUsageMetrics()
 	return v.bbClient
 }
 
@@ -53,13 +58,36 @@ func (v *Provider) SetBitBucketClient(client *bbv1.APIClient) {
 }
 
 func (v Provider) ScmClient() *scm.Client {
+	v.recordAPIUsageMetrics()
 	return v.scmClient
 }
 
 func (v *Provider) SetScmClient(client *scm.Client) {
 	v.scmClient = client
 }
 
+func (v *Provider) recordAPIUsageMetrics() {
+	if v.metrics == nil {
+		m, err := metrics.NewRecorder()
+		if err != nil {
+			v.Logger.Errorf("Error initializing bitbucketserver metrics recorder: %v", err)
+			return
+		}
+		v.metrics = m
+	}
+
+	name := ""
+	namespace := ""
+	if v.repo != nil {
+		name = v.repo.Name
+		namespace = v.repo.Namespace
+	}
+
+	if err := v.metrics.ReportGitProviderAPIUsage("bitbucketserver", v.triggerEvent, namespace, name); err != nil {
+		v.Logger.Errorf("Error reporting git API usage metrics: %v", err)
+	}
+}
+
 func (v *Provider) SetPacInfo(pacInfo *info.PacOpts) {
 	v.pacInfo = pacInfo
 }
@@ -273,7 +301,7 @@ func removeLastSegment(urlStr string) string {
 	return u.String()
 }
 
-func (v *Provider) SetClient(ctx context.Context, run *params.Run, event *info.Event, _ *v1alpha1.Repository, _ *events.EventEmitter) error {
+func (v *Provider) SetClient(ctx context.Context, run *params.Run, event *info.Event, repo *v1alpha1.Repository, _ *events.EventEmitter) error {
 	if event.Provider.User == "" {
 		return fmt.Errorf("no spec.git_provider.user has been set in the repo crd")
 	}
@@ -318,6 +346,8 @@ func (v *Provider) SetClient(ctx context.Context, run *params.Run, event *info.E
 		v.scmClient = client
 	}
 	v.run = run
+	v.repo = repo
+	v.triggerEvent = event.EventType
 	_, resp, err := v.ScmClient().Users.FindLogin(ctx, event.Provider.User)
 	if resp != nil && resp.Status == http.StatusUnauthorized {
 		return fmt.Errorf("cannot get user %s with token: %w", event.Provider.User, err)

pkg/provider/gitea/gitea.go

@@ -14,6 +14,7 @@ import (
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/apis/pipelinesascode/v1alpha1"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/changedfiles"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/events"
+	"github.com/openshift-pipelines/pipelines-as-code/pkg/metrics"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/opscomments"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/params"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/params/info"
@@ -45,6 +46,7 @@ var _ provider.Interface = (*Provider)(nil)
 type Provider struct {
 	giteaClient      *gitea.Client
 	Logger           *zap.SugaredLogger
+	metrics          *metrics.Recorder
 	pacInfo          *info.PacOpts
 	Token            *string
 	giteaInstanceURL string
@@ -53,16 +55,40 @@ type Provider struct {
 	repo         *v1alpha1.Repository
 	eventEmitter *events.EventEmitter
 	run          *params.Run
+	triggerEvent string
 }
 
 func (v Provider) Client() *gitea.Client {
+	v.recordAPIUsageMetrics()
 	return v.giteaClient
 }
 
 func (v *Provider) SetGiteaClient(client *gitea.Client) {
 	v.giteaClient = client
 }
 
+func (v *Provider) recordAPIUsageMetrics() {
+	if v.metrics == nil {
+		m, err := metrics.NewRecorder()
+		if err != nil {
+			v.Logger.Errorf("Error initializing gitea metrics recorder: %v", err)
+			return
+		}
+		v.metrics = m
+	}
+
+	name := ""
+	namespace := ""
+	if v.repo != nil {
+		name = v.repo.Name
+		namespace = v.repo.Namespace
+	}
+
+	if err := v.metrics.ReportGitProviderAPIUsage(v.giteaInstanceURL, v.triggerEvent, namespace, name); err != nil {
+		v.Logger.Errorf("Error reporting git API usage metrics: %v", err)
+	}
+}
+
 func (v *Provider) SetPacInfo(pacInfo *info.PacOpts) {
 	v.pacInfo = pacInfo
 }
@@ -118,6 +144,7 @@ func (v *Provider) SetClient(_ context.Context, run *params.Run, runevent *info.
 	v.eventEmitter = emitter
 	v.repo = repo
 	v.run = run
+	v.triggerEvent = runevent.EventType
 	return nil
 }