Store webhook event in database (go-gitea#29145)

Refactor the webhook logic, to have the type-dependent processing happen
only in one place.

---

## Current webhook flow

1. An event happens
2. It is pre-processed (depending on the webhook type) and its body is
added to a task queue
3. When the task is processed, some more logic (depending on the webhook
type as well) is applied to make an HTTP request

This means that webhook-type dependant logic is needed in step 2 and 3.
This is cumbersome and brittle to maintain.

Updated webhook flow with this PR:
1. An event happens
2. It is stored as-is and added to a task queue
3. When the task is processed, the event is processed (depending on the
webhook type) to make an HTTP request

So the only webhook-type dependent logic happens in one place (step 3)
which should be much more robust.

## Consequences of the refactor

- the raw event must be stored in the hooktask (until now, the
pre-processed body was stored)
- to ensure that previous hooktasks are correctly sent, a
`payload_version` is added (version 1: the body has already been
pre-process / version 2: the body is the raw event)

So future webhook additions will only have to deal with creating an
http.Request based on the raw event (no need to adjust the code in
multiple places, like currently).

Moreover since this processing happens when fetching from the task
queue, it ensures that the queuing of new events (upon a `git push` for
instance) does not get slowed down by a slow webhook.

As a concrete example, the PR go-gitea#19307 for custom webhooks, should be
substantially smaller:
- no need to change `services/webhook/deliver.go` 
- minimal change in `services/webhook/webhook.go` (add the new webhook
to the map)
- no need to change all the individual webhook files (since with this
refactor the `*webhook_model.Webhook` is provided as argument)

Author: oliverpool

models/fixtures/hook_task.yml

@@ -3,3 +3,35 @@
   hook_id: 1
   uuid: uuid1
   is_delivered: true
+  is_succeed: false
+  request_content: >
+    {
+      "url": "/matrix-delivered",
+      "http_method":"PUT",
+      "headers": {
+        "X-Head": "42"
+      },
+      "body": "{}"
+    }
+
+-
+  id: 2
+  hook_id: 1
+  uuid: uuid2
+  is_delivered: false
+
+-
+  id: 3
+  hook_id: 1
+  uuid: uuid3
+  is_delivered: true
+  is_succeed: true
+  payload_content: '{"key":"value"}' # legacy task, payload saved in payload_content (and not in request_content)
+  request_content: >
+    {
+      "url": "/matrix-success",
+      "http_method":"PUT",
+      "headers": {
+        "X-Head": "42"
+      }
+    }

models/migrations/migrations.go

@@ -564,6 +564,8 @@ var migrations = []Migration{
 	NewMigration("Add user_blocking table", v1_22.AddUserBlockingTable),
 	// v289 -> v290
 	NewMigration("Add default_wiki_branch to repository table", v1_22.AddDefaultWikiBranch),
+	// v290 -> v291
+	NewMigration("Add PayloadVersion to HookTask", v1_22.AddPayloadVersionToHookTaskTable),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v1_22/v290.go

@@ -0,0 +1,17 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_22 //nolint
+
+import (
+	"xorm.io/xorm"
+)
+
+type HookTask struct {
+	PayloadVersion int `xorm:"DEFAULT 1"`
+}
+
+func AddPayloadVersionToHookTaskTable(x *xorm.Engine) error {
+	// create missing column
+	return x.Sync(new(HookTask))
+}

models/webhook/hooktask.go

@@ -5,13 +5,13 @@ package webhook
 
 import (
 	"context"
+	"errors"
 	"time"
 
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
-	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
 	webhook_module "code.gitea.io/gitea/modules/webhook"
 
@@ -31,6 +31,7 @@ type HookRequest struct {
 	URL        string            `json:"url"`
 	HTTPMethod string            `json:"http_method"`
 	Headers    map[string]string `json:"headers"`
+	Body       string            `json:"body"`
 }
 
 // HookResponse represents hook task response information.
@@ -45,11 +46,15 @@ type HookTask struct {
 	ID             int64  `xorm:"pk autoincr"`
 	HookID         int64  `xorm:"index"`
 	UUID           string `xorm:"unique"`
-	api.Payloader  `xorm:"-"`
 	PayloadContent string `xorm:"LONGTEXT"`
-	EventType      webhook_module.HookEventType
-	IsDelivered    bool
-	Delivered      timeutil.TimeStampNano
+	// PayloadVersion number to allow for smooth version upgrades:
+	//  - PayloadVersion 1: PayloadContent contains the JSON as sent to the URL
+	//  - PayloadVersion 2: PayloadContent contains the original event
+	PayloadVersion int `xorm:"DEFAULT 1"`
+
+	EventType   webhook_module.HookEventType
+	IsDelivered bool
+	Delivered   timeutil.TimeStampNano
 
 	// History info.
 	IsSucceed       bool
@@ -115,16 +120,12 @@ func HookTasks(ctx context.Context, hookID int64, page int) ([]*HookTask, error)
 // it handles conversion from Payload to PayloadContent.
 func CreateHookTask(ctx context.Context, t *HookTask) (*HookTask, error) {
 	t.UUID = gouuid.New().String()
-	if t.Payloader != nil {
-		data, err := t.Payloader.JSONPayload()
-		if err != nil {
-			return nil, err
-		}
-		t.PayloadContent = string(data)
-	}
 	if t.Delivered == 0 {
 		t.Delivered = timeutil.TimeStampNanoNow()
 	}
+	if t.PayloadVersion == 0 {
+		return nil, errors.New("missing HookTask.PayloadVersion")
+	}
 	return t, db.Insert(ctx, t)
 }
 
@@ -165,6 +166,7 @@ func ReplayHookTask(ctx context.Context, hookID int64, uuid string) (*HookTask,
 		HookID:         task.HookID,
 		PayloadContent: task.PayloadContent,
 		EventType:      task.EventType,
+		PayloadVersion: task.PayloadVersion,
 	})
 }
 

models/webhook/webhook_test.go

@@ -12,7 +12,6 @@ import (
 	"code.gitea.io/gitea/models/unittest"
 	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/optional"
-	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
 	webhook_module "code.gitea.io/gitea/modules/webhook"
 
@@ -35,8 +34,10 @@ func TestWebhook_History(t *testing.T) {
 	webhook := unittest.AssertExistsAndLoadBean(t, &Webhook{ID: 1})
 	tasks, err := webhook.History(db.DefaultContext, 0)
 	assert.NoError(t, err)
-	if assert.Len(t, tasks, 1) {
-		assert.Equal(t, int64(1), tasks[0].ID)
+	if assert.Len(t, tasks, 3) {
+		assert.Equal(t, int64(3), tasks[0].ID)
+		assert.Equal(t, int64(2), tasks[1].ID)
+		assert.Equal(t, int64(1), tasks[2].ID)
 	}
 
 	webhook = unittest.AssertExistsAndLoadBean(t, &Webhook{ID: 2})
@@ -197,8 +198,10 @@ func TestHookTasks(t *testing.T) {
 	assert.NoError(t, unittest.PrepareTestDatabase())
 	hookTasks, err := HookTasks(db.DefaultContext, 1, 1)
 	assert.NoError(t, err)
-	if assert.Len(t, hookTasks, 1) {
-		assert.Equal(t, int64(1), hookTasks[0].ID)
+	if assert.Len(t, hookTasks, 3) {
+		assert.Equal(t, int64(3), hookTasks[0].ID)
+		assert.Equal(t, int64(2), hookTasks[1].ID)
+		assert.Equal(t, int64(1), hookTasks[2].ID)
 	}
 
 	hookTasks, err = HookTasks(db.DefaultContext, unittest.NonexistentID, 1)
@@ -209,8 +212,8 @@ func TestHookTasks(t *testing.T) {
 func TestCreateHookTask(t *testing.T) {
 	assert.NoError(t, unittest.PrepareTestDatabase())
 	hookTask := &HookTask{
-		HookID:    3,
-		Payloader: &api.PushPayload{},
+		HookID:         3,
+		PayloadVersion: 2,
 	}
 	unittest.AssertNotExistsBean(t, hookTask)
 	_, err := CreateHookTask(db.DefaultContext, hookTask)
@@ -232,10 +235,10 @@ func TestUpdateHookTask(t *testing.T) {
 func TestCleanupHookTaskTable_PerWebhook_DeletesDelivered(t *testing.T) {
 	assert.NoError(t, unittest.PrepareTestDatabase())
 	hookTask := &HookTask{
-		HookID:      3,
-		Payloader:   &api.PushPayload{},
-		IsDelivered: true,
-		Delivered:   timeutil.TimeStampNanoNow(),
+		HookID:         3,
+		IsDelivered:    true,
+		Delivered:      timeutil.TimeStampNanoNow(),
+		PayloadVersion: 2,
 	}
 	unittest.AssertNotExistsBean(t, hookTask)
 	_, err := CreateHookTask(db.DefaultContext, hookTask)
@@ -249,9 +252,9 @@ func TestCleanupHookTaskTable_PerWebhook_DeletesDelivered(t *testing.T) {
 func TestCleanupHookTaskTable_PerWebhook_LeavesUndelivered(t *testing.T) {
 	assert.NoError(t, unittest.PrepareTestDatabase())
 	hookTask := &HookTask{
-		HookID:      4,
-		Payloader:   &api.PushPayload{},
-		IsDelivered: false,
+		HookID:         4,
+		IsDelivered:    false,
+		PayloadVersion: 2,
 	}
 	unittest.AssertNotExistsBean(t, hookTask)
 	_, err := CreateHookTask(db.DefaultContext, hookTask)
@@ -265,10 +268,10 @@ func TestCleanupHookTaskTable_PerWebhook_LeavesUndelivered(t *testing.T) {
 func TestCleanupHookTaskTable_PerWebhook_LeavesMostRecentTask(t *testing.T) {
 	assert.NoError(t, unittest.PrepareTestDatabase())
 	hookTask := &HookTask{
-		HookID:      4,
-		Payloader:   &api.PushPayload{},
-		IsDelivered: true,
-		Delivered:   timeutil.TimeStampNanoNow(),
+		HookID:         4,
+		IsDelivered:    true,
+		Delivered:      timeutil.TimeStampNanoNow(),
+		PayloadVersion: 2,
 	}
 	unittest.AssertNotExistsBean(t, hookTask)
 	_, err := CreateHookTask(db.DefaultContext, hookTask)
@@ -282,10 +285,10 @@ func TestCleanupHookTaskTable_PerWebhook_LeavesMostRecentTask(t *testing.T) {
 func TestCleanupHookTaskTable_OlderThan_DeletesDelivered(t *testing.T) {
 	assert.NoError(t, unittest.PrepareTestDatabase())
 	hookTask := &HookTask{
-		HookID:      3,
-		Payloader:   &api.PushPayload{},
-		IsDelivered: true,
-		Delivered:   timeutil.TimeStampNano(time.Now().AddDate(0, 0, -8).UnixNano()),
+		HookID:         3,
+		IsDelivered:    true,
+		Delivered:      timeutil.TimeStampNano(time.Now().AddDate(0, 0, -8).UnixNano()),
+		PayloadVersion: 2,
 	}
 	unittest.AssertNotExistsBean(t, hookTask)
 	_, err := CreateHookTask(db.DefaultContext, hookTask)
@@ -299,9 +302,9 @@ func TestCleanupHookTaskTable_OlderThan_DeletesDelivered(t *testing.T) {
 func TestCleanupHookTaskTable_OlderThan_LeavesUndelivered(t *testing.T) {
 	assert.NoError(t, unittest.PrepareTestDatabase())
 	hookTask := &HookTask{
-		HookID:      4,
-		Payloader:   &api.PushPayload{},
-		IsDelivered: false,
+		HookID:         4,
+		IsDelivered:    false,
+		PayloadVersion: 2,
 	}
 	unittest.AssertNotExistsBean(t, hookTask)
 	_, err := CreateHookTask(db.DefaultContext, hookTask)
@@ -315,10 +318,10 @@ func TestCleanupHookTaskTable_OlderThan_LeavesUndelivered(t *testing.T) {
 func TestCleanupHookTaskTable_OlderThan_LeavesTaskEarlierThanAgeToDelete(t *testing.T) {
 	assert.NoError(t, unittest.PrepareTestDatabase())
 	hookTask := &HookTask{
-		HookID:      4,
-		Payloader:   &api.PushPayload{},
-		IsDelivered: true,
-		Delivered:   timeutil.TimeStampNano(time.Now().AddDate(0, 0, -6).UnixNano()),
+		HookID:         4,
+		IsDelivered:    true,
+		Delivered:      timeutil.TimeStampNano(time.Now().AddDate(0, 0, -6).UnixNano()),
+		PayloadVersion: 2,
 	}
 	unittest.AssertNotExistsBean(t, hookTask)
 	_, err := CreateHookTask(db.DefaultContext, hookTask)