Skip to content

Commit e587bc4

Browse files
spacewanderspacewander
authored
feat: allow each logger to define custom log format in its conf (#8806)
1 parent a18573a commit e587bc4

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

63 files changed

+1254
-5
lines changed

‎apisix/plugins/clickhouse-logger.lua

+1
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ local schema = {
4040
timeout = {type = "integer", minimum = 1, default = 3},
4141
name = {type = "string", default = "clickhouse logger"},
4242
ssl_verify = {type = "boolean", default = true},
43+
log_format = {type = "object"},
4344
},
4445
oneOf = {
4546
{required = {"endpoint_addr", "user", "password", "database", "logtable"}},

‎apisix/plugins/elasticsearch-logger.lua

+1
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ local schema = {
5252
},
5353
required = {"index"}
5454
},
55+
log_format = {type = "object"},
5556
auth = {
5657
type = "object",
5758
properties = {

‎apisix/plugins/file-logger.lua

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ local schema = {
3030
path = {
3131
type = "string"
3232
},
33+
log_format = {type = "object"},
3334
include_resp_body = {type = "boolean", default = false},
3435
include_resp_body_expr = {
3536
type = "array",

‎apisix/plugins/google-cloud-logging.lua

+1
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ local schema = {
8686
type = "string",
8787
default = "apisix.apache.org%2Flogs"
8888
},
89+
log_format = {type = "object"},
8990
},
9091
oneOf = {
9192
{ required = { "auth_config" } },

‎apisix/plugins/http-logger.lua

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ local schema = {
3333
uri = core.schema.uri_def,
3434
auth_header = {type = "string"},
3535
timeout = {type = "integer", minimum = 1, default = 3},
36+
log_format = {type = "object"},
3637
include_req_body = {type = "boolean", default = false},
3738
include_resp_body = {type = "boolean", default = false},
3839
include_resp_body_expr = {

‎apisix/plugins/kafka-logger.lua

+1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ local schema = {
3737
default = "default",
3838
enum = {"default", "origin"},
3939
},
40+
log_format = {type = "object"},
4041
-- deprecated, use "brokers" instead
4142
broker_list = {
4243
type = "object",

‎apisix/plugins/loggly.lua

+1
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ local schema = {
8585
type = "boolean",
8686
default = true
8787
},
88+
log_format = {type = "object"},
8889
severity_map = {
8990
type = "object",
9091
description = "upstream response code vs syslog severity mapping",

‎apisix/plugins/rocketmq-logger.lua

+1
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ local schema = {
4646
topic = {type = "string"},
4747
key = {type = "string"},
4848
tag = {type = "string"},
49+
log_format = {type = "object"},
4950
timeout = {type = "integer", minimum = 1, default = 3},
5051
use_tls = {type = "boolean", default = false},
5152
access_key = {type = "string", default = ""},

‎apisix/plugins/skywalking-logger.lua

+1
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ local schema = {
3636
endpoint_addr = core.schema.uri_def,
3737
service_name = {type = "string", default = "APISIX"},
3838
service_instance_name = {type = "string", default = "APISIX Instance Name"},
39+
log_format = {type = "object"},
3940
timeout = {type = "integer", minimum = 1, default = 3},
4041
include_req_body = {type = "boolean", default = false},
4142
},

‎apisix/plugins/sls-logger.lua

+1
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ local schema = {
3434
properties = {
3535
include_req_body = {type = "boolean", default = false},
3636
timeout = {type = "integer", minimum = 1, default= 5000},
37+
log_format = {type = "object"},
3738
host = {type = "string"},
3839
port = {type = "integer"},
3940
project = {type = "string"},

‎apisix/plugins/splunk-hec-logging.lua

+1
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ local schema = {
5656
type = "boolean",
5757
default = true
5858
},
59+
log_format = {type = "object"},
5960
},
6061
required = { "endpoint" },
6162
}

‎apisix/plugins/syslog.lua

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ local schema = {
3333
sock_type = {type = "string", default = "tcp", enum = {"tcp", "udp"}},
3434
pool_size = {type = "integer", minimum = 5, default = 5},
3535
tls = {type = "boolean", default = false},
36+
log_format = {type = "object"},
3637
include_req_body = {type = "boolean", default = false}
3738
},
3839
required = {"host", "port"}

‎apisix/plugins/tcp-logger.lua

+1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ local schema = {
3232
tls = {type = "boolean", default = false},
3333
tls_options = {type = "string"},
3434
timeout = {type = "integer", minimum = 1, default= 1000},
35+
log_format = {type = "object"},
3536
include_req_body = {type = "boolean", default = false}
3637
},
3738
required = {"host", "port"}

‎apisix/plugins/tencent-cloud-cls.lua

+1
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ local schema = {
4141
include_req_body = { type = "boolean", default = false },
4242
include_resp_body = { type = "boolean", default = false },
4343
global_tag = { type = "object" },
44+
log_format = {type = "object"},
4445
},
4546
encrypt_fields = {"secret_key"},
4647
required = { "cls_host", "cls_topic", "secret_id", "secret_key" }

‎apisix/plugins/udp-logger.lua

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ local schema = {
3030
host = {type = "string"},
3131
port = {type = "integer", minimum = 0},
3232
timeout = {type = "integer", minimum = 1, default = 3},
33+
log_format = {type = "object"},
3334
include_req_body = {type = "boolean", default = false}
3435
},
3536
required = {"host", "port"}

‎apisix/stream/plugins/syslog.lua

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ local schema = {
3030
flush_limit = {type = "integer", minimum = 1, default = 4096},
3131
drop_limit = {type = "integer", default = 1048576},
3232
timeout = {type = "integer", minimum = 1, default = 3000},
33+
log_format = {type = "object"},
3334
sock_type = {type = "string", default = "tcp", enum = {"tcp", "udp"}},
3435
pool_size = {type = "integer", minimum = 5, default = 5},
3536
tls = {type = "boolean", default = false}

‎apisix/utils/log-util.lua

+4-3
Original file line numberDiff line numberDiff line change
@@ -217,11 +217,12 @@ function _M.get_log_entry(plugin_name, conf, ctx)
217217
local entry
218218
local customized = false
219219

220-
if metadata and metadata.value.log_format
220+
local has_meta_log_format = metadata and metadata.value.log_format
221221
and core.table.nkeys(metadata.value.log_format) > 0
222-
then
222+
223+
if conf.log_format or has_meta_log_format then
223224
customized = true
224-
entry = get_custom_format_log(ctx, metadata.value.log_format)
225+
entry = get_custom_format_log(ctx, conf.log_format or metadata.value.log_format)
225226
else
226227
if is_http then
227228
entry = get_full_log(ngx, conf)

‎docs/en/latest/plugins/clickhouse-logger.md

+1
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ The `clickhouse-logger` Plugin is used to push logs to [ClickHouse](https://clic
4444
| timeout | integer | False | 3 | [1,...] | Time to keep the connection alive for after sending a request. |
4545
| name | string | False | "clickhouse logger" | | Unique identifier for the logger. |
4646
| ssl_verify | boolean | False | true | [true,false] | When set to `true`, verifies SSL. |
47+
| log_format | object | False | | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
4748

4849
NOTE: `encrypt_fields = {"password"}` is also defined in the schema, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields).
4950

‎docs/en/latest/plugins/elasticsearch-logger.md

+1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ When the Plugin is enabled, APISIX will serialize the request context informatio
4242
| field | array | True | | Elasticsearch `field` configuration. |
4343
| field.index | string | True | | Elasticsearch [_index field](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-index-field.html#mapping-index-field). |
4444
| field.type | string | False | Elasticsearch default value | Elasticsearch [_type field](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/mapping-type-field.html#mapping-type-field). |
45+
| log_format | object | False | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
4546
| auth | array | False | | Elasticsearch [authentication](https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-up-authentication.html) configuration. |
4647
| auth.username | string | True | | Elasticsearch [authentication](https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-up-authentication.html) username. |
4748
| auth.password | string | True | | Elasticsearch [authentication](https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-up-authentication.html) password. |

‎docs/en/latest/plugins/file-logger.md

+1
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ The `file-logger` Plugin is used to push log streams to a specific location.
4646
| Name | Type | Required | Description |
4747
| ---- | ------ | -------- | ------------- |
4848
| path | string | True | Log file path. |
49+
| log_format | object | False | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
4950
| include_resp_body | boolean | False | When set to `true` includes the response body in the log file. |
5051
| include_resp_body_expr | array | False | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response into file if the expression evaluates to `true`. |
5152

‎docs/en/latest/plugins/google-cloud-logging.md

+1
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ This plugin also allows to push logs as a batch to your Google Cloud Logging Ser
4646
| ssl_verify | False | true | When set to `true`, enables SSL verification as mentioned in [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake). |
4747
| resource | False | {"type": "global"} | Google monitor resource. See [MonitoredResource](https://cloud.google.com/logging/docs/reference/v2/rest/v2/MonitoredResource) for more details. |
4848
| log_id | False | apisix.apache.org%2Flogs | Google Cloud logging ID. See [LogEntry](https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry) for details. |
49+
| log_format | False | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
4950

5051
NOTE: `encrypt_fields = {"auth_config.private_key"}` is also defined in the schema, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields).
5152

‎docs/en/latest/plugins/http-logger.md

+1
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ This will allow the ability to send log data requests as JSON objects to monitor
4040
| uri | string | True | | | URI of the HTTP/HTTPS server. |
4141
| auth_header | string | False | | | Authorization headers if required. |
4242
| timeout | integer | False | 3 | [1,...] | Time to keep the connection alive for after sending a request. |
43+
| log_format | object | False | | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
4344
| include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. |
4445
| include_resp_body | boolean | False | false | [false, true] | When set to `true` includes the response body in the log. |
4546
| include_resp_body_expr | array | False | | | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`. |

‎docs/en/latest/plugins/kafka-logger.md

+1
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ It might take some time to receive the log data. It will be automatically sent a
5252
| timeout | integer | False | 3 | [1,...] | Timeout for the upstream to send data. |
5353
| name | string | False | "kafka logger" | | Unique identifier for the batch processor. |
5454
| meta_format | enum | False | "default" | ["default"，"origin"] | Format to collect the request information. Setting to `default` collects the information in JSON format and `origin` collects the information with the original HTTP request. See [examples](#meta_format-example) below. |
55+
| log_format | object | False | | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
5556
| include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. |
5657
| include_req_body_expr | array | False | | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. |
5758
| include_resp_body | boolean | False | false | [false, true] | When set to `true` includes the response body in the log. |

‎docs/en/latest/plugins/loggly.md

+1
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ When the maximum batch size is exceeded, the data in the queue is pushed to Logg
4343
| severity | string (enum) | False | INFO | Syslog log event severity level. Choose between: `DEBUG`, `INFO`, `NOTICE`, `WARNING`, `ERR`, `CRIT`, `ALERT`, and `EMEGR`. |
4444
| severity_map | object | False | nil | A way to map upstream HTTP response codes to Syslog severity. Key-value pairs where keys are the HTTP response codes and the values are the Syslog severity levels. For example `{"410": "CRIT"}`. |
4545
| tags | array | False | | Metadata to be included with any event log to aid in segmentation and filtering. |
46+
| log_format | object | False | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
4647
| include_req_body | boolean | False | false | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. |
4748
| include_resp_body | boolean | False | false | When set to `true` includes the response body in the log. |
4849
| include_resp_body_expr | array | False | | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`. |

0 commit comments

Comments
 (0)