Vector log collector config file exmaple

Vector config file vector.yaml

---
data_dir: "/var/lib/vector/data"

# The log file config here
sources:
  test_nginx_logs:
    type: "file"
    include:
      - "/data1/logs/nginx/*_access.log"
    ignore_older: 86400                    # 1 day

# use trasforms to customize nginx log 
transforms:
  # parse the customize nginx access log 
  parse_nginx_access_logs:
    type: "remap"
    inputs:
      # here is the sources
      - test_nginx_logs
    # use vrl file to parse the nginx log
    # the file in the same path of this file
    file: "./program.vrl"

  # after remap filter some log for drop 
  filter_nginx_access_logs:
    type: "filter"
    inputs:
      - parse_nginx_access_logs
    # filter the zabbix health check log
    # the .http_user_agent  after remap,the raw not have this
    condition:
      type: "vrl"
      source: |
        !contains(string!(.http_user_agent), "Zabbix")
        
sinks:
  # for test usage output to console
  stdout:
    type: "console"
    inputs:
      - filter_nginx_access_logs
    encoding:
      codec: "json"
      json:
        pretty: true

The nginx access log format

    log_format  mylog '[$time_local]\t$host\t$remote_addr\t"$http_x_forwarded_for"\t'
                         '$remote_user\t"$http_user_agent"\t"$request"\t'
                         '$request_length\t$request_time\t$status\t$body_bytes_sent\t'
                         '"$arg_url"\t"$arg_referrer"\t"$request_body"\t"$http_referer"';

The vrl file program.vrl

.time_local = replace!(split(string!(.message), "\t")[0], "[", "")
.time_local = replace(.time_local, "]", "")
.host = split(string!(.message), "\t")[1]
.remote_addr = split(string!(.message), "\t")[2]
.http_x_forwarded_for = split(string!(.message), "\t")[3]
.remote_user = split(string!(.message), "\t")[4]
.http_user_agent = split(string!(.message), "\t")[5]
.request = split(string!(.message), "\t")[6]
.request_length = to_int!(split(string!(.message), "\t")[7])
.request_time = to_float!(split(string!(.message), "\t")[8])
.status = to_int!(split(string!(.message), "\t")[9])
.body_bytes_sent = to_int!(split(string!(.message), "\t")[10])
.arg_url = split(string!(.message), "\t")[11]
.arg_referrer = split(string!(.message), "\t")[12]
.request_body = split(string!(.message), "\t")[13]
.http_referer = split(string!(.message), "\t")[14]
del(.message)

The nginx access log example

[14/Jan/2025:17:08:54 +0800]	www.example.com	192.168.1.10	"10.10.10.3, 172.16.23.33"	-	"Apache-HttpClient/4.5 (Java/1.8.0_161)"	"POST /domain/getinfo HTTP/1.1"	752	0.111	200	531	"-"	"-"	"data=aXXXXXXXX%2XXXXXXXXX%2XXXXXXXXX%3D%3D&sign=25xxxx6xx591%2BwJyXXXXX8%3D&appkey=N2xxxxx3"	"-"

The console output

{
  "arg_referrer": "\"-\"",
  "arg_url": "\"-\"",
  "body_bytes_sent": 531,
  "file": "/data1/logs/nginx/nginx_access.log",
  "host": "www.example.com",
  "http_referer": "\"-\"",
  "http_user_agent": "\"Apache-HttpClient/4.5 (Java/1.8.0_161)\"",
  "http_x_forwarded_for": "\"10.10.10.3, 172.16.23.33\"",
  "remote_addr": "192.168.1.10",
  "remote_user": "-",
  "request": "\"POST /domain/getinfo HTTP/1.1\"",
  "request_body": "\"data=aXXXXXXXX%2XXXXXXXXX%2XXXXXXXXX%3D%3D&sign=25xxxx6xx591%2BwJyXXXXX8%3D&appkey=N2xxxxx3\"",
  "request_length": 752,
  "request_time": 0.111,
  "source_type": "file",
  "status": 200,
  "time_local": "14/Jan/2025:17:08:54 +0800",
  "timestamp": "2025-01-14T09:13:30.665868982Z"
}