Parser Example
Parser Example - WAF Unified
Parser Example - WAF Unified
Overview
Parser đồng thời xử lý 3 loại log WAF:
- Access Log: Log truy cập thành công (có field
remote_addr) - CCLOUD_WAAP: Log phát hiện threat (chứa text
CCLOUD_WAAP) - Rate Limiting: Log giới hạn tốc độ (chứa text
limiting requests)
Sample Logs
1. Access Log
{"@timestamp":"2026-03-16T07:16:57.278Z","body_bytes_sent":"183891","host_header":"webapp.example.com","http_referer":"https://webapp.example.com/products/item-details.html","http_user_agent":"Mozilla/5.0 (iPad; CPU OS 26_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/145.0.7632.108 Mobile/15E148 Safari/604.1","remote_addr":"203.0.113.10","request":"GET /api/employers?page=1 HTTP/1.1","request_id":"abcdef1234567890abcdef1234567890","request_time":"1.389","server_name":"webapp.example.com","status":"200","timestamp":"2026-03-16T07:17:00.124Z","upstream_addr":"198.51.100.100:443"}
2. CCLOUD_WAAP Threat Log
{"@timestamp":"2026-03-16T07:20:58.978Z","message":"2026/03/16 07:20:58 [error] 12345#12345: *100000001 CCLOUD_WAAP: ip=203.0.113.20&server=portal.example.vn&uri=%2Fapi%2Fv1%2Fsubmissions%2Fpost&config=learning&rid=fedcba0987654321fedcba0987654321&cscore0=$TRAVERSAL&score0=4&cscore1=$XSS&score1=104&cscore2=$SQL&score2=58, client: 203.0.113.20, server: portal.example.vn, request: \"POST /api/v1/submissions/post HTTP/1.1\", host: \"portal.example.vn\", referrer: \"https://portal.example.vn/portal/submissions/create\"","timestamp":"2026-03-16T07:21:00.927Z"}
3. Rate Limiting Log
{"@timestamp":"2026-03-13T02:31:31.182Z","message":"2026/03/13 02:31:31 [error] 12346#12346: *100000002 limiting requests, excess: 5.410 by zone \"RateLimitZone01\", client: 203.0.113.30, server: service.example.vn, request: \"GET /app/_next/static/chunks/abc123def456.js HTTP/1.1\", host: \"service.example.vn\", referrer: \"https://service.example.vn/app\"","timestamp":"2026-03-13T02:31:32.851Z"}
Parser Configuration
#regex
#conditional
event_timestamp = .timestamp
src_ip = ""
http_method = ""
url_full = ""
http_protocol = ""
http_status = ""
response_bytes = ""
http_referer = ""
user_agent = ""
server_hostname = ""
host_header = ""
request_duration = ""
event_id = ""
upstream_server = ""
log_level = ""
event_action = ""
threat_score = ""
threat_type = ""
threat_rule = ""
threat_status = ""
remote_ip_str = ""
if (remote_ip, err = to_string(.remote_addr); err == null) { remote_ip_str = remote_ip }
if remote_ip_str != "" {
src_ip = remote_ip_str
log_level = "info"
event_action = "access"
if (req_val, err = to_string(.request); err == null) {
request_parts = split(req_val, " ")
http_method = request_parts[0]
url_full = request_parts[1]
http_protocol = request_parts[2]
}
if (status_val, err = to_string(.status); err == null) { http_status = status_val }
if (bytes_val, err = to_string(.body_bytes_sent); err == null) { response_bytes = bytes_val }
if (ref_val, err = to_string(.http_referer); err == null) { http_referer = ref_val }
if (ua_val, err = to_string(.http_user_agent); err == null) { user_agent = ua_val }
if (hh_val, err = to_string(.host_header); err == null) { host_header = hh_val }
if (sn_val, err = to_string(.server_name); err == null) { server_hostname = sn_val }
if (rt_val, err = to_string(.request_time); err == null) { request_duration = rt_val }
if (rid_val, err = to_string(.request_id); err == null) { event_id = rid_val }
if (ua_val, err = to_string(.upstream_addr); err == null) { upstream_server = ua_val }
} else {
log_level = "error"
if (msg_val, err = to_string(.message); err == null) {
msg_str = msg_val
if (client_m, err = parse_regex(msg_str, r'client: (?P<ip>\d+\.\d+\.\d+\.\d+)'); err == null) {
src_ip = client_m.ip
}
if (server_m, err = parse_regex(msg_str, r'server: (?P<server>[^,]+)'); err == null) {
server_hostname = server_m.server
}
if (req_m, err = parse_regex(msg_str, r'request: "(?P<method>\w+) (?P<uri>[^ ]+) (?P<protocol>[^"]+)"'); err == null) {
http_method = req_m.method
url_full = req_m.uri
http_protocol = req_m.protocol
}
if (host_m, err = parse_regex(msg_str, r'host: "(?P<host>[^"]+)"'); err == null) {
host_header = host_m.host
}
if (ref_m, err = parse_regex(msg_str, r'referrer: "(?P<ref>[^"]+)"'); err == null) {
http_referer = ref_m.ref
}
if contains(msg_str, "CCLOUD_WAAP") {
event_action = "threat_detected"
threat_status = "detected"
if (rid_m, err = parse_regex(msg_str, r'rid=(?P<rid>[a-f0-9]+)'); err == null) {
event_id = rid_m.rid
}
threat_type = ""
if contains(msg_str, "$TRAVERSAL") { threat_type = "PATH_TRAVERSAL" }
if contains(msg_str, "$XSS") {
if threat_type == "" { threat_type = "XSS" } else { threat_type = threat_type + ",XSS" }
}
if contains(msg_str, "$SQL") {
if threat_type == "" { threat_type = "SQL_INJECTION" } else { threat_type = threat_type + ",SQL_INJECTION" }
}
}
if contains(msg_str, "limiting requests") {
event_action = "rate_limited"
threat_status = "blocked"
if (zone_m, err = parse_regex(msg_str, r'by zone "(?P<zone>[^"]+)"'); err == null) {
threat_rule = zone_m.zone
}
if (excess_m, err = parse_regex(msg_str, r'excess: (?P<excess>[\d.]+)'); err == null) {
threat_score = excess_m.excess
}
}
}
}
#normalize
timestamp: format_timestamp!(parse_timestamp!(event_timestamp, "%Y-%m-%dT%H:%M:%S.%3fZ"), "%Y-%m-%d %H:%M:%S")
source.ip: src_ip
http.request.method: http_method
url.full: url_full
http.version: http_protocol
http.response.status_code: http_status
destination.bytes: response_bytes
http.request.referrer: http_referer
user_agent.original: user_agent
url.domain: host_header
host.name: server_hostname
event.duration: request_duration
event.id: event_id
server.ip: upstream_server
log.level: log_level
event.action: event_action
threat.score: threat_score
threat.type: threat_type
threat.rule: threat_rule
threat.status: threat_status
Output (ECS Format)
1. Access Log Output
{
"timestamp": "2026-03-16T07:17:00.124Z",
"source.ip": "203.0.113.10",
"http.request.method": "GET",
"url.full": "/api/employers?page=1",
"http.version": "HTTP/1.1",
"http.response.status_code": "200",
"destination.bytes": "183891",
"http.request.referrer": "https://webapp.example.com/products/item-details.html",
"user_agent.original": "Mozilla/5.0 (iPad; CPU OS 26_3_1 like Mac OS X)...",
"url.domain": "webapp.example.com",
"host.name": "webapp.example.com",
"event.duration": "1.389",
"event.id": "abcdef1234567890abcdef1234567890",
"server.ip": "198.51.100.100:443",
"log.level": "info",
"event.action": "access"
}
2. CCLOUD_WAAP Threat Output
{
"timestamp": "2026-03-16T07:21:00.927Z",
"source.ip": "203.0.113.20",
"http.request.method": "POST",
"url.full": "/api/v1/submissions/post",
"http.version": "HTTP/1.1",
"http.request.referrer": "https://portal.example.vn/portal/submissions/create",
"url.domain": "portal.example.vn",
"host.name": "portal.example.vn",
"event.id": "fedcba0987654321fedcba0987654321",
"log.level": "error",
"event.action": "threat_detected",
"threat.status": "detected",
"threat.type": "PATH_TRAVERSAL,XSS,SQL_INJECTION"
}
3. Rate Limiting Output
{
"timestamp": "2026-03-13T02:31:32.851Z",
"source.ip": "203.0.113.30",
"http.request.method": "GET",
"url.full": "/app/_next/static/chunks/abc123def456.js",
"http.version": "HTTP/1.1",
"http.request.referrer": "https://service.example.vn/app",
"url.domain": "service.example.vn",
"host.name": "service.example.vn",
"log.level": "error",
"event.action": "rate_limited",
"threat.status": "blocked",
"threat.rule": "RateLimitZone01",
"threat.score": "5.410"
}