Parser Example

Firewall Unified

Overview

Parser đồng thời xử lý 3 loại log Firewall:

Fortigate: Log từ thiết bị Fortinet (field vendor="Fortigate", format key=value)
CheckPoint: Log từ thiết bị Check Point (field vendor="CheckPoint", format key:"value")
pfSense: Log từ pfSense filterlog (chứa text filterlog, format CSV)

Sample Logs

1. Fortigate Log

{"@timestamp":"2026-03-04T02:03:00.232Z","message":"devname=\"HNI001VASPSFW01\" devid=\"FG3K7E3919608899\" logid=\"0001000014\" type=\"traffic\" subtype=\"forward\" level=\"notice\" vd=\"VSPS-Private\" eventtime=1741053819591015336 tz=\"+0700\" srcip=172.20.68.4 srcport=10032 srcintf=\"VLAN500\" srcintfrole=\"undefined\" dstip=172.22.4.10 dstport=20826 dstintf=\"VLAN200\" dstintfrole=\"undefined\" srccountry=\"Reserved\" dstcountry=\"Reserved\" sessionid=3795028614 proto=17 action=\"accept\" policyid=112 policytype=\"policy\" poluuid=\"6c09cd1e-ed5c-51ed-9b9d-68a88e9eb2a2\" policyname=\"vos-to-private\" service=\"udp/20826\" trandisp=\"noop\" duration=127 sentbyte=393000 rcvdbyte=0 sentpkt=1000 rcvdpkt=0 appcat=\"unscanned\" srcmac=\"b4:45:06:f7:bc:37\" dstmac=\"50:87:89:3b:9d:a2\"","timestamp":"2026-03-04T02:03:00.232Z","vendor":"Fortigate","host":"HNI001VASPSFW01"}

2. CheckPoint Log

{"@timestamp":"2026-03-04T02:03:00.606Z","message":"time=1741053776|action:\"Accept\"|conn_direction:\"Outgoing\"|ifdir:\"outbound\"|ifname:\"bond1.3211\"|logid:\"0\"|loguid:\"{0xa6f6feb6,0x2eb8b933,0xb8d2077b,0x6ea31667}\"|origin:\"172.28.146.11\"|originsicname:\"CN=HaNoi-SMG165,O=HAL06-S-165..\"|sequencenum:\"32\"|time:\"1741053776\"|version:\"5\"|context_num:\"1\"|dst:\"172.23.5.2\"|hll_key:\"1844012965429127614\"|layer_name:\"NB Security Policy Layer\"|layer_name:\"Application\"|layer_uuid:\"22aee7a4-f0bc-4da3-9448-0b34dedf1f8e\"|layer_uuid:\"875acdde-9cc7-4f41-8b01-25f3eee9c9f4\"|match_id:\"58\"|match_id:\"16777221\"|parent_rule:\"0\"|parent_rule:\"0\"|rule_action:\"Accept\"|rule_action:\"Accept\"|rule_name:\"Rule tam thoi\"|rule_uid:\"b2e74a92-a7db-46b0-a8b6-d8f1bcd2d63d\"|rule_uid:\"b3c4beb4-76eb-42f4-ae14-44e5ba5f01cf\"|nat_addtnl_rulenum:\"0\"|nat_rulenum:\"0\"|product:\"VPN-1 & FireWall-1\"|proto:\"17\"|s_port:\"58084\"|service:\"161\"|service_id:\"snmp\"|src:\"172.23.16.92\"","timestamp":"2026-03-04T02:03:00.606Z","vendor":"CheckPoint","host":"172.28.146.11"}

3. pfSense Log

{"@timestamp":"2026-03-16T10:33:00.759Z","message":"<134>1 2026-03-16T17:33:00+07:00 172-16-123-201 filterlog 94122 - - 141,,,ee5f19bacd61f4e3b27e76a8ea59e3e8,vtnet2,match,block,in,6,0x00,0x00000,64,VRRP,58,255,fe80::22ed:4702:91ea:2fd5,ff02::12,carp","timestamp":"2026-03-16T10:33:00.759Z","host":"10.6.24.10","vendor":"Unknown"}

Parser Configuration

#regex


#conditional
event_timestamp = .timestamp
src_ip = ""
src_port = ""
dst_ip = ""
dst_port = ""
network_protocol = ""
event_action = ""
rule_name = ""
observer_vendor = ""
observer_hostname = ""
event_type = ""
network_direction = ""
source_mac = ""
destination_mac = ""
network_bytes = ""
source_bytes = ""
destination_bytes = ""
network_iana_number = ""
observer_interface = ""
log_level = ""

vendor_str = ""
if (v, err = to_string(.vendor); err == null) { vendor_str = v }

host_str = ""
if (h, err = to_string(.host); err == null) { host_str = h }

msg_str = ""
if (m, err = to_string(.message); err == null) { msg_str = m }

if vendor_str == "Fortigate" {
  observer_vendor = "Fortinet"
  event_type = "firewall"
  
  if (srcip_m, err = parse_regex(msg_str, r'srcip=(?P<ip>[0-9.]+)'); err == null) {
    src_ip = srcip_m.ip
  }
  if (srcport_m, err = parse_regex(msg_str, r'srcport=(?P<port>\d+)'); err == null) {
    src_port = srcport_m.port
  }
  if (dstip_m, err = parse_regex(msg_str, r'dstip=(?P<ip>[0-9.]+)'); err == null) {
    dst_ip = dstip_m.ip
  }
  if (dstport_m, err = parse_regex(msg_str, r'dstport=(?P<port>\d+)'); err == null) {
    dst_port = dstport_m.port
  }
  if (proto_m, err = parse_regex(msg_str, r'proto=(?P<proto>\d+)'); err == null) {
    network_iana_number = proto_m.proto
    if proto_m.proto == "6" { network_protocol = "tcp" }
    if proto_m.proto == "17" { network_protocol = "udp" }
    if proto_m.proto == "1" { network_protocol = "icmp" }
  }
  if (action_m, err = parse_regex(msg_str, r'action="(?P<action>[^"]+)"'); err == null) {
    event_action = action_m.action
  }
  if (policy_m, err = parse_regex(msg_str, r'policyname="(?P<name>[^"]+)"'); err == null) {
    rule_name = policy_m.name
  }
  if (devname_m, err = parse_regex(msg_str, r'devname="(?P<name>[^"]+)"'); err == null) {
    observer_hostname = devname_m.name
  }
  if (srcmac_m, err = parse_regex(msg_str, r'srcmac="(?P<mac>[^"]+)"'); err == null) {
    source_mac = srcmac_m.mac
  }
  if (dstmac_m, err = parse_regex(msg_str, r'dstmac="(?P<mac>[^"]+)"'); err == null) {
    destination_mac = dstmac_m.mac
  }
  if (sent_m, err = parse_regex(msg_str, r'sentbyte=(?P<bytes>\d+)'); err == null) {
    source_bytes = sent_m.bytes
  }
  if (rcvd_m, err = parse_regex(msg_str, r'rcvdbyte=(?P<bytes>\d+)'); err == null) {
    destination_bytes = rcvd_m.bytes
  }
  if (srcintf_m, err = parse_regex(msg_str, r'srcintf="(?P<intf>[^"]+)"'); err == null) {
    observer_interface = srcintf_m.intf
  }
  if (level_m, err = parse_regex(msg_str, r'level="(?P<level>[^"]+)"'); err == null) {
    log_level = level_m.level
  }
  if (subtype_m, err = parse_regex(msg_str, r'subtype="(?P<sub>[^"]+)"'); err == null) {
    if subtype_m.sub == "forward" { network_direction = "internal" }
  }
}

if vendor_str == "CheckPoint" {
  observer_vendor = "Check Point"
  event_type = "firewall"
  
  if (src_m, err = parse_regex(msg_str, r'src:"(?P<ip>[^"]+)"'); err == null) {
    src_ip = src_m.ip
  }
  if (sport_m, err = parse_regex(msg_str, r's_port:"(?P<port>[^"]+)"'); err == null) {
    src_port = sport_m.port
  }
  if (dst_m, err = parse_regex(msg_str, r'dst:"(?P<ip>[^"]+)"'); err == null) {
    dst_ip = dst_m.ip
  }
  if (svc_m, err = parse_regex(msg_str, r'service:"(?P<port>[^"]+)"'); err == null) {
    dst_port = svc_m.port
  }
  if (proto_m, err = parse_regex(msg_str, r'proto:"(?P<proto>[^"]+)"'); err == null) {
    network_iana_number = proto_m.proto
    if proto_m.proto == "6" { network_protocol = "tcp" }
    if proto_m.proto == "17" { network_protocol = "udp" }
    if proto_m.proto == "1" { network_protocol = "icmp" }
  }
  if (action_m, err = parse_regex(msg_str, r'action:"(?P<action>[^"]+)"'); err == null) {
    act_lower = to_string(action_m.action)
    if act_lower == "Accept" { event_action = "accept" }
    if act_lower == "Drop" { event_action = "drop" }
    if act_lower == "Reject" { event_action = "reject" }
  }
  if (rule_m, err = parse_regex(msg_str, r'rule_name:"(?P<name>[^"]+)"'); err == null) {
    rule_name = rule_m.name
  }
  if (origin_m, err = parse_regex(msg_str, r'origin:"(?P<host>[^"]+)"'); err == null) {
    observer_hostname = origin_m.host
  }
  if (ifname_m, err = parse_regex(msg_str, r'ifname:"(?P<intf>[^"]+)"'); err == null) {
    observer_interface = ifname_m.intf
  }
  if (dir_m, err = parse_regex(msg_str, r'conn_direction:"(?P<dir>[^"]+)"'); err == null) {
    if dir_m.dir == "Outgoing" { network_direction = "outbound" }
    if dir_m.dir == "Incoming" { network_direction = "inbound" }
  }
}

if contains(msg_str, "filterlog") {
  observer_vendor = "Netgate"
  event_type = "firewall"
  observer_hostname = host_str
  
  if (csv_m, err = parse_regex(msg_str, r'filterlog \d+ - - (?P<csv>.+)$'); err == null) {
    csv_str = to_string(csv_m.csv)
    csv_parts = split(csv_str, ",")
    
    event_action = to_string(csv_parts[6])
    dir_val = to_string(csv_parts[7])
    if dir_val == "in" { network_direction = "inbound" }
    if dir_val == "out" { network_direction = "outbound" }
    observer_interface = to_string(csv_parts[4])
    
    ipver = to_string(csv_parts[8])
    
    if ipver == "4" {
      proto_val = to_string(csv_parts[16])
      network_iana_number = proto_val
      if proto_val == "6" { network_protocol = "tcp" }
      if proto_val == "17" { network_protocol = "udp" }
      if proto_val == "1" { network_protocol = "icmp" }
      src_ip = to_string(csv_parts[18])
      dst_ip = to_string(csv_parts[19])
    }
    
    if ipver == "6" {
      proto_val = to_string(csv_parts[12])
      network_iana_number = proto_val
      if proto_val == "tcp" { network_protocol = "tcp" }
      if proto_val == "udp" { network_protocol = "udp" }
      if proto_val == "icmp6" { network_protocol = "icmp6" }
      if proto_val == "VRRP" { network_protocol = "vrrp" }
      src_ip = to_string(csv_parts[15])
      dst_ip = to_string(csv_parts[16])
    }
  }
}

#normalize
timestamp: format_timestamp!(parse_timestamp!(event_timestamp, "%Y-%m-%dT%H:%M:%S.%3fZ"), "%Y-%m-%d %H:%M:%S")
source.ip: src_ip
source.port: src_port
destination.ip: dst_ip
destination.port: dst_port
network.transport: network_protocol
network.iana_number: network_iana_number
network.direction: network_direction
event.action: event_action
event.type: event_type
rule.name: rule_name
observer.vendor: observer_vendor
observer.hostname: observer_hostname
observer.ingress.interface.name: observer_interface
source.mac: source_mac
destination.mac: destination_mac
source.bytes: source_bytes
destination.bytes: destination_bytes
log.level: log_level

Output (ECS Format)

1. Fortigate Output

{
  "timestamp": "2026-03-04T02:03:00.232Z",
  "source.ip": "172.20.68.4",
  "source.port": "10032",
  "destination.ip": "172.22.4.10",
  "destination.port": "20826",
  "network.transport": "udp",
  "network.iana_number": "17",
  "network.direction": "internal",
  "event.action": "accept",
  "event.type": "firewall",
  "rule.name": "vos-to-private",
  "observer.vendor": "Fortinet",
  "observer.hostname": "HNI001VASPSFW01",
  "observer.ingress.interface.name": "VLAN500",
  "source.mac": "b4:45:06:f7:bc:37",
  "destination.mac": "50:87:89:3b:9d:a2",
  "source.bytes": "393000",
  "destination.bytes": "0",
  "log.level": "notice"
}

2. CheckPoint Output

{
  "timestamp": "2026-03-04T02:03:00.606Z",
  "source.ip": "172.23.16.92",
  "source.port": "58084",
  "destination.ip": "172.23.5.2",
  "destination.port": "161",
  "network.transport": "udp",
  "network.iana_number": "17",
  "network.direction": "outbound",
  "event.action": "accept",
  "event.type": "firewall",
  "rule.name": "Rule tam thoi",
  "observer.vendor": "Check Point",
  "observer.hostname": "172.28.146.11",
  "observer.ingress.interface.name": "bond1.3211"
}

3. pfSense Output

{
  "timestamp": "2026-03-16T10:33:00.759Z",
  "source.ip": "fe80::22ed:4702:91ea:2fd5",
  "destination.ip": "ff02::12",
  "network.transport": "vrrp",
  "network.iana_number": "VRRP",
  "network.direction": "inbound",
  "event.action": "block",
  "event.type": "firewall",
  "observer.vendor": "Netgate",
  "observer.hostname": "10.6.24.10",
  "observer.ingress.interface.name": "vtnet2"
}