Parser Example

Firewall Unified

Firewall Unified

Overview

Parser đồng thời xử lý 3 loại log Firewall:

  • Fortigate: Log từ thiết bị Fortinet (field vendor="Fortigate", format key=value)
  • CheckPoint: Log từ thiết bị Check Point (field vendor="CheckPoint", format key:"value")
  • pfSense: Log từ pfSense filterlog (chứa text filterlog, format CSV)

Sample Logs

1. Fortigate Log

{"@timestamp":"2026-03-04T02:03:00.232Z","message":"devname=\"FG-BRANCH-01\" devid=\"FG3K0000000000001\" logid=\"0001000014\" type=\"traffic\" subtype=\"forward\" level=\"notice\" vd=\"vdom1\" eventtime=1741053819591015336 tz=\"+0700\" srcip=192.168.100.4 srcport=10032 srcintf=\"VLAN100\" srcintfrole=\"undefined\" dstip=192.168.200.10 dstport=20826 dstintf=\"VLAN200\" dstintfrole=\"undefined\" srccountry=\"Reserved\" dstcountry=\"Reserved\" sessionid=3795028614 proto=17 action=\"accept\" policyid=112 policytype=\"policy\" poluuid=\"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx\" policyname=\"lan-to-wan\" service=\"udp/20826\" trandisp=\"noop\" duration=127 sentbyte=393000 rcvdbyte=0 sentpkt=1000 rcvdpkt=0 appcat=\"unscanned\" srcmac=\"aa:bb:cc:11:22:33\" dstmac=\"aa:bb:cc:44:55:66\"","timestamp":"2026-03-04T02:03:00.232Z","vendor":"Fortigate","host":"FG-BRANCH-01"}

2. CheckPoint Log

{"@timestamp":"2026-03-04T02:03:00.606Z","message":"time=1741053776|action:\"Accept\"|conn_direction:\"Outgoing\"|ifdir:\"outbound\"|ifname:\"eth0.100\"|logid:\"0\"|loguid:\"{0x00000001,0x00000001,0x00000001,0x00000001}\"|origin:\"192.168.1.1\"|originsicname:\"CN=FW-BRANCH-01,O=MGMT-SERVER-01..\"|sequencenum:\"32\"|time:\"1741053776\"|version:\"5\"|context_num:\"1\"|dst:\"192.168.3.2\"|hll_key:\"0\"|layer_name:\"NB Security Policy Layer\"|layer_name:\"Application\"|layer_uuid:\"00000000-1111-0000-1111-000000000003\"|layer_uuid:\"00000000-1111-0000-1111-000000000004\"|match_id:\"58\"|match_id:\"16777221\"|parent_rule:\"0\"|parent_rule:\"0\"|rule_action:\"Accept\"|rule_action:\"Accept\"|rule_name:\"Temporary-Rule-01\"|rule_uid:\"00000000-1111-0000-1111-000000000001\"|rule_uid:\"00000000-1111-0000-1111-000000000002\"|nat_addtnl_rulenum:\"0\"|nat_rulenum:\"0\"|product:\"VPN-1 & FireWall-1\"|proto:\"17\"|s_port:\"58084\"|service:\"161\"|service_id:\"snmp\"|src:\"192.168.2.92\"","timestamp":"2026-03-04T02:03:00.606Z","vendor":"CheckPoint","host":"192.168.1.1"}

3. pfSense Log

{"@timestamp":"2026-03-16T10:33:00.759Z","message":"<134>1 2026-03-16T17:33:00+07:00 192-168-1-1 filterlog 94122 - - 141,,,ee5f19bacd61f4e3b27e76a8ea59e3e8,vtnet2,match,block,in,6,0x00,0x00000,64,VRRP,58,255,fe80::22ed:4702:91ea:2fd5,ff02::12,carp","timestamp":"2026-03-16T10:33:00.759Z","host":"192.168.0.1","vendor":"Unknown"}

Parser Configuration

#regex


#conditional
event_timestamp = .timestamp
src_ip = ""
src_port = ""
dst_ip = ""
dst_port = ""
network_protocol = ""
event_action = ""
rule_name = ""
observer_vendor = ""
observer_hostname = ""
event_type = ""
network_direction = ""
source_mac = ""
destination_mac = ""
network_bytes = ""
source_bytes = ""
destination_bytes = ""
network_iana_number = ""
observer_interface = ""
log_level = ""

vendor_str = ""
if (v, err = to_string(.vendor); err == null) { vendor_str = v }

host_str = ""
if (h, err = to_string(.host); err == null) { host_str = h }

msg_str = ""
if (m, err = to_string(.message); err == null) { msg_str = m }

if vendor_str == "Fortigate" {
  observer_vendor = "Fortinet"
  event_type = "firewall"
  
  if (srcip_m, err = parse_regex(msg_str, r'srcip=(?P<ip>[0-9.]+)'); err == null) {
    src_ip = srcip_m.ip
  }
  if (srcport_m, err = parse_regex(msg_str, r'srcport=(?P<port>\d+)'); err == null) {
    src_port = srcport_m.port
  }
  if (dstip_m, err = parse_regex(msg_str, r'dstip=(?P<ip>[0-9.]+)'); err == null) {
    dst_ip = dstip_m.ip
  }
  if (dstport_m, err = parse_regex(msg_str, r'dstport=(?P<port>\d+)'); err == null) {
    dst_port = dstport_m.port
  }
  if (proto_m, err = parse_regex(msg_str, r'proto=(?P<proto>\d+)'); err == null) {
    network_iana_number = proto_m.proto
    if proto_m.proto == "6" { network_protocol = "tcp" }
    if proto_m.proto == "17" { network_protocol = "udp" }
    if proto_m.proto == "1" { network_protocol = "icmp" }
  }
  if (action_m, err = parse_regex(msg_str, r'action="(?P<action>[^"]+)"'); err == null) {
    event_action = action_m.action
  }
  if (policy_m, err = parse_regex(msg_str, r'policyname="(?P<name>[^"]+)"'); err == null) {
    rule_name = policy_m.name
  }
  if (devname_m, err = parse_regex(msg_str, r'devname="(?P<name>[^"]+)"'); err == null) {
    observer_hostname = devname_m.name
  }
  if (srcmac_m, err = parse_regex(msg_str, r'srcmac="(?P<mac>[^"]+)"'); err == null) {
    source_mac = srcmac_m.mac
  }
  if (dstmac_m, err = parse_regex(msg_str, r'dstmac="(?P<mac>[^"]+)"'); err == null) {
    destination_mac = dstmac_m.mac
  }
  if (sent_m, err = parse_regex(msg_str, r'sentbyte=(?P<bytes>\d+)'); err == null) {
    source_bytes = sent_m.bytes
  }
  if (rcvd_m, err = parse_regex(msg_str, r'rcvdbyte=(?P<bytes>\d+)'); err == null) {
    destination_bytes = rcvd_m.bytes
  }
  if (srcintf_m, err = parse_regex(msg_str, r'srcintf="(?P<intf>[^"]+)"'); err == null) {
    observer_interface = srcintf_m.intf
  }
  if (level_m, err = parse_regex(msg_str, r'level="(?P<level>[^"]+)"'); err == null) {
    log_level = level_m.level
  }
  if (subtype_m, err = parse_regex(msg_str, r'subtype="(?P<sub>[^"]+)"'); err == null) {
    if subtype_m.sub == "forward" { network_direction = "internal" }
  }
}

if vendor_str == "CheckPoint" {
  observer_vendor = "Check Point"
  event_type = "firewall"
  
  if (src_m, err = parse_regex(msg_str, r'src:"(?P<ip>[^"]+)"'); err == null) {
    src_ip = src_m.ip
  }
  if (sport_m, err = parse_regex(msg_str, r's_port:"(?P<port>[^"]+)"'); err == null) {
    src_port = sport_m.port
  }
  if (dst_m, err = parse_regex(msg_str, r'dst:"(?P<ip>[^"]+)"'); err == null) {
    dst_ip = dst_m.ip
  }
  if (svc_m, err = parse_regex(msg_str, r'service:"(?P<port>[^"]+)"'); err == null) {
    dst_port = svc_m.port
  }
  if (proto_m, err = parse_regex(msg_str, r'proto:"(?P<proto>[^"]+)"'); err == null) {
    network_iana_number = proto_m.proto
    if proto_m.proto == "6" { network_protocol = "tcp" }
    if proto_m.proto == "17" { network_protocol = "udp" }
    if proto_m.proto == "1" { network_protocol = "icmp" }
  }
  if (action_m, err = parse_regex(msg_str, r'action:"(?P<action>[^"]+)"'); err == null) {
    act_lower = to_string(action_m.action)
    if act_lower == "Accept" { event_action = "accept" }
    if act_lower == "Drop" { event_action = "drop" }
    if act_lower == "Reject" { event_action = "reject" }
  }
  if (rule_m, err = parse_regex(msg_str, r'rule_name:"(?P<name>[^"]+)"'); err == null) {
    rule_name = rule_m.name
  }
  if (origin_m, err = parse_regex(msg_str, r'origin:"(?P<host>[^"]+)"'); err == null) {
    observer_hostname = origin_m.host
  }
  if (ifname_m, err = parse_regex(msg_str, r'ifname:"(?P<intf>[^"]+)"'); err == null) {
    observer_interface = ifname_m.intf
  }
  if (dir_m, err = parse_regex(msg_str, r'conn_direction:"(?P<dir>[^"]+)"'); err == null) {
    if dir_m.dir == "Outgoing" { network_direction = "outbound" }
    if dir_m.dir == "Incoming" { network_direction = "inbound" }
  }
}

if contains(msg_str, "filterlog") {
  observer_vendor = "Netgate"
  event_type = "firewall"
  observer_hostname = host_str
  
  if (csv_m, err = parse_regex(msg_str, r'filterlog \d+ - - (?P<csv>.+)$'); err == null) {
    csv_str = to_string(csv_m.csv)
    csv_parts = split(csv_str, ",")
    
    event_action = to_string(csv_parts[6])
    dir_val = to_string(csv_parts[7])
    if dir_val == "in" { network_direction = "inbound" }
    if dir_val == "out" { network_direction = "outbound" }
    observer_interface = to_string(csv_parts[4])
    
    ipver = to_string(csv_parts[8])
    
    if ipver == "4" {
      proto_val = to_string(csv_parts[16])
      network_iana_number = proto_val
      if proto_val == "6" { network_protocol = "tcp" }
      if proto_val == "17" { network_protocol = "udp" }
      if proto_val == "1" { network_protocol = "icmp" }
      src_ip = to_string(csv_parts[18])
      dst_ip = to_string(csv_parts[19])
    }
    
    if ipver == "6" {
      proto_val = to_string(csv_parts[12])
      network_iana_number = proto_val
      if proto_val == "tcp" { network_protocol = "tcp" }
      if proto_val == "udp" { network_protocol = "udp" }
      if proto_val == "icmp6" { network_protocol = "icmp6" }
      if proto_val == "VRRP" { network_protocol = "vrrp" }
      src_ip = to_string(csv_parts[15])
      dst_ip = to_string(csv_parts[16])
    }
  }
}

#normalize
timestamp: format_timestamp!(parse_timestamp!(event_timestamp, "%Y-%m-%dT%H:%M:%S.%3fZ"), "%Y-%m-%d %H:%M:%S")
source.ip: src_ip
source.port: src_port
destination.ip: dst_ip
destination.port: dst_port
network.transport: network_protocol
network.iana_number: network_iana_number
network.direction: network_direction
event.action: event_action
event.type: event_type
rule.name: rule_name
observer.vendor: observer_vendor
observer.hostname: observer_hostname
observer.ingress.interface.name: observer_interface
source.mac: source_mac
destination.mac: destination_mac
source.bytes: source_bytes
destination.bytes: destination_bytes
log.level: log_level

Output (ECS Format)

1. Fortigate Output

{
  "timestamp": "2026-03-04T02:03:00.232Z",
  "source.ip": "192.168.100.4",
  "source.port": "10032",
  "destination.ip": "192.168.200.10",
  "destination.port": "20826",
  "network.transport": "udp",
  "network.iana_number": "17",
  "network.direction": "internal",
  "event.action": "accept",
  "event.type": "firewall",
  "rule.name": "lan-to-wan",
  "observer.vendor": "Fortinet",
  "observer.hostname": "FG-BRANCH-01",
  "observer.ingress.interface.name": "VLAN100",
  "source.mac": "aa:bb:cc:11:22:33",
  "destination.mac": "aa:bb:cc:44:55:66",
  "source.bytes": "393000",
  "destination.bytes": "0",
  "log.level": "notice"
}

2. CheckPoint Output

{
  "timestamp": "2026-03-04T02:03:00.606Z",
  "source.ip": "192.168.2.92",
  "source.port": "58084",
  "destination.ip": "192.168.3.2",
  "destination.port": "161",
  "network.transport": "udp",
  "network.iana_number": "17",
  "network.direction": "outbound",
  "event.action": "accept",
  "event.type": "firewall",
  "rule.name": "Temporary-Rule-01",
  "observer.vendor": "Check Point",
  "observer.hostname": "192.168.1.1",
  "observer.ingress.interface.name": "eth0.100"
}

3. pfSense Output

{
  "timestamp": "2026-03-16T10:33:00.759Z",
  "source.ip": "fe80::22ed:4702:91ea:2fd5",
  "destination.ip": "ff02::12",
  "network.transport": "vrrp",
  "network.iana_number": "VRRP",
  "network.direction": "inbound",
  "event.action": "block",
  "event.type": "firewall",
  "observer.vendor": "Netgate",
  "observer.hostname": "192.168.0.1",
  "observer.ingress.interface.name": "vtnet2"
}