diff --git a/config/config.go b/config/config.go index da37764..cb77e3a 100644 --- a/config/config.go +++ b/config/config.go @@ -21,6 +21,7 @@ type HeplifyServer struct { LokiBuffer int `default:"100000"` LokiHEPFilter []int `default:"1,5,100"` LokiIPPortLabels bool `default:"false"` + LokiSkipTCPPortLabels bool `default:"true"` LokiFromToLabels bool `default:"false"` LokiCallIDLabels bool `default:"false"` LokiAllowOutOfOrder bool `default:"false"` diff --git a/example/homer5_config/heplify-server.toml b/example/homer5_config/heplify-server.toml index ae69472..b012f23 100644 --- a/example/homer5_config/heplify-server.toml +++ b/example/homer5_config/heplify-server.toml @@ -8,6 +8,8 @@ LokiBulk = 200 LokiTimer = 4 LokiBuffer = 100000 LokiHEPFilter = [1,5,100] +LokiIPPortLabels = false +LokiSkipTCPPortLabels = true LokiAllowOutOfOrder = false LokiCustomLabels = [] PromAddr = "" diff --git a/example/homer7_config/heplify-server.toml b/example/homer7_config/heplify-server.toml index 2910c38..3f10cdc 100644 --- a/example/homer7_config/heplify-server.toml +++ b/example/homer7_config/heplify-server.toml @@ -9,6 +9,8 @@ LokiBulk = 200 LokiTimer = 4 LokiBuffer = 100000 LokiHEPFilter = [1,5,100] +LokiIPPortLabels = false +LokiSkipTCPPortLabels = true LokiAllowOutOfOrder = false LokiCustomLabels = [] ForceHEPPayload = [] diff --git a/example/lineproto_config.toml b/example/lineproto_config.toml index 70f02bb..b61f2f0 100644 --- a/example/lineproto_config.toml +++ b/example/lineproto_config.toml @@ -42,7 +42,8 @@ LokiBulk = 400 LokiTimer = 4 LokiBuffer = 100000 LokiHEPFilter = [1, 5, 100] -LokiIPPortLabels = false +LokiIPPortLabels = false # Set to true to include src_ip, src_port, dst_ip, dst_port as labels +LokiSkipTCPPortLabels = true # Skip port labels for TCP to reduce cardinality (only applies when LokiIPPortLabels is true) LokiAllowOutOfOrder = false LokiCustomLabels = [] diff --git a/remotelog/loki.go b/remotelog/loki.go index e421810..39be39c 100644 --- a/remotelog/loki.go +++ b/remotelog/loki.go @@ -193,9 +193,14 @@ func (l *Loki) start(hCh chan *decoder.HEP) { if config.Setting.LokiIPPortLabels { l.entry.labels["src_ip"] = model.LabelValue(pkt.SrcIP) - l.entry.labels["src_port"] = model.LabelValue(strconv.FormatUint(uint64(pkt.SrcPort), 10)) l.entry.labels["dst_ip"] = model.LabelValue(pkt.DstIP) - l.entry.labels["dst_port"] = model.LabelValue(strconv.FormatUint(uint64(pkt.DstPort), 10)) + // Skip port labels for TCP if LokiSkipTCPPortLabels is enabled (default: true) + // This reduces cardinality since TCP source ports are typically dynamic + skipPortLabels := config.Setting.LokiSkipTCPPortLabels && pkt.Protocol == 6 + if !skipPortLabels { + l.entry.labels["src_port"] = model.LabelValue(strconv.FormatUint(uint64(pkt.SrcPort), 10)) + l.entry.labels["dst_port"] = model.LabelValue(strconv.FormatUint(uint64(pkt.DstPort), 10)) + } } for k, v := range pkt.CustomLokiLabels { diff --git a/remotelog/loki_test.go b/remotelog/loki_test.go index a029ce9..ba35fac 100644 --- a/remotelog/loki_test.go +++ b/remotelog/loki_test.go @@ -6,7 +6,10 @@ import ( "net/http/httptest" "testing" + "github.com/prometheus/common/model" "github.com/sipcapture/heplify-server/config" + "github.com/sipcapture/heplify-server/decoder" + "github.com/sipcapture/heplify-server/remotelog/logproto" ) func withConfig(t *testing.T, fn func()) { @@ -95,3 +98,124 @@ func TestLokiSendSendsOrgIDHeaderAndFailsOnNon2xx(t *testing.T) { } }) } + +func TestLokiSkipsTCPPortLabels(t *testing.T) { + withConfig(t, func() { + config.Setting.LokiIPPortLabels = true + config.Setting.LokiSkipTCPPortLabels = true + + loki := &Loki{} + loki.entry = entry{model.LabelSet{}, logproto.Entry{}} + + // Test TCP packet (protocol 6) + tcpPkt := &decoder.HEP{ + Protocol: 6, // TCP + SrcIP: "192.168.1.1", + DstIP: "192.168.1.2", + SrcPort: 12345, + DstPort: 5060, + } + + // Simulate the label setting logic from loki.go + if config.Setting.LokiIPPortLabels { + loki.entry.labels["src_ip"] = model.LabelValue(tcpPkt.SrcIP) + loki.entry.labels["dst_ip"] = model.LabelValue(tcpPkt.DstIP) + skipPortLabels := config.Setting.LokiSkipTCPPortLabels && tcpPkt.Protocol == 6 + if !skipPortLabels { + loki.entry.labels["src_port"] = model.LabelValue("12345") + loki.entry.labels["dst_port"] = model.LabelValue("5060") + } + } + + // Verify TCP ports are NOT in labels + if _, exists := loki.entry.labels["src_port"]; exists { + t.Error("expected src_port label to be skipped for TCP") + } + if _, exists := loki.entry.labels["dst_port"]; exists { + t.Error("expected dst_port label to be skipped for TCP") + } + // Verify IPs are still present + if loki.entry.labels["src_ip"] != "192.168.1.1" { + t.Errorf("expected src_ip label to be set, got %v", loki.entry.labels["src_ip"]) + } + if loki.entry.labels["dst_ip"] != "192.168.1.2" { + t.Errorf("expected dst_ip label to be set, got %v", loki.entry.labels["dst_ip"]) + } + }) +} + +func TestLokiIncludesUDPPortLabels(t *testing.T) { + withConfig(t, func() { + config.Setting.LokiIPPortLabels = true + config.Setting.LokiSkipTCPPortLabels = true + + loki := &Loki{} + loki.entry = entry{model.LabelSet{}, logproto.Entry{}} + + // Test UDP packet (protocol 17) + udpPkt := &decoder.HEP{ + Protocol: 17, // UDP + SrcIP: "192.168.1.1", + DstIP: "192.168.1.2", + SrcPort: 5060, + DstPort: 5060, + } + + // Simulate the label setting logic from loki.go + if config.Setting.LokiIPPortLabels { + loki.entry.labels["src_ip"] = model.LabelValue(udpPkt.SrcIP) + loki.entry.labels["dst_ip"] = model.LabelValue(udpPkt.DstIP) + skipPortLabels := config.Setting.LokiSkipTCPPortLabels && udpPkt.Protocol == 6 + if !skipPortLabels { + loki.entry.labels["src_port"] = model.LabelValue("5060") + loki.entry.labels["dst_port"] = model.LabelValue("5060") + } + } + + // Verify UDP ports ARE in labels + if loki.entry.labels["src_port"] != "5060" { + t.Errorf("expected src_port label to be set for UDP, got %v", loki.entry.labels["src_port"]) + } + if loki.entry.labels["dst_port"] != "5060" { + t.Errorf("expected dst_port label to be set for UDP, got %v", loki.entry.labels["dst_port"]) + } + }) +} + +func TestLokiIncludesTCPPortLabelsWhenSkipDisabled(t *testing.T) { + withConfig(t, func() { + config.Setting.LokiIPPortLabels = true + config.Setting.LokiSkipTCPPortLabels = false + + loki := &Loki{} + loki.entry = entry{model.LabelSet{}, logproto.Entry{}} + + // Test TCP packet (protocol 6) + tcpPkt := &decoder.HEP{ + Protocol: 6, // TCP + SrcIP: "192.168.1.1", + DstIP: "192.168.1.2", + SrcPort: 12345, + DstPort: 5060, + } + + // Simulate the label setting logic from loki.go + if config.Setting.LokiIPPortLabels { + loki.entry.labels["src_ip"] = model.LabelValue(tcpPkt.SrcIP) + loki.entry.labels["dst_ip"] = model.LabelValue(tcpPkt.DstIP) + skipPortLabels := config.Setting.LokiSkipTCPPortLabels && tcpPkt.Protocol == 6 + if !skipPortLabels { + loki.entry.labels["src_port"] = model.LabelValue("12345") + loki.entry.labels["dst_port"] = model.LabelValue("5060") + } + } + + // Verify TCP ports ARE in labels when skip is disabled + if loki.entry.labels["src_port"] != "12345" { + t.Errorf("expected src_port label to be set when skip is disabled, got %v", loki.entry.labels["src_port"]) + } + if loki.entry.labels["dst_port"] != "5060" { + t.Errorf("expected dst_port label to be set when skip is disabled, got %v", loki.entry.labels["dst_port"]) + } + }) +}