-
-
Notifications
You must be signed in to change notification settings - Fork 109
Support sending data to kafka #531
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| HEPAddr = "0.0.0.0:9060" | ||
| HEPTCPAddr = "" | ||
| HEPTLSAddr = "0.0.0.0:9060" | ||
| HEPWSAddr = "0.0.0.0:3000" | ||
| ESAddr = "" | ||
| ESDiscovery = true | ||
| LokiURL = "" | ||
| LokiBulk = 200 | ||
| LokiTimer = 4 | ||
| LokiBuffer = 100000 | ||
| LokiHEPFilter = [1,5,100] | ||
| ForceHEPPayload = [] | ||
| PromAddr = "" | ||
| PromTargetIP = "" | ||
| PromTargetName = "" | ||
| DBShema = "homer7" | ||
| DBDriver = "postgres" | ||
| DBAddr = "localhost:5432" | ||
| DBUser = "postgres" | ||
| DBPass = "" | ||
| DBDataTable = "homer_data" | ||
| DBConfTable = "homer_config" | ||
| DBBulk = 200 | ||
| DBTimer = 4 | ||
| DBBuffer = 400000 | ||
| DBWorker = 8 | ||
| DBRotate = true | ||
| DBPartLog = "2h" | ||
| DBPartSip = "1h" | ||
| DBPartQos = "6h" | ||
| DBDropDays = 14 | ||
| DBDropDaysCall = 0 | ||
| DBDropDaysRegister = 0 | ||
| DBDropDaysDefault = 0 | ||
| DBDropOnStart = false | ||
| DBUsageProtection = true | ||
| DBUsageScheme = "percentage" | ||
| DBPercentageUsage = "80%" | ||
| DBMaxSize = "30MB" | ||
| Dedup = false | ||
| DiscardMethod = [] | ||
| AlegIDs = [] | ||
| CustomHeader = [] | ||
| SIPHeader = [] | ||
| LogDbg = "" | ||
| LogLvl = "info" | ||
| LogStd = false | ||
| LogSys = false | ||
| Config = "./heplify-server.toml" | ||
| ConfigHTTPAddr = "" | ||
| KafkaBroker = "" | ||
| KafkaTopic = "" | ||
| KafkaDeliveryTimeout = 60000 | ||
| KafkaBatchNum = 10 | ||
| KafkaLinger = 5000 | ||
| # Examples: | ||
| # ------------------------------------- | ||
| # ESAddr = "http://127.0.0.1:9200" | ||
| # DBShema = "homer7" | ||
| # DBDriver = "postgres" | ||
| # LokiURL = "http://localhost:3100/api/prom/push" | ||
| # LokiHEPFilter = [1,5,100] | ||
| # PromAddr = "0.0.0.0:8899" | ||
| # PromTargetIP = "10.1.2.111,10.1.2.4,10.1.2.5,10.1.2.6,10.12.44.222" | ||
| # PromTargetName = "sbc_access,sbc_core,kamailio,asterisk,pstn_gateway" | ||
| # AlegIDs = ["X-CID","P-Charging-Vector,icid-value=\"?(.*?)(?:\"|;|$)","X-BroadWorks-Correlation-Info"] | ||
| # DiscardMethod = ["OPTIONS","NOTIFY"] | ||
| # CustomHeader = ["X-CustomerIP","X-Billing"] | ||
| # SIPHeader = ["callid","callid_aleg","method","ruri_user","ruri_domain","from_user","from_domain","from_tag","to_user","to_domain","to_tag","via","contact_user"] | ||
| # LogDbg = "hep,sql,loki" | ||
| # LogLvl = "warning" | ||
| # ConfigHTTPAddr = "0.0.0.0:9876" | ||
| # ------------------------------------- | ||
| # To hot reload PromTargetIP and PromTargetName run: | ||
| # killall -HUP heplify-server |
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| @@ -0,0 +1,78 @@ | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| package remotelog | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| import ( | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "context" | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "encoding/json" | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "time" | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "github.com/confluentinc/confluent-kafka-go/kafka" | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "github.com/negbie/logp" | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "github.com/sipcapture/heplify-server/config" | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "github.com/sipcapture/heplify-server/decoder" | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| ) | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| type Kafka struct { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| producer *kafka.Producer | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| topic string | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| ctx context.Context | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| func (k *Kafka) setup() error { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| k.ctx = context.Background() | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| k.topic = config.Setting.KafkaTopic | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| return k.createProducer() | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| func (k *Kafka) start(hCh chan *decoder.HEP) { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| logp.Info("start kafka...") | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| go func() { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| for e := range k.producer.Events() { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| switch ev := e.(type) { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| case *kafka.Message: | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| if ev.TopicPartition.Error != nil { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| logp.Err("Delivery failed: %v, payload: %s", ev.TopicPartition, string(ev.Value)) | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| logp.Err("Delivery failed: %v, payload: %s", ev.TopicPartition, string(ev.Value)) | |
| logp.Err("Delivery failed: %v, payload size: %d bytes", ev.TopicPartition, len(ev.Value)) |
Copilot
AI
Mar 10, 2026
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
start() starts a goroutine ranging over k.producer.Events() but the producer is never Flush()ed/Close()d when the input channel closes. This will leak the goroutine and can drop buffered messages on shutdown. Add a deferred shutdown path that flushes and closes the producer (and ensure the events goroutine exits cleanly).
Copilot
AI
Mar 10, 2026
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If json.Marshal(pkt) fails, the code logs the error but still enqueues a Kafka message with Value set to the (likely nil) v. That can lead to producing empty/invalid records and hides the original failure. Consider continueing (or otherwise skipping production) when marshaling fails.
Copilot
AI
Mar 10, 2026
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The hourly ticker recreates the producer only when k.producer == nil, but (a) the returned error is ignored, and (b) if you ever add re-creation on failures, the current structure would also need to close the old producer and synchronize access to avoid races with the events goroutine. Since librdkafka already handles reconnects, consider removing this ticker entirely or handling retries with proper error handling and resource cleanup.
| ticker := time.NewTicker(1 * time.Hour) | |
| defer ticker.Stop() | |
| for { | |
| select { | |
| case pkt, ok := <-hCh: | |
| if !ok { | |
| return | |
| } | |
| v, err := json.Marshal(pkt) | |
| if err != nil { | |
| logp.Err("%v", err) | |
| } | |
| k.producer.ProduceChannel() <- &kafka.Message{ | |
| TopicPartition: kafka.TopicPartition{Topic: &k.topic, Partition: kafka.PartitionAny}, | |
| Value: v, | |
| } | |
| case <-ticker.C: | |
| if k.producer == nil { | |
| k.createProducer() | |
| } | |
| } | |
| for pkt := range hCh { | |
| v, err := json.Marshal(pkt) | |
| if err != nil { | |
| logp.Err("%v", err) | |
| } | |
| k.producer.ProduceChannel() <- &kafka.Message{ | |
| TopicPartition: kafka.TopicPartition{Topic: &k.topic, Partition: kafka.PartitionAny}, | |
| Value: v, | |
| } |
Copilot
AI
Mar 10, 2026
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
createProducer() can return nil error while leaving k.producer nil when either KafkaBroker or KafkaTopic is empty. Since start() later dereferences k.producer, this can lead to a nil-pointer panic. Recommend returning an error when required settings are missing (or ensure Kafka is not started without them).
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Adding
confluent-kafka-gointroduces a CGO dependency onlibrdkafka. Current Docker builds use Alpine with static linking; that setup typically fails or becomes non-trivial with librdkafka (musl/static). Please ensure the build pipeline and Dockerfiles are updated to install/build librdkafka appropriately (or document the new build requirement).