abango-rest/kafka.go

package abango

import (
	"fmt"
	"strings"
	"sync"

	"github.com/IBM/sarama"
	e "github.com/dabory/abango-rest/etc"
)

var (
	KAFKA_CONN      string
	COMSUMER_TOPICS []string
	KAFKA_TIMEOUT   string
)

func KafkaInit() {
	KAFKA_CONN = XConfig["KafkaConnString"]
	COMSUMER_TOPICS = strings.Split(strings.Replace(XConfig["ConsumerTopics"], " ", "", -1), ",")
	KAFKA_TIMEOUT = XConfig["KafkaTimeout"]
	e.OkLog("== KAFKA_CONN is : " + KAFKA_CONN + " ==")
	e.OkLog("== COMSUMER_TOPICS is : " + XConfig["ConsumerTopics"] + " ==")
	e.OkLog("== KAFKA_TIMEOUT is : " + KAFKA_TIMEOUT + " ==")
}

func KafkaProducer(key string, headers []*sarama.RecordHeader, message []byte, conCurr string, topic string) (int32, int64, error) {

	kfcf := sarama.NewConfig()
	kfcf.Producer.Retry.Max = 5
	kfcf.Producer.RequiredAcks = sarama.WaitForAll
	kfcf.Producer.Return.Successes = true
	conHeaders := e.ConvertKafkaHeaders(headers)
	if conCurr == "async" {
		if prd, err := sarama.NewAsyncProducer([]string{KAFKA_CONN}, kfcf); err == nil {
			prd.Input() <- &sarama.ProducerMessage{
				Topic:   topic,
				Key:     sarama.StringEncoder(key),
				Headers: conHeaders,
				Value:   sarama.ByteEncoder(message),
			}
			return 0, 0, nil
		} else {
			return 0, 0, e.MyErr("QEJHDRTTRRW-Kafka-NewAsyncProducer-End", err, true)
		}
	} else if conCurr == "sync" {
		fmt.Println("conCurr:", conCurr)
		if prd, err := sarama.NewSyncProducer([]string{KAFKA_CONN}, kfcf); err == nil {
			msg := &sarama.ProducerMessage{
				Topic:   topic,
				Key:     sarama.StringEncoder(key),
				Headers: conHeaders,
				Value:   sarama.ByteEncoder(message),
			}
			if part, offset, err := prd.SendMessage(msg); err == nil {
				return part, offset, nil
			} else {
				return 0, 0, e.MyErr("QEJIOPRTRRTRRW-Kafka-Sync-SendMessage", err, true)
			}
		} else {
			return 0, 0, e.MyErr("QEJHGTRSDRTTRRW-Kafka-NewSyncProducer-End", err, true)
		}
	} else {
		return 0, 0, e.MyErr("QEJHGTRSW-Kafka-ApiMethod Not available-End", nil, true)
	}
}

func KafkaConsumer(ConsumeHandler func(msg *sarama.ConsumerMessage), topic string) {

	// Create a new configuration for the consumer
	config := sarama.NewConfig()
	config.Consumer.Return.Errors = true

	// Specify the list of brokers
	brokers := []string{KAFKA_CONN}

	// Create a new consumer
	consumer, err := sarama.NewConsumer(brokers, config)
	if err != nil {
		e.OkLog("Failed to create consumer of topic : " + topic + " == : " + err.Error())
	}
	defer func() {
		if err := consumer.Close(); err != nil {
			e.OkLog("Error closing consumer: of topic : " + topic + " == : " + err.Error())
		}
	}()

	// Consume messages from each partition asynchronously
	partitions, err := consumer.Partitions(topic)
	if err != nil {
		e.OkLog("Failed to get partitions: " + topic + " == : " + err.Error())
	}

	var wg sync.WaitGroup
	wg.Add(len(partitions))

	for _, partition := range partitions {
		go func(partition int32) {
			defer wg.Done()

			// Create a new partition consumer
			partitionConsumer, err := consumer.ConsumePartition(topic, partition, sarama.OffsetNewest)
			if err != nil {
				e.OkLog("Failed to create partition for consumer: " + topic + " partition: " + e.NumToStr(partition) + " " + err.Error())
				return
			}
			defer func() {
				if err := partitionConsumer.Close(); err != nil {
					e.OkLog("Error closing partition for consumer: " + topic + " partition: " + e.NumToStr(partition) + " " + err.Error())
				}
			}()

			// Process messages
			for msg := range partitionConsumer.Messages() {
				ConsumeHandler(msg)
				e.OkLog("Consuming topic: " + topic + " partition: " + e.NumToStr(partition))
				// log.Printf("Partition-kk %d | Offset %d | Key: %s | Value: %s", message.Partition, message.Offset, string(message.Key), string(message.Value))
			}
		}(partition)
	}
	// Wait for the consumer to finish
	wg.Wait()
	// }
}