huly-platform/foundations/stream/internal/pkg/queue/queue.go

// Copyright © 2025 Hardcore Engineering Inc.
//
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. You may
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.

// Package queue provides access to kafka queue
package queue

import (
	"context"
	"encoding/json"
	"fmt"
	"time"

	"github.com/hcengineering/stream/internal/pkg/log"
	"github.com/segmentio/kafka-go"
	"go.uber.org/zap"
)

// TopicTranscodeRequest is the name of the transcode requests topic.
const TopicTranscodeRequest = "stream.transcode.request"

// TopicTranscodeResult is the name of the transcode results topic.
const TopicTranscodeResult = "stream.transcode.result"

// TranscodeRequest represents transcode request.
type TranscodeRequest struct {
	BlobID        string
	WorkspaceUUID string
	ContentType   string
	Source        map[string]any
}

// TranscodeResult represents transcode result.
type TranscodeResult struct {
	BlobID        string         `json:"blobId"`
	WorkspaceUUID string         `json:"workspaceUuid"`
	Source        map[string]any `json:"source"`
	Playlist      string         `json:"playlist"`
	Thumbnail     string         `json:"thumbnail"`
}

// ConsumerOptions represents options for the consumer
type ConsumerOptions struct {
	Topic  string
	Group  string
	Config Config
}

// Consumer provides a consumer interface to a Kafka queue
type Consumer interface {
	Read(ctx context.Context) (kafka.Message, error)
	Fetch(ctx context.Context) (kafka.Message, error)
	Commit(ctx context.Context, msg kafka.Message) error
	Close() error
}

// ProducerOptions represents options for the producer
type ProducerOptions struct {
	Topic      string
	Group      string
	Config     Config
	RetryCount int
}

// Producer provides a producer interface to a Kafka queue
type Producer interface {
	Send(ctx context.Context, workspaceID string, data any) error
	Close() error
}

// TConsumer implements Consumer interface
type TConsumer struct {
	topic   string
	reader  *kafka.Reader
	options ConsumerOptions
}

// TProducer implements Producer interface
type TProducer struct {
	topic   string
	writer  *kafka.Writer
	options ProducerOptions
}

// Logger is kafka.Logger implementation
type Logger struct {
	logger *zap.Logger
}

var _ Producer = (*TProducer)(nil)
var _ Consumer = (*TConsumer)(nil)
var _ kafka.Logger = (*Logger)(nil)

// NewLogger creates new kafka.Logger
func NewLogger(ctx context.Context) kafka.Logger {
	return &Logger{
		logger: log.FromContext(ctx),
	}
}

// Printf logs a message
func (l *Logger) Printf(msg string, args ...any) {
	l.logger.Sugar().Debugf(msg, args...)
}

// NewConsumer creates a new transcoding request consumer
func NewConsumer(ctx context.Context, options ConsumerOptions) Consumer {
	platformTopic := makeTopicID(options.Topic, options.Config)
	groupID := makeGroupID(options.Group, options.Topic, options.Config)

	reader := kafka.NewReader(kafka.ReaderConfig{
		Brokers:           options.Config.Brokers,
		GroupID:           groupID,
		Topic:             platformTopic,
		Logger:            NewLogger(ctx),
		HeartbeatInterval: 1 * time.Second,
		SessionTimeout:    10 * time.Second,
	})

	return &TConsumer{
		options: options,
		topic:   platformTopic,
		reader:  reader,
	}
}

// Read reads (fetch and commit) a queue message.
func (c *TConsumer) Read(ctx context.Context) (kafka.Message, error) {
	return c.reader.ReadMessage(ctx)
}

// Fetch fetches a queue message.
func (c *TConsumer) Fetch(ctx context.Context) (kafka.Message, error) {
	return c.reader.FetchMessage(ctx)
}

// Commit commits the queue message.
func (c *TConsumer) Commit(ctx context.Context, msg kafka.Message) error {
	return c.reader.CommitMessages(ctx, msg)
}

// Close closes the consumer.
func (c *TConsumer) Close() error {
	return c.reader.Close()
}

// NewProducer creates a new transcoding result producer
func NewProducer(ctx context.Context, options ProducerOptions) Producer {
	platformTopic := makeTopicID(options.Topic, options.Config)
	writer := &kafka.Writer{
		Addr:                   kafka.TCP(options.Config.Brokers...),
		Topic:                  platformTopic,
		AllowAutoTopicCreation: true,
		Logger:                 NewLogger(ctx),
	}
	return &TProducer{
		options: options,
		topic:   platformTopic,
		writer:  writer,
	}
}

// Send sends a message to the queue topic
func (p *TProducer) Send(ctx context.Context, workspaceID string, data any) error {
	if data == nil {
		return fmt.Errorf("event data is empty")
	}

	value, err := json.Marshal(data)

	if err != nil {
		return err
	}

	for range p.options.RetryCount {
		err = p.writer.WriteMessages(
			ctx,
			kafka.Message{
				Key:   []byte(workspaceID),
				Value: value,
			},
		)
		if err == nil {
			break
		}
	}

	return err
}

// Close closes the producer.
func (p *TProducer) Close() error {
	return p.writer.Close()
}

func makeGroupID(groupID, topic string, config Config) string {
	return makeTopicID(topic, config) + "-" + groupID
}

func makeTopicID(topic string, config Config) string {
	if config.Region != "" {
		return config.Region + "." + topic + config.Postfix
	}
	return topic + config.Postfix
}