mirror of
https://github.com/hcengineering/platform.git
synced 2026-09-01 04:38:50 +02:00
139 lines
3.6 KiB
Go
139 lines
3.6 KiB
Go
// Copyright © 2025 Hardcore Engineering Inc.
|
|
//
|
|
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License. You may
|
|
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
//
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package queue
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/hcengineering/stream/internal/pkg/config"
|
|
"github.com/hcengineering/stream/internal/pkg/log"
|
|
"github.com/hcengineering/stream/internal/pkg/mediaconvert"
|
|
"github.com/pkg/errors"
|
|
"github.com/segmentio/kafka-go"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
type Worker struct {
|
|
logger *zap.Logger
|
|
cfg *config.Config
|
|
consumer Consumer
|
|
}
|
|
|
|
func NewWorker(ctx context.Context, consumer Consumer, cfg *config.Config) *Worker {
|
|
logger := log.FromContext(ctx).With(zap.String("queue", "worker"))
|
|
|
|
return &Worker{
|
|
logger: logger,
|
|
cfg: cfg,
|
|
consumer: consumer,
|
|
}
|
|
}
|
|
|
|
func (w *Worker) Start(ctx context.Context) error {
|
|
w.logger.Info("starting worker")
|
|
defer w.logger.Info("worker stopped")
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
w.logger.Info("worker shutdown requested")
|
|
return ctx.Err()
|
|
default:
|
|
if err := w.fetchAndProcessMessage(ctx); err != nil {
|
|
if errors.Is(err, context.Canceled) {
|
|
return err
|
|
}
|
|
w.logger.Error("failed to process message", zap.Error(err))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (w *Worker) fetchAndProcessMessage(ctx context.Context) error {
|
|
// TODO We should use Fetch and Commit here but as long processing
|
|
// time is longer than the max poll interval, we have to use Read.
|
|
// msg, err := w.consumer.Fetch(ctx)
|
|
commit := false
|
|
msg, err := w.consumer.Read(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("read message: %w", err)
|
|
}
|
|
|
|
logger := w.logger.With(
|
|
zap.String("message-key", string(msg.Key)),
|
|
zap.Int64("message-offset", msg.Offset),
|
|
zap.Int("partition", msg.Partition),
|
|
)
|
|
logger.Debug("message", zap.ByteString("message", msg.Value))
|
|
|
|
_, err = w.processMessage(ctx, msg, logger)
|
|
if err != nil {
|
|
w.logger.Error("failed to process message", zap.Error(err))
|
|
}
|
|
|
|
if commit {
|
|
if err := w.commitMessage(ctx, msg, logger); err != nil {
|
|
return errors.Wrapf(err, "failed to commit message")
|
|
}
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func (w *Worker) commitMessage(ctx context.Context, msg kafka.Message, logger *zap.Logger) error {
|
|
maxRetries := 3
|
|
for i := range maxRetries {
|
|
err := w.consumer.Commit(ctx, msg)
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
|
|
if i < maxRetries-1 {
|
|
logger.Error("failed to commit message", zap.Error(err), zap.Int("attempt", i+1))
|
|
time.Sleep(time.Duration(100*(i+1)) * time.Millisecond)
|
|
} else {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (w *Worker) processMessage(ctx context.Context, msg kafka.Message, logger *zap.Logger) (bool, error) {
|
|
var req TranscodeRequest
|
|
if err := json.Unmarshal(msg.Value, &req); err != nil {
|
|
return true, fmt.Errorf("parse failed: %w", err)
|
|
}
|
|
|
|
if mediaconvert.IsSupportedMediaType(req.ContentType) == false {
|
|
logger.Debug("unsupported content type", zap.String("contentType", req.ContentType))
|
|
return true, nil
|
|
}
|
|
|
|
task := mediaconvert.Task{
|
|
ID: req.BlobID,
|
|
Workspace: req.WorkspaceUUID,
|
|
Source: req.BlobID,
|
|
}
|
|
|
|
transcoder := mediaconvert.NewTranscoder(ctx, w.cfg)
|
|
transcoder.Transcode(ctx, &task)
|
|
|
|
return true, nil
|
|
}
|