mirror of
https://github.com/hcengineering/platform.git
synced 2026-08-21 11:52:23 +02:00
+1








98652c6476
* Add bump-changes * Add utility tests * Add utility tests * Bump to new version of esbuild and typescript * v0.7.3 * use platform rig 0.7.10 * upgrade: memory engine optimized; change name to (was recommended by Copilot and Onnikov, TODO: CHANGE CLIENT TOO!!!) Signed-off-by: Leonid Kaganov <lleo@lleo.me> * Fix rate limits bug * Bump versions * Fix lock file * Fix bug in queue cleanup * Add more tests for queue * Add api-test tests * Initial commit * Improve hierarchy + tests Add tests for hierarchy and few performance/memory optimizations. * Add more hierarchy tests * Move from Huly platform repository * Add docker tests setup * Fix test to be executed only once * Add connection tests * Fix package include source files * More tests * Create README.md * Fix pnpm lock * Fix packages publish * Remove broken tests * feat: adjust hulylake client for storage adapter Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Fix export * Fix publish * Fix message update (#114) Signed-off-by: Kristina Fefelova <kristin.fefelova@gmail.com> * Bump version Signed-off-by: Kristina Fefelova <kristin.fefelova@gmail.com> * Bump versions Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Fix lang store (#115) Signed-off-by: Kristina Fefelova <kristin.fefelova@gmail.com> * update hulylake client Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Bump version Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Add hulylake storage adapter Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Bump version Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * fix validation issues Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * fix: do not fail on deseralization error and add logs Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * bump version -> 0.1.14 Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * fix collaboration test Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Update prettier and new update-deps script Prettier + svelte support * fix unstable ydoc tests Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Add tx ordering middleware * Fix ordering tests * Fix Kafka close of admin * Add tests for measurement and understand overhead * Fix not updated lock file * Fix update-deps * Fix update-deps * Use latest platform-rig * Fix deps * Add rush check to CI * Use latest versions * Bump versions * Fix lock file * validate json patch Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * bump version -> 0.1.15 Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * fix merge unit tests Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Script to sync eslint deps * Fix deps * Fix tests * Fix platform-rig detection * Update to latest platform-rig * Update to latest platform rig and core * Bump typescript * Bump typescript * Rollback eslint plugins * Fix lock file * Bump platform-rig * Update to latest platform-rig * update to latest platform-rig * Allow to compile svelte files * Add ui-test component for checking compile * Fix log levels rename compile ui -> compile ui-esbuild * Fix build * Bump esbuild svelte version * Chore: use fixed versions in update-deps Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com> * Chore: commit changes Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com> * Update deps * Add tests for session manager * Fix txOrdering implementation * Bump ordering * Prevent metrics zero values in measure + Fix format svelte files * Revert update-deps script logic * v0.7.19 * update to latest platform-rig * Update deps * Fix pnpm * Session counters * Fix pnpm lock * Add storage client Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Bump versions Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * fix versions Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Bump core * Fix pnpm * Get rid of communication dependency * Add copilot memory file * Use proper name for instructions file * Fix instructions * Use domain instead of test name in gauges * Update instructions file * fix front service upload Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * remove incorrect test Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Move packages to huly.core * Move packages to core, since they are not utils * Add global user profile Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com> * Fix lock file Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com> * Add support for memory limit check * Bump version * Fix pnpm * report more accurate upload progress Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * fic validation issues Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Fix deps Signed-off-by: Kristina Fefelova <kristin.fefelova@gmail.com> * Move LowLevelStorage to server * Fix linting * Revert "Fix linting" This reverts commit54631d353e. * Revert "Move LowLevelStorage to server" This reverts commitaafb8f6f12. * feature: add regorus engine with permit file Signed-off-by: Leonid Kaganov <lleo@lleo.me> * Fix one second counters for memory usage * Fix kafka test * use fresh core * Version bump * fix: key parameter added Signed-off-by: Leonid Kaganov <lleo@lleo.me> * Fix readme and few author mistakes * Export domain schemas * Bump version * Tests (#117) Signed-off-by: Kristina Fefelova <kristin.fefelova@gmail.com> * Bump version Signed-off-by: Kristina Fefelova <kristin.fefelova@gmail.com> * feat: compact compact worker (#4) Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * bump version -> 0.1.16 Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Add TypeIdentifier Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * Add change logs Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * rename send -> try_send Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Bump Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * Fix pnpm lock Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * Add identifier middleware, bump core Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * Add subsciption methods to account client Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com> * Fix lock file Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com> * Fix reaction notification (#118) Signed-off-by: Kristina Fefelova <kristin.fefelova@gmail.com> * Bump version Signed-off-by: Kristina Fefelova <kristin.fefelova@gmail.com> * Improve find methods schemas to convert to valid types Signed-off-by: Nikolay Marchuk <nikolay.marchuk@hardcoreeng.com> * Add change description Signed-off-by: Nikolay Marchuk <nikolay.marchuk@hardcoreeng.com> * Do not transcode while recording Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Open telemetry support Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * use proper content type in multipart upload Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Bump versions Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * fix build (#26) Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Fix peers (#120) Signed-off-by: Kristina Fefelova <kristin.fefelova@gmail.com> * Bump version Signed-off-by: Kristina Fefelova <kristin.fefelova@gmail.com> * Add ActivityCollaborativeChange Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * Update pnpm Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * Allow to suspend errors on with * Fix pnpm cache * update versions * v0.7.17 for all * v0.7.11 * v0.7.14 * remove arc from worker Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * bump version -> 0.1.17 Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Rank for attributes Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * Update pnpm Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * Fix one second counters * Fix withContext and allow pass options * Fix formatting * Use updated deps * Bump versions * Update deps * Update deps to platform.core * add support for textColor mark Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * add support for textStyle mark Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Bump versions Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Bump versions again Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Fix Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * Rework on second timers * fix merge of large blobs feched from s3 Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * bump version -> 0.1.18 Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * New subscription methods in account-client * Update lock file Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com> * Send error on find for wrong domain * Suspend connect custom errors events in traces * Bump client * Bump core * update deps * Fix lock file * Sorting for TypeIdentifier Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * Bump version Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * add workspace usage info Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Bump versions Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> * Improve pg security perfomance Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * Fix identifier middleware Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * Update TxAccessLevel interface Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com> * Allow guest to update its identities Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com> * Add password login locked platform status Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com> * Fix Uptrace normalizeMarkdown errors Signed-off-by: Artem Savchenko <armisav@gmail.com> * Add change log Signed-off-by: Artem Savchenko <armisav@gmail.com> * Add txMatch to permission Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * update pnpm lock Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * Bump Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * Fix permission middleware Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * Fix enum sorting Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * Enable formatting check Signed-off-by: Andrey Sobolev <haiodo@gmail.com> * Enable formatting check * Add change Signed-off-by: Andrey Sobolev <haiodo@gmail.com> * Fix Uptrace NaN error Signed-off-by: Artem Savchenko <armisav@gmail.com> * feature: removed actors, improved performance Signed-off-by: Leonid Kaganov <lleo@lleo.me> * feature: ping from server to clients added Signed-off-by: Leonid Kaganov <lleo@lleo.me> * feature: ping from server to clients added Signed-off-by: Leonid Kaganov <lleo@lleo.me> * Compress kafka messages and fix exception in findAll Signed-off-by: Artem Savchenko <armisav@gmail.com> * Bump versions Signed-off-by: Artem Savchenko <armisav@gmail.com> * Bump versions Signed-off-by: Artem Savchenko <armisav@gmail.com> * Rush change Signed-off-by: Artem Savchenko <armisav@gmail.com> * Fix compression param Signed-off-by: Artem Savchenko <armisav@gmail.com> * Trigger change Signed-off-by: Artem Savchenko <armisav@gmail.com> * Clean up Signed-off-by: Artem Savchenko <armisav@gmail.com> * Trigger change Signed-off-by: Artem Savchenko <armisav@gmail.com> * Bump markdown version Signed-off-by: Artem Savchenko <armisav@gmail.com> * Enable sub projects * Fix wrong double symbol scripts * Include foundation packages * Add support for custom exclude filters Add support for custom exclude filters - by Andrey Sobolev - haiodo@gmail.com Signed-off-by: Andrey Sobolev <haiodo@gmail.com> * Bump Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> * Fix Uptrace filter is not a function error Signed-off-by: Artem Savchenko <armisav@gmail.com> * Sync versions Signed-off-by: Andrey Sobolev <haiodo@gmail.com> --------- Signed-off-by: Leonid Kaganov <lleo@lleo.me> Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com> Signed-off-by: Kristina Fefelova <kristin.fefelova@gmail.com> Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com> Signed-off-by: Denis Bykhov <bykhov.denis@gmail.com> Signed-off-by: Nikolay Marchuk <nikolay.marchuk@hardcoreeng.com> Signed-off-by: Artem Savchenko <armisav@gmail.com> Signed-off-by: Andrey Sobolev <haiodo@gmail.com> Co-authored-by: Leonid Kaganov <lleo@lleo.me> Co-authored-by: Alexander Onnikov <Alexander.Onnikov@xored.com> Co-authored-by: Alexander Onnikov <Alexander.Onnikov@gmail.com> Co-authored-by: Kristina <kristin.fefelova@gmail.com> Co-authored-by: Alexey Zinoviev <alexey.zinoviev@xored.com> Co-authored-by: Denis Bykhov <bykhov.denis@gmail.com> Co-authored-by: Nikolay Marchuk <nikolay.marchuk@hardcoreeng.com> Co-authored-by: Alexander Onnikov <aonnikov@hardcoreeng.com> Co-authored-by: Artem Savchenko <armisav@gmail.com>
183 lines
5.2 KiB
Go
183 lines
5.2 KiB
Go
//
|
|
// Copyright © 2025 Hardcore Engineering Inc.
|
|
//
|
|
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License. You may
|
|
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
//
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
//
|
|
|
|
package mediaconvert
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/pkg/errors"
|
|
|
|
"github.com/hcengineering/stream/internal/pkg/log"
|
|
"github.com/hcengineering/stream/internal/pkg/manifest"
|
|
"github.com/hcengineering/stream/internal/pkg/profile"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
// LogLevel is ffmpeg log level
|
|
type LogLevel string
|
|
|
|
const (
|
|
// LogLevelQuiet is quiet log level
|
|
LogLevelQuiet LogLevel = "quiet"
|
|
// LogLevelPanic is panic log level
|
|
LogLevelPanic LogLevel = "panic"
|
|
// LogLevelFatal is fatal log level
|
|
LogLevelFatal LogLevel = "fatal"
|
|
// LogLevelError is error log level
|
|
LogLevelError LogLevel = "error"
|
|
// LogLevelWarning is warning log level
|
|
LogLevelWarning LogLevel = "warning"
|
|
// LogLevelInfo is info log level
|
|
LogLevelInfo LogLevel = "info"
|
|
// LogLevelVerbose is verbose log level
|
|
LogLevelVerbose LogLevel = "verbose"
|
|
// LogLevelDebug is debug log level
|
|
LogLevelDebug LogLevel = "debug"
|
|
// LogLevelTrace is trace log level
|
|
LogLevelTrace LogLevel = "trace"
|
|
)
|
|
|
|
// Options represents configuration for the ffmpeg command
|
|
type Options struct {
|
|
Input string
|
|
OutputDir string
|
|
LogLevel LogLevel
|
|
Threads int
|
|
UploadID string
|
|
Profiles []profile.VideoProfile
|
|
}
|
|
|
|
func newFfmpegCommand(ctx context.Context, in io.Reader, args []string) (*exec.Cmd, error) {
|
|
if ctx == nil {
|
|
return nil, errors.New("ctx should not be nil")
|
|
}
|
|
|
|
var cmd = exec.CommandContext(ctx, "ffmpeg", args...)
|
|
cmd.Stdin = in
|
|
|
|
var logger = log.FromContext(ctx).With(zap.String("func", "newFFMpegCommand"))
|
|
logger.Debug("prepared command: ", zap.String("cmd", cmd.String()))
|
|
|
|
return cmd, nil
|
|
}
|
|
|
|
func buildCommonCommand(opts *Options) []string {
|
|
var result = []string{
|
|
"-y", // Overwrite output files without asking.
|
|
"-v", string(opts.LogLevel),
|
|
"-err_detect", "ignore_err",
|
|
"-fflags", "+discardcorrupt",
|
|
"-threads", fmt.Sprint(opts.Threads),
|
|
"-i", opts.Input,
|
|
}
|
|
|
|
// If input is a URL, add HTTP specific parameters
|
|
if strings.HasPrefix(opts.Input, "http://") || strings.HasPrefix(opts.Input, "https://") {
|
|
result = append(result,
|
|
"-reconnect", "1",
|
|
"-reconnect_streamed", "1",
|
|
"-reconnect_delay_max", "5",
|
|
)
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
func buildHLSCommand(profile profile.VideoProfile, opts *Options) []string {
|
|
return []string{
|
|
"-f", "hls",
|
|
"-hls_time", "5",
|
|
// Use HLS flags
|
|
// - split_by_time
|
|
// Allow segments to start on frames other than key frames.
|
|
// This improves behavior on some players when the time between key frames is inconsistent,
|
|
// but may make things worse on others, and can cause some oddities during seeking.
|
|
// This flag should be used with the hls_time option.
|
|
// - temp_file
|
|
// Write segment data to filename.tmp and rename to filename only once the segment is complete.
|
|
"-hls_flags", "split_by_time+temp_file",
|
|
// Do not limit number of HLS segments
|
|
"-hls_list_size", "0",
|
|
"-hls_segment_filename", filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", profile.Name)),
|
|
}
|
|
}
|
|
|
|
func buildVideoCommand(profile profile.VideoProfile) []string {
|
|
crf := profile.CRF
|
|
if crf == 0 {
|
|
crf = 23
|
|
}
|
|
command := []string{
|
|
// Transcode only first video and optionally audio stream
|
|
"-map", "0:v:0",
|
|
"-map", "0:a?",
|
|
// Set up codecs
|
|
"-c:a", profile.AudioCodec,
|
|
"-c:v", profile.VideoCodec,
|
|
"-preset", "veryfast",
|
|
"-crf", strconv.Itoa(crf),
|
|
"-g", "60",
|
|
}
|
|
|
|
if profile.VideoCodec != "copy" && profile.Scale {
|
|
command = append(command, "-vf", "scale=-2:"+strconv.Itoa(profile.Height))
|
|
}
|
|
|
|
return command
|
|
}
|
|
|
|
// BuildAudioCommand returns flags for getting the audio from the input
|
|
func BuildAudioCommand(opts *Options) []string {
|
|
var commonPart = buildCommonCommand(opts)
|
|
|
|
return append(commonPart,
|
|
"-vn", "-acodec",
|
|
"copy", filepath.Join(opts.OutputDir, opts.UploadID),
|
|
)
|
|
}
|
|
|
|
// BuildVideoCommand returns ffmpeg command for converting video.
|
|
func BuildVideoCommand(opts *Options) []string {
|
|
if len(opts.Profiles) == 0 {
|
|
return []string{}
|
|
}
|
|
|
|
var command = buildCommonCommand(opts)
|
|
for _, profile := range opts.Profiles {
|
|
command = append(command, buildVideoCommand(profile)...)
|
|
command = append(command, buildHLSCommand(profile, opts)...)
|
|
command = append(command, filepath.Join(opts.OutputDir, opts.UploadID, manifest.PlaylistFileName(opts.UploadID, profile.Name)))
|
|
}
|
|
return command
|
|
}
|
|
|
|
// BuildThumbnailCommand creates a command that creates a thumbnail for the input video
|
|
func BuildThumbnailCommand(opts *Options) []string {
|
|
return append([]string{},
|
|
"-y", // Overwrite output files without asking.
|
|
"-i", opts.Input,
|
|
"-vframes", "1",
|
|
"-update", "1",
|
|
filepath.Join(opts.OutputDir, opts.UploadID, opts.UploadID+".jpg"),
|
|
)
|
|
}
|