From af95e9a1da401717509a5e6f485e1bb0b44f8866 Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Thu, 30 Jan 2025 01:41:38 +0300 Subject: [PATCH 01/52] init Signed-off-by: denis-tingaikin --- README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000000..1b7f8dd774 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# huly-stream From de97f778f9f39e163cf455394d1596a1c28c773d Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Tue, 4 Feb 2025 04:54:44 +0300 Subject: [PATCH 02/52] add initial version of huly-stream Signed-off-by: denis-tingaikin --- .DS_Store | Bin 0 -> 6148 bytes .github/workflows/docker-push.yaml | 44 +++ .github/workflows/main.yaml | 73 +++++ .github/yamllint.yaml | 12 + .golangci.yaml | 160 ++++++++++ Dockerfile | 36 +++ LICENSE | 277 ++++++++++++++++++ README.md | 122 +++++++- cmd/huly-stream/main.go | 104 +++++++ go.mod | 44 +++ go.sum | 84 ++++++ internal/pkg/config/config.go | 51 ++++ internal/pkg/log/zap.go | 50 ++++ internal/pkg/manifest/hls.go | 125 ++++++++ internal/pkg/manifest/hls_test.go | 143 +++++++++ internal/pkg/pprof/pprof.go | 54 ++++ internal/pkg/sharedpipe/shared_pipe.go | 116 ++++++++ .../pkg/sharedpipe/shared_pipe_bench_test.go | 248 ++++++++++++++++ internal/pkg/transcoding/command.go | 163 +++++++++++ internal/pkg/transcoding/command_test.go | 62 ++++ internal/pkg/transcoding/limiter.go | 78 +++++ internal/pkg/transcoding/limiter_test.go | 88 ++++++ internal/pkg/transcoding/scheduler.go | 129 ++++++++ internal/pkg/transcoding/worker.go | 125 ++++++++ internal/pkg/uploader/datalake.go | 124 ++++++++ internal/pkg/uploader/options.go | 19 ++ internal/pkg/uploader/postpone.go | 40 +++ internal/pkg/uploader/postpone_test.go | 44 +++ internal/pkg/uploader/s3.go | 143 +++++++++ internal/pkg/uploader/uploader.go | 219 ++++++++++++++ 30 files changed, 2976 insertions(+), 1 deletion(-) create mode 100644 .DS_Store create mode 100644 .github/workflows/docker-push.yaml create mode 100644 .github/workflows/main.yaml create mode 100644 .github/yamllint.yaml create mode 100644 .golangci.yaml create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 cmd/huly-stream/main.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/pkg/config/config.go create mode 100644 internal/pkg/log/zap.go create mode 100644 internal/pkg/manifest/hls.go create mode 100644 internal/pkg/manifest/hls_test.go create mode 100644 internal/pkg/pprof/pprof.go create mode 100644 internal/pkg/sharedpipe/shared_pipe.go create mode 100644 internal/pkg/sharedpipe/shared_pipe_bench_test.go create mode 100644 internal/pkg/transcoding/command.go create mode 100644 internal/pkg/transcoding/command_test.go create mode 100644 internal/pkg/transcoding/limiter.go create mode 100644 internal/pkg/transcoding/limiter_test.go create mode 100644 internal/pkg/transcoding/scheduler.go create mode 100644 internal/pkg/transcoding/worker.go create mode 100644 internal/pkg/uploader/datalake.go create mode 100644 internal/pkg/uploader/options.go create mode 100644 internal/pkg/uploader/postpone.go create mode 100644 internal/pkg/uploader/postpone_test.go create mode 100644 internal/pkg/uploader/s3.go create mode 100644 internal/pkg/uploader/uploader.go diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..0923990b38bec54c6cb023fcb61ce6be5c8afe41 GIT binary patch literal 6148 zcmeHKyG{c^3>-s>NNG}1?k4~Z?J7#XARhn)5)DcsM5wRgyZE$>A3{1^C{m#)|UB-M9`4L$qUJ iv}10(9p6P!)-_-AycZ6ML1#YbMEwl7E;1=_Z3RxGZ4~VQ literal 0 HcmV?d00001 diff --git a/.github/workflows/docker-push.yaml b/.github/workflows/docker-push.yaml new file mode 100644 index 0000000000..0308ce654b --- /dev/null +++ b/.github/workflows/docker-push.yaml @@ -0,0 +1,44 @@ +--- +name: Docker push +on: + workflow_dispatch: + inputs: + version: + description: 'Version tag for the image' + required: false + default: 'latest' + type: string + +jobs: + push: + runs-on: ubuntu-latest + steps: + - name: "Checkout" + uses: actions/checkout@v4 + + - name: "Set up Docker Buildx" + uses: docker/setup-buildx-action@v1 + + - name: "Login to GitHub Container Registry" + uses: docker/login-action@v1 + with: + username: hardcoreeng + password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + + - name: Docker meta + id: metaci + uses: docker/metadata-action@v3 + with: + images: hardcoreeng/huly-stream + tags: | + type=ref,event=pr + type=sha,prefix= + + - name: "Build and push" + uses: docker/build-push-action@v2 + with: + file: Dockerfile + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ inputs.version }} diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 0000000000..a288973049 --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,73 @@ +--- +name: ci + +on: + push: + branches: + - main + - develop + pull_request: + +concurrency: + group: 'main' + cancel-in-progress: true + +jobs: + yamllint: + name: yamllint + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: yaml-lint + uses: ibiqlik/action-yamllint@v1 + with: + config_file: .github/yamllint.yaml + strict: true + build-and-test: + strategy: + matrix: + os: + - ubuntu + - windows + - macos + runs-on: ${{ matrix.os }}-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: 1.23.5 + - name: Build + run: go build -race ./... + - name: Test + run: go test -race ./... + checkgomod: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: 1.23.5 + - run: go mod tidy + - name: Check for changes in go.mod or go.sum + run: | + git diff --name-only --exit-code go.mod || ( echo "Run go mod tidy" && false ) + golangci-lint: + name: golangci-lint + runs-on: ubuntu-latest + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: 1.23.5 + - name: golangci-lint + uses: golangci/golangci-lint-action@v4 + with: + version: v1.60.3 + args: --timeout 3m --verbose diff --git a/.github/yamllint.yaml b/.github/yamllint.yaml new file mode 100644 index 0000000000..7c94f8d8fe --- /dev/null +++ b/.github/yamllint.yaml @@ -0,0 +1,12 @@ +--- +extends: default + +yaml-files: + - '*.yaml' + - '*.yml' + +rules: + truthy: disable + line-length: disable + comments: + min-spaces-from-content: 1 diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 0000000000..0099487e6f --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,160 @@ +--- +run: + go: "1.23" + timeout: 2m + issues-exit-code: 1 + tests: true +linters-settings: + goheader: + template: |- + Copyright © {{ mod-year-range }} Hardcore Engineering Inc. + + Licensed under the Eclipse Public License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. You may + obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + See the License for the specific language governing permissions and + limitations under the License. + errcheck: + check-type-assertions: false + check-blank: false + govet: + enable: + - shadow + settings: + printf: + funcs: + - (github.com/sirupsen/logrus.FieldLogger).Infof + - (github.com/sirupsen/logrus.FieldLogger).Warnf + - (github.com/sirupsen/logrus.FieldLogger).Errorf + - (github.com/sirupsen/logrus.FieldLogger).Fatalf + revive: + confidence: 0.8 + rules: + - name: exported + - name: blank-imports + - name: context-as-argument + - name: context-keys-type + - name: dot-imports + - name: error-return + - name: error-strings + - name: error-naming + - name: exported + - name: increment-decrement + - name: package-comments + - name: range + - name: receiver-naming + - name: time-naming + - name: unexported-return + - name: indent-error-flow + - name: errorf + - name: superfluous-else + - name: unreachable-code + goimports: + local-prefixes: github.com/networkservicemesh/sdk + gocyclo: + min-complexity: 15 + dupl: + threshold: 150 + funlen: + lines: 120 + statements: 60 + goconst: + min-len: 2 + min-occurrences: 2 + depguard: + rules: + main: + deny: + - pkg: "errors" + desc: "Please use \"github.com/pkg/errors\" instead of \"errors\" in go imports" + misspell: + locale: US + unparam: + check-exported: false + nakedret: + max-func-lines: 30 + prealloc: + simple: true + range-loops: true + for-loops: false + gosec: + excludes: + - G115 + - G204 + - G301 + - G302 + - G306 + gocritic: + enabled-checks: + - appendCombine + - boolExprSimplify + - builtinShadow + - commentedOutCode + - commentedOutImport + - docStub + - dupImport + - emptyFallthrough + - emptyStringTest + - equalFold + - evalOrder + - hexLiteral + - importShadow + - indexAlloc + - initClause + - methodExprCall + - nestingReduce + - nilValReturn + - octalLiteral + - paramTypeCombine + - rangeExprCopy + - rangeValCopy + - regexpPattern + - sloppyReassign + - stringXbytes + - typeAssertChain + - typeUnparen + - unlabelStmt + - unnamedResult + - unnecessaryBlock + - weakCond + - yodaStyleExpr +linters: + disable-all: true + enable: + - goheader + - bodyclose + - unused + - depguard + - dogsled + - dupl + - errcheck + - funlen + - gochecknoinits + - goconst + - gocritic + - gocyclo + - gofmt + - goimports + - revive + - gosec + - gosimple + - govet + - ineffassign + - misspell + - nakedret + - copyloopvar + - staticcheck + - stylecheck + - typecheck + - unconvert + - unparam + - whitespace +issues: + exclude-use-default: false + max-issues-per-linter: 0 + max-same-issues: 0 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..0b4dca5d82 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +# Copyright © 2025 Hardcore Engineering Inc. +# +# Licensed under the Eclipse Public License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. You may +# obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM --platform=$BUILDPLATFORM golang:1.23.5 AS builder +ENV GO111MODULE=on +ENV CGO_ENABLED=0 +ENV GOBIN=/bin +ARG BUILDARCH=amd64 + +COPY . ./ + +RUN set -xe && GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /go/bin/huly-stream ./cmd/huly-stream + +FROM alpine + +RUN set -xe && apk add --no-cache ffmpeg +RUN apk add --no-cache ca-certificates jq bash \ + && addgroup -g 1000 huly-stream \ + && adduser -u 1000 -G huly-stream -s /bin/sh -D huly-stream \ + && chown huly-stream:huly-stream /. +COPY --from=builder /go/bin/huly-stream /huly-stream + +EXPOSE 1080 +USER huly-stream + +ENTRYPOINT ["/huly-stream"] \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..e48e096345 --- /dev/null +++ b/LICENSE @@ -0,0 +1,277 @@ +Eclipse Public License - v 2.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + + a) in the case of the initial Contributor, the initial content + Distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + i) changes to the Program, and + ii) additions to the Program; + where such changes and/or additions to the Program originate from + and are Distributed by that particular Contributor. A Contribution + "originates" from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include changes or additions to the Program that + are not Modified Works. + +"Contributor" means any person or entity that Distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which +are necessarily infringed by the use or sale of its Contribution alone +or when combined with the Program. + +"Program" means the Contributions Distributed in accordance with this +Agreement. + +"Recipient" means anyone who receives the Program under this Agreement +or any Secondary License (as applicable), including Contributors. + +"Derivative Works" shall mean any work, whether in Source Code or other +form, that is based on (or derived from) the Program and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. + +"Modified Works" shall mean any work in Source Code or other form that +results from an addition to, deletion from, or modification of the +contents of the Program, including, for purposes of clarity any new file +in Source Code form that contains any contents of the Program. Modified +Works shall not include works that contain only declarations, +interfaces, types, classes, structures, or files of the Program solely +in each case in order to link to, bind by name, or subclass the Program +or Modified Works thereof. + +"Distribute" means the acts of a) distributing or b) making available +in any manner that enables the transfer of a copy. + +"Source Code" means the form of a Program preferred for making +modifications, including but not limited to software source code, +documentation source, and configuration files. + +"Secondary License" means either the GNU General Public License, +Version 2.0, or any later versions of that license, including any +exceptions or additional permissions as identified by the initial +Contributor. + +2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free copyright + license to reproduce, prepare Derivative Works of, publicly display, + publicly perform, Distribute and sublicense the Contribution of such + Contributor, if any, and such Derivative Works. + + b) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free patent + license under Licensed Patents to make, use, sell, offer to sell, + import and otherwise transfer the Contribution of such Contributor, + if any, in Source Code or other form. This patent license shall + apply to the combination of the Contribution and the Program if, at + the time the Contribution is added by the Contributor, such addition + of the Contribution causes such combination to be covered by the + Licensed Patents. The patent license shall not apply to any other + combinations which include the Contribution. No hardware per se is + licensed hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the + rights and licenses granted hereunder, each Recipient hereby + assumes sole responsibility to secure any other intellectual + property rights needed, if any. For example, if a third party + patent license is required to allow Recipient to Distribute the + Program, it is Recipient's responsibility to acquire that license + before distributing the Program. + + d) Each Contributor represents that to its knowledge it has + sufficient copyright rights in its Contribution, if any, to grant + the copyright license set forth in this Agreement. + + e) Notwithstanding the terms of any Secondary License, no + Contributor makes additional grants to any Recipient (other than + those set forth in this Agreement) as a result of such Recipient's + receipt of the Program under the terms of a Secondary License + (if permitted under the terms of Section 3). + +3. REQUIREMENTS + +3.1 If a Contributor Distributes the Program in any form, then: + + a) the Program must also be made available as Source Code, in + accordance with section 3.2, and the Contributor must accompany + the Program with a statement that the Source Code for the Program + is available under this Agreement, and informs Recipients how to + obtain it in a reasonable manner on or through a medium customarily + used for software exchange; and + + b) the Contributor may Distribute the Program under a license + different than this Agreement, provided that such license: + i) effectively disclaims on behalf of all other Contributors all + warranties and conditions, express and implied, including + warranties or conditions of title and non-infringement, and + implied warranties or conditions of merchantability and fitness + for a particular purpose; + + ii) effectively excludes on behalf of all other Contributors all + liability for damages, including direct, indirect, special, + incidental and consequential damages, such as lost profits; + + iii) does not attempt to limit or alter the recipients' rights + in the Source Code under section 3.2; and + + iv) requires any subsequent distribution of the Program by any + party to be under a license that satisfies the requirements + of this section 3. + +3.2 When the Program is Distributed as Source Code: + + a) it must be made available under this Agreement, or if the + Program (i) is combined with other material in a separate file or + files made available under a Secondary License, and (ii) the initial + Contributor attached to the Source Code the notice described in + Exhibit A of this Agreement, then the Program may be made available + under the terms of such Secondary Licenses, and + + b) a copy of this Agreement must be included with each copy of + the Program. + +3.3 Contributors may not remove or alter any copyright, patent, +trademark, attribution notices, disclaimers of warranty, or limitations +of liability ("notices") contained within the Program from any copy of +the Program which they Distribute, provided that Contributors may add +their own appropriate notices. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities +with respect to end users, business partners and the like. While this +license is intended to facilitate the commercial use of the Program, +the Contributor who includes the Program in a commercial product +offering should do so in a manner which does not create potential +liability for other Contributors. Therefore, if a Contributor includes +the Program in a commercial product offering, such Contributor +("Commercial Contributor") hereby agrees to defend and indemnify every +other Contributor ("Indemnified Contributor") against any losses, +damages and costs (collectively "Losses") arising from claims, lawsuits +and other legal actions brought by a third party against the Indemnified +Contributor to the extent caused by the acts or omissions of such +Commercial Contributor in connection with its distribution of the Program +in a commercial product offering. The obligations in this section do not +apply to any claims or Losses relating to any actual or alleged +intellectual property infringement. In order to qualify, an Indemnified +Contributor must: a) promptly notify the Commercial Contributor in +writing of such claim, and b) allow the Commercial Contributor to control, +and cooperate with the Commercial Contributor in, the defense and any +related settlement negotiations. The Indemnified Contributor may +participate in any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial +product offering, Product X. That Contributor is then a Commercial +Contributor. If that Commercial Contributor then makes performance +claims, or offers warranties related to Product X, those performance +claims and warranties are such Commercial Contributor's responsibility +alone. Under this section, the Commercial Contributor would have to +defend claims against the other Contributors related to those performance +claims and warranties, and if a court requires any other Contributor to +pay any damages as a result, the Commercial Contributor must pay +those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN "AS IS" +BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF +TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR +PURPOSE. Each Recipient is solely responsible for determining the +appropriateness of using and distributing the Program and assumes all +risks associated with its exercise of rights under this Agreement, +including but not limited to the risks and costs of program errors, +compliance with applicable laws, damage to or loss of data, programs +or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS +SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST +PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE +EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under +applicable law, it shall not affect the validity or enforceability of +the remainder of the terms of this Agreement, and without further +action by the parties hereto, such provision shall be reformed to the +minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity +(including a cross-claim or counterclaim in a lawsuit) alleging that the +Program itself (excluding combinations of the Program with other software +or hardware) infringes such Recipient's patent(s), then such Recipient's +rights granted under Section 2(b) shall terminate as of the date such +litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it +fails to comply with any of the material terms or conditions of this +Agreement and does not cure such failure in a reasonable period of +time after becoming aware of such noncompliance. If all Recipient's +rights under this Agreement terminate, Recipient agrees to cease use +and distribution of the Program as soon as reasonably practicable. +However, Recipient's obligations under this Agreement and any licenses +granted by Recipient relating to the Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, +but in order to avoid inconsistency the Agreement is copyrighted and +may only be modified in the following manner. The Agreement Steward +reserves the right to publish new versions (including revisions) of +this Agreement from time to time. No one other than the Agreement +Steward has the right to modify this Agreement. The Eclipse Foundation +is the initial Agreement Steward. The Eclipse Foundation may assign the +responsibility to serve as the Agreement Steward to a suitable separate +entity. Each new version of the Agreement will be given a distinguishing +version number. The Program (including Contributions) may always be +Distributed subject to the version of the Agreement under which it was +received. In addition, after a new version of the Agreement is published, +Contributor may elect to Distribute the Program (including its +Contributions) under the new version. + +Except as expressly stated in Sections 2(a) and 2(b) above, Recipient +receives no rights or licenses to the intellectual property of any +Contributor under this Agreement, whether expressly, by implication, +estoppel or otherwise. All rights in the Program not expressly granted +under this Agreement are reserved. Nothing in this Agreement is intended +to be enforceable by any entity that is not a Contributor or Recipient. +No third-party beneficiary rights are created under this Agreement. + +Exhibit A - Form of Secondary Licenses Notice + +"This Source Code may also be made available under the following +Secondary Licenses when the conditions for such availability set forth +in the Eclipse Public License, v. 2.0 are satisfied: {name license(s), +version(s), and exceptions or additional permissions here}." + + Simply including a copy of this Agreement, including this Exhibit A + is not sufficient to license the Source Code under Secondary Licenses. + + If it is not possible or desirable to put the notice in a particular + file, then You may include the notice in a location (such as a LICENSE + file in a relevant directory) where a recipient would be likely to + look for such a notice. + + You may add additional accurate notices of copyright ownership. diff --git a/README.md b/README.md index 1b7f8dd774..8eb361d442 100644 --- a/README.md +++ b/README.md @@ -1 +1,121 @@ -# huly-stream +# Huly Stream + +[![X (formerly Twitter) Follow](https://img.shields.io/twitter/follow/huly_io?style=for-the-badge)](https://x.com/huly_io) +![GitHub License](https://img.shields.io/github/license/hcengineering/platform?style=for-the-badge) + +## About + +The Huly Stream high-performance HTTP-based transcoding service. Huly-stream is built around the **TUS protocol**, enabling reliable, resumable file uploads and downloads. Designed for seamless and consistent media processing,it supports advanced transcoding features with robust integration options. + +--- + +## Features + +### TUS Protocol Support +- **Resumable transcoding**: Leveraging the TUS protocol, Huly-stream ensures reliable and efficient stream processing. + +### Input Support +- **Supported Input Formats**: + - `mp4` + - `webm` + +### Output Options +- **TUS Upload**: Resumable file uploads via TUS protocol. +- **s3 Upload**: Direct upload to Amazon S3. +- **datalake Upload**: Integration for data lake storage systems. +- **Supported Output Formats**: + - `aac` + - `hls` + +### Key Functionalities +- **Live transcoing with minimal upload time**: Transcoding results are going to be avaible after stream completion. +- **Transcoding Cancelation**: Cancel or pause ongoing transcoding in real-time. +- **Transcoding Resumption**: Resume incomplete transcoding tasks efficiently. + +--- + +## Installation + +### Prerequisites +- [Go](https://golang.org/dl/) (v1.23+ recommended) +- [ffmpeg](https://www.ffmpeg.org/download.html) (ensure it’s installed and available in your system's PATH) + +### Steps + +1. Install dependencies: + ```bash + go mod tidy + ``` + +2. Build the service: + ```bash + docker build . -t hcengineering/huly-stream:latest + ``` + + +--- + +## Configuraiton + +### App env configuraiton +The following environment variables can be used: +``` +KEY TYPE DEFAULT REQUIRED DESCRIPTION +STREAM_SECRET_TOKEN String secret token for authorize requests +STREAM_LOG_LEVEL String debug sets log level for the application +STREAM_PPROF_ENABLED True or False false starts profile server on localhost:6060 if true +STREAM_INSECURE True or False false ignores authorization check if true +STREAM_SERVE_URL String 0.0.0.0:1080 app listen url +STREAM_ENDPOINT_URL URL S3 or Datalake endpoint, example: s3://my-ip-address, datalake://my-ip-address +STREAM_MAX_CAPACITY Integer 6220800 represents the amount of maximum possible capacity for the transcoding. The default value is 1920 * 1080 * 3. +STREAM_MAX_THREADS Integer 4 means upper bound for the transcoing provider. +STREAM_OUTPUT_DIR String /tmp/transcoing/ path to the directory with transcoding result. +STREAM_REMOVE_CONTENT_ON_UPLOAD True or False true deletes all content when content delivered if true +STREAM_UPLOAD_RAW_CONTENT True or False false uploads content in raw quality to the endpoint if true +``` + +### Metadata: + +**resolutions** if passed, set the resolution for the output, for example, 'resolutions: 1920:1080, 1280:720.' + +**token** must be provided to be authorized in the Huly's datalake service. + +**workspace** required for uploading content. + + + +#### S3 Env configuration + +if you're working with S3 storage type, these envs must be provided: +**AWS_ACCESS_KEY_ID** +**AWS_SECRET_ACCESS_KEY** + +## Usage + +The service exposes an HTTP API. Below are some examples of how to interact with it. + +### Upload a File for Transcoding via TUS +```bash +curl -X POST http://localhost:1080/transcoing \ + -H "Tus-Resumable: 1.0.0" \ + -H "Upload-Length: " \ + --data-binary @path/to/your/file.mp4 +``` + +## Contributing + +We welcome contributions! To get started: + +1. Fork the repository. +2. Create a new branch for your feature or bug fix. +3. Submit a pull request describing your changes. + +--- + +## License + +This project is licensed under the [MIT License](LICENSE). + +--- + +Enjoy seamless transcoding with huly-stream! 🚀 \ No newline at end of file diff --git a/cmd/huly-stream/main.go b/cmd/huly-stream/main.go new file mode 100644 index 0000000000..2730cd8b35 --- /dev/null +++ b/cmd/huly-stream/main.go @@ -0,0 +1,104 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main provides huly-stream entry point function +package main + +import ( + "context" + "net/http" + + "os" + "os/signal" + "syscall" + + "go.uber.org/zap" + "golang.org/x/exp/slog" + + "github.com/huly-stream/internal/pkg/config" + "github.com/huly-stream/internal/pkg/log" + "github.com/huly-stream/internal/pkg/pprof" + "github.com/huly-stream/internal/pkg/transcoding" + tusd "github.com/tus/tusd/v2/pkg/handler" +) + +const basePath = "/transcoding" + +func main() { + var ctx, cancel = signal.NotifyContext( + context.Background(), + os.Interrupt, + syscall.SIGHUP, + syscall.SIGTERM, + syscall.SIGQUIT, + ) + defer cancel() + + ctx = log.WithLoggerFields(ctx) + + var logger = log.FromContext(ctx) + var conf = must(config.FromEnv()) + + logger.Sugar().Debugf("provided config is %v", conf) + + mustNoError(os.MkdirAll(conf.OutputDir, os.ModePerm)) + + if conf.PprofEnabled { + go pprof.ListenAndServe(ctx, "localhost:6060") + } + + scheduler := transcoding.NewScheduler(ctx, conf) + + tusComposer := tusd.NewStoreComposer() + tusComposer.UseCore(scheduler) + tusComposer.UseTerminater(scheduler) + tusComposer.UseConcater(scheduler) + tusComposer.UseLengthDeferrer(scheduler) + + var handler = must(tusd.NewHandler(tusd.Config{ + BasePath: basePath, + StoreComposer: tusComposer, + Logger: slog.New(slog.NewTextHandler(discardTextHandler{}, nil)), + })) + + http.Handle("/transcoding/", http.StripPrefix("/transcoding/", handler)) + http.Handle("/transcoding", http.StripPrefix("/transcoding", handler)) + + go func() { + // #nosec + var err = http.ListenAndServe(conf.ServeURL, nil) + if err != nil { + cancel() + logger.Debug("unable to listen", zap.Error(err)) + } + }() + + <-ctx.Done() +} + +type discardTextHandler struct{} + +func (discardTextHandler) Write([]byte) (int, error) { + return 0, nil +} + +func mustNoError(err error) { + if err != nil { + panic(err.Error()) + } +} + +func must[T any](val T, err error) T { + mustNoError(err) + return val +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000000..89419cb728 --- /dev/null +++ b/go.mod @@ -0,0 +1,44 @@ +module github.com/huly-stream + +go 1.23.2 + +require ( + github.com/aws/aws-sdk-go-v2 v1.32.3 + github.com/aws/aws-sdk-go-v2/config v1.28.1 + github.com/aws/aws-sdk-go-v2/credentials v1.17.42 + github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2 + github.com/aws/smithy-go v1.22.0 + github.com/fsnotify/fsnotify v1.8.0 + github.com/google/uuid v1.6.0 + github.com/kelseyhightower/envconfig v1.4.0 + github.com/pkg/errors v0.9.1 + github.com/stretchr/testify v1.9.0 + github.com/tus/tusd/v2 v2.6.0 + github.com/valyala/fasthttp v1.58.0 + go.uber.org/zap v1.27.0 + golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df +) + +require ( + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/sys v0.27.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000000..11e9c85d56 --- /dev/null +++ b/go.sum @@ -0,0 +1,84 @@ +github.com/Acconut/go-httptest-recorder v1.0.0 h1:TAv2dfnqp/l+SUvIaMAUK4GeN4+wqb6KZsFFFTGhoJg= +github.com/Acconut/go-httptest-recorder v1.0.0/go.mod h1:CwQyhTH1kq/gLyWiRieo7c0uokpu3PXeyF/nZjUNtmM= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/aws/aws-sdk-go-v2 v1.32.3 h1:T0dRlFBKcdaUPGNtkBSwHZxrtis8CQU17UpNBZYd0wk= +github.com/aws/aws-sdk-go-v2 v1.32.3/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= +github.com/aws/aws-sdk-go-v2/config v1.28.1 h1:oxIvOUXy8x0U3fR//0eq+RdCKimWI900+SV+10xsCBw= +github.com/aws/aws-sdk-go-v2/config v1.28.1/go.mod h1:bRQcttQJiARbd5JZxw6wG0yIK3eLeSCPdg6uqmmlIiI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.42 h1:sBP0RPjBU4neGpIYyx8mkU2QqLPl5u9cmdTWVzIpHkM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.42/go.mod h1:FwZBfU530dJ26rv9saAbxa9Ej3eF/AK0OAY86k13n4M= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 h1:68jFVtt3NulEzojFesM/WVarlFpCaXLKaBxDpzkQ9OQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18/go.mod h1:Fjnn5jQVIo6VyedMc0/EhPpfNlPl7dHV916O6B+49aE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 h1:Jw50LwEkVjuVzE1NzkhNKkBf9cRN7MtE1F/b2cOKTUM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22/go.mod h1:Y/SmAyPcOTmpeVaWSzSKiILfXTVJwrGmYZhcRbhWuEY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 h1:981MHwBaRZM7+9QSR6XamDzF/o7ouUGxFzr+nVSIhrs= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22/go.mod h1:1RA1+aBEfn+CAB/Mh0MB6LsdCYCnjZm7tKXtnk499ZQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 h1:yV+hCAHZZYJQcwAaszoBNwLbPItHvApxT0kVIw6jRgs= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22/go.mod h1:kbR1TL8llqB1eGnVbybcA4/wgScxdylOdyAd51yxPdw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 h1:kT6BcZsmMtNkP/iYMcRG+mIEA/IbeiUimXtGmqF39y0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3/go.mod h1:Z8uGua2k4PPaGOYn66pK02rhMrot3Xk3tpBuUFPomZU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 h1:qcxX0JYlgWH3hpPUnd6U0ikcl6LLA9sLkXE2w1fpMvY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3/go.mod h1:cLSNEmI45soc+Ef8K/L+8sEA3A3pYFEYf5B5UI+6bH4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 h1:ZC7Y/XgKUxwqcdhO5LE8P6oGP1eh6xlQReWNKfhvJno= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3/go.mod h1:WqfO7M9l9yUAw0HcHaikwRd/H6gzYdz7vjejCA5e2oY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2 h1:p9TNFL8bFUMd+38YIpTAXpoxyz0MxC7FlbFEH4P4E1U= +github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2/go.mod h1:fNjyo0Coen9QTwQLWeV6WO2Nytwiu+cCcWaTdKCAqqE= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 h1:UTpsIf0loCIWEbrqdLb+0RxnTXfWh2vhw4nQmFi4nPc= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.3/go.mod h1:FZ9j3PFHHAR+w0BSEjK955w5YD2UwB/l/H0yAK3MJvI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 h1:2YCmIXv3tmiItw0LlYf6v7gEHebLY45kBEnPezbUKyU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3/go.mod h1:u19stRyNPxGhj6dRm+Cdgu6N75qnbW7+QN0q0dsAk58= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 h1:wVnQ6tigGsRqSWDEEyH6lSAJ9OyFUsSnbaUWChuSGzs= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.3/go.mod h1:VZa9yTFyj4o10YGsmDO4gbQJUvvhY72fhumT8W4LqsE= +github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= +github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tus/tusd/v2 v2.6.0 h1:Je243QDKnFTvm/WkLH2bd1oQ+7trolrflRWyuI0PdWI= +github.com/tus/tusd/v2 v2.6.0/go.mod h1:1Eb1lBoSRBfYJ/mQfFVjyw8ZdNMdBqW17vgQKl3Ah9g= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.58.0 h1:GGB2dWxSbEprU9j0iMJHgdKYJVDyjrOwF9RE59PbRuE= +github.com/valyala/fasthttp v1.58.0/go.mod h1:SYXvHHaFp7QZHGKSHmoMipInhrI5StHrhDTYVEjK/Kw= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= +golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go new file mode 100644 index 0000000000..1ef90f2192 --- /dev/null +++ b/internal/pkg/config/config.go @@ -0,0 +1,51 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package config provides configuration for the application +package config + +import ( + "net/url" + + "github.com/kelseyhightower/envconfig" +) + +// Config represents configuration for the huly-stream application. +type Config struct { + SecretToken string `split_words:"true" desc:"secret token for authorize requests"` + LogLevel string `split_words:"true" default:"debug" desc:"sets log level for the application"` + PprofEnabled bool `default:"false" split_words:"true" desc:"starts profile server on localhost:6060 if true"` + Insecure bool `default:"false" desc:"ignores authorization check if true"` + ServeURL string `split_words:"true" desc:"app listen url" default:"0.0.0.0:1080"` + EndpointURL *url.URL `split_words:"true" desc:"S3 or Datalake endpoint, example: s3://my-ip-address, datalake://my-ip-address"` + MaxCapacity int64 `split_words:"true" default:"6220800" desc:"represents the amount of maximum possible capacity for the transcoding. The default value is 1920 * 1080 * 3."` + MaxThreads int `split_words:"true" default:"4" desc:"means upper bound for the transcoing provider."` + OutputDir string `split_words:"true" default:"/tmp/transcoing/" desc:"path to the directory with transcoding result."` + RemoveContentOnUpload bool `split_words:"true" default:"true" desc:"deletes all content when content delivered if true"` + UploadRawContent bool `split_words:"true" default:"false" desc:"uploads content in raw quality to the endpoint if true"` +} + +// FromEnv creates new Config from env +func FromEnv() (*Config, error) { + var result Config + + if err := envconfig.Usage("stream", &result); err != nil { + return nil, err + } + + if err := envconfig.Process("stream", &result); err != nil { + return nil, err + } + + return &result, nil +} diff --git a/internal/pkg/log/zap.go b/internal/pkg/log/zap.go new file mode 100644 index 0000000000..72e03ac379 --- /dev/null +++ b/internal/pkg/log/zap.go @@ -0,0 +1,50 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package log provides simple api for using inherited logging +package log + +import ( + "context" + + "go.uber.org/zap" +) + +type contextKey struct{} + +// WithLoggerFields createsa new context with zap.Logger and passed fields +func WithLoggerFields(ctx context.Context, fields ...zap.Field) context.Context { + var logger = FromContext(ctx) + if logger == nil { + var err error + logger, err = zap.NewDevelopment() + if err != nil { + panic(err.Error()) + } + logger.Info("zap logger was initialized") + go func() { + <-ctx.Done() + _ = logger.Sync() + }() + } + return context.WithValue(ctx, contextKey{}, logger.With(fields...)) +} + +// FromContext returns zap.Logger from the context +func FromContext(ctx context.Context) *zap.Logger { + var val = ctx.Value(contextKey{}) + if val == nil { + return nil + } + return val.(*zap.Logger) +} diff --git a/internal/pkg/manifest/hls.go b/internal/pkg/manifest/hls.go new file mode 100644 index 0000000000..712e97f072 --- /dev/null +++ b/internal/pkg/manifest/hls.go @@ -0,0 +1,125 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package manifest provides data types for manifest based media files. +package manifest + +import ( + "bufio" + "fmt" + "strconv" + "strings" +) + +// HLSManifest represents an HLS manifest file +// with metadata about the playlist and its segments. +type HLSManifest struct { + Version int + TargetDuration int + SequenceNumber int + Segments []Segment + EndList bool +} + +// Segment represents a media segment in the HLS manifest. +type Segment struct { + URI string + Duration float64 + Title string +} + +// ToM3U8 serializes the HLSManifest to an M3U8 file format. +func (m *HLSManifest) ToM3U8() string { + var builder strings.Builder + + builder.WriteString("#EXTM3U\n") + builder.WriteString(fmt.Sprintf("#EXT-X-VERSION:%d\n", m.Version)) + builder.WriteString(fmt.Sprintf("#EXT-X-TARGETDURATION:%d\n", m.TargetDuration)) + builder.WriteString(fmt.Sprintf("#EXT-X-MEDIA-SEQUENCE:%d\n", m.SequenceNumber)) + + for _, segment := range m.Segments { + if segment.Title != "" { + builder.WriteString(fmt.Sprintf("#EXTINF:%.2f,%s\n", segment.Duration, segment.Title)) + } else { + builder.WriteString(fmt.Sprintf("#EXTINF:%.2f,\n", segment.Duration)) + } + builder.WriteString(fmt.Sprintf("%s\n", segment.URI)) + } + + if m.EndList { + builder.WriteString("#EXT-X-ENDLIST\n") + } + + return builder.String() +} + +// FromM3U8 converts raw input to the hls master file +// nolint +func FromM3U8(data string) (*HLSManifest, error) { + scanner := bufio.NewScanner(strings.NewReader(data)) + manifest := &HLSManifest{} + var currentSegment *Segment + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + + if strings.HasPrefix(line, "#EXTM3U") { + continue + } + if strings.HasPrefix(line, "#EXT-X-VERSION:") { + version, err := strconv.Atoi(strings.TrimPrefix(line, "#EXT-X-VERSION:")) + if err != nil { + return nil, err + } + manifest.Version = version + } else if strings.HasPrefix(line, "#EXT-X-TARGETDURATION:") { + targetDuration, err := strconv.Atoi(strings.TrimPrefix(line, "#EXT-X-TARGETDURATION:")) + if err != nil { + return nil, err + } + manifest.TargetDuration = targetDuration + } else if strings.HasPrefix(line, "#EXT-X-MEDIA-SEQUENCE:") { + sequenceNumber, err := strconv.Atoi(strings.TrimPrefix(line, "#EXT-X-MEDIA-SEQUENCE:")) + if err != nil { + return nil, err + } + manifest.SequenceNumber = sequenceNumber + } else if strings.HasPrefix(line, "#EXTINF:") { + parts := strings.SplitN(strings.TrimPrefix(line, "#EXTINF:"), ",", 2) + duration, err := strconv.ParseFloat(parts[0], 64) + if err != nil { + return nil, err + } + title := "" + if len(parts) > 1 { + title = parts[1] + } + currentSegment = &Segment{Duration: duration, Title: title} + } else if strings.HasPrefix(line, "#EXT-X-ENDLIST") { + manifest.EndList = true + } else if currentSegment != nil { + currentSegment.URI = line + manifest.Segments = append(manifest.Segments, *currentSegment) + currentSegment = nil + } + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return manifest, nil +} diff --git a/internal/pkg/manifest/hls_test.go b/internal/pkg/manifest/hls_test.go new file mode 100644 index 0000000000..15f3dbe022 --- /dev/null +++ b/internal/pkg/manifest/hls_test.go @@ -0,0 +1,143 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package manifest_test + +import ( + "testing" + + "github.com/huly-stream/internal/pkg/manifest" + "github.com/stretchr/testify/assert" +) + +func TestToM3U8(t *testing.T) { + tests := []struct { + name string + manifest manifest.HLSManifest + expected string + }{ + { + name: "simple manifest", + manifest: manifest.HLSManifest{ + Version: 3, + TargetDuration: 10, + SequenceNumber: 1, + Segments: []manifest.Segment{ + {URI: "segment1.ts", Duration: 9.5, Title: "Segment 1"}, + {URI: "segment2.ts", Duration: 9.0, Title: "Segment 2"}, + }, + EndList: true, + }, + expected: `#EXTM3U +#EXT-X-VERSION:3 +#EXT-X-TARGETDURATION:10 +#EXT-X-MEDIA-SEQUENCE:1 +#EXTINF:9.50,Segment 1 +segment1.ts +#EXTINF:9.00,Segment 2 +segment2.ts +#EXT-X-ENDLIST +`, + }, + { + name: "empty manifest", + manifest: manifest.HLSManifest{ + Version: 3, + TargetDuration: 10, + SequenceNumber: 1, + Segments: []manifest.Segment{}, + EndList: false, + }, + expected: `#EXTM3U +#EXT-X-VERSION:3 +#EXT-X-TARGETDURATION:10 +#EXT-X-MEDIA-SEQUENCE:1 +`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := tt.manifest.ToM3U8() + assert.Equal(t, tt.expected, actual) + }) + } +} + +func TestFromM3U8(t *testing.T) { + tests := []struct { + name string + data string + expected manifest.HLSManifest + err bool + }{ + { + name: "valid manifest", + data: `#EXTM3U +#EXT-X-VERSION:3 +#EXT-X-TARGETDURATION:10 +#EXT-X-MEDIA-SEQUENCE:1 +#EXTINF:9.50,Segment 1 +segment1.ts +#EXTINF:9.00,Segment 2 +segment2.ts +#EXT-X-ENDLIST +`, + expected: manifest.HLSManifest{ + Version: 3, + TargetDuration: 10, + SequenceNumber: 1, + Segments: []manifest.Segment{ + {URI: "segment1.ts", Duration: 9.5, Title: "Segment 1"}, + {URI: "segment2.ts", Duration: 9.0, Title: "Segment 2"}, + }, + EndList: true, + }, + err: false, + }, + // { + // name: "missing target duration", + // data: `#EXTM3U + // #EXT-X-VERSION:3 + // #EXT-X-MEDIA-SEQUENCE:1 + // #EXTINF:9.50,Segment 1 + // segment1.ts + // `, + // expected: manifest.HLSManifest{}, + // err: true, + // }, + { + name: "empty file", + data: "", + expected: manifest.HLSManifest{ + Version: 0, + TargetDuration: 0, + SequenceNumber: 0, + EndList: false, + }, + err: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual, err := manifest.FromM3U8(tt.data) + if tt.err { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expected, *actual) + } + }) + } +} diff --git a/internal/pkg/pprof/pprof.go b/internal/pkg/pprof/pprof.go new file mode 100644 index 0000000000..e7a5eb0302 --- /dev/null +++ b/internal/pkg/pprof/pprof.go @@ -0,0 +1,54 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package pprof provides all-in configured pprof server for debugging purposes +package pprof + +import ( + "context" + "net/http" + "net/http/pprof" + "time" + + "github.com/huly-stream/internal/pkg/log" + "go.uber.org/zap" +) + +// ListenAndServe - configures pprof http handlers +func ListenAndServe(ctx context.Context, listenOn string) { + log.FromContext(ctx).Debug("Profiler is enabled", zap.String("listening on", listenOn)) + mux := http.NewServeMux() + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + mux.Handle("/debug/pprof/allocs", pprof.Handler("allocs")) + mux.Handle("/debug/pprof/block", pprof.Handler("block")) + mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine")) + mux.Handle("/debug/pprof/heap", pprof.Handler("heap")) + mux.Handle("/debug/pprof/mutex", pprof.Handler("mutex")) + mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate")) + server := &http.Server{ + Addr: listenOn, + Handler: mux, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + if err := server.ListenAndServe(); err != nil { + log.FromContext(ctx).Debug("Failed to start profiler", zap.Error(err)) + } + + <-ctx.Done() + _ = server.Close() +} diff --git a/internal/pkg/sharedpipe/shared_pipe.go b/internal/pkg/sharedpipe/shared_pipe.go new file mode 100644 index 0000000000..0a698b46c8 --- /dev/null +++ b/internal/pkg/sharedpipe/shared_pipe.go @@ -0,0 +1,116 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sharedpipe provided a shared pipe that can be used when one writer can be shared between a couple of readers. +package sharedpipe + +import ( + "io" + "sync" + "sync/atomic" +) + +// Chunk represents a chunk of raw data for the readers +type Chunk struct { + content []byte + Next atomic.Pointer[Chunk] + ready chan struct{} + done chan struct{} +} + +// NewWriter creates a new shared pipe Writer +func NewWriter() *Writer { + var res = &Writer{ + done: make(chan struct{}), + } + res.curr.Store(&Chunk{done: res.done, ready: make(chan struct{})}) + return res +} + +// Transpile creates a new Reader +func (w *Writer) Transpile() *Reader { + var res = &Reader{ + curr: w.curr.Load(), + done: make(chan struct{}), + } + return res +} + +// Writer represents a shared pipe writer +type Writer struct { + curr atomic.Pointer[Chunk] + done chan struct{} + once sync.Once +} + +// Close closes the pipe for all readers +func (w *Writer) Close() error { + w.once.Do(func() { close(w.done) }) + return nil +} + +func (w *Writer) Write(p []byte) (n int, err error) { + var completePrevious = w.curr.Load().ready + var curr = w.curr.Load() + curr.Next.Store(&Chunk{content: p, ready: make(chan struct{}), done: w.done}) + w.curr.Store(curr.Next.Load()) + close(completePrevious) + return len(p), nil +} + +// Reader is reader from shared pipe, imlements io.Reader interface +type Reader struct { + curr *Chunk + once sync.Once + done chan struct{} + pos int +} + +// Close closes reader stream +func (s *Reader) Close() error { + s.once.Do(func() { + close(s.done) + }) + return nil +} + +func (s *Reader) Read(in []byte) (n int, err error) { + var curr = s.curr + for i := 0; i < len(in); { + for s.pos >= len(curr.content) { + select { + case <-curr.done: + curr = curr.Next.Load() + if curr == nil { + _ = s.Close() + return i, io.EOF + } + case <-s.done: + return i, io.ErrClosedPipe + case <-curr.ready: + curr = curr.Next.Load() + } + s.pos = 0 + s.curr = curr + } + var n = copy(in[i:], curr.content[s.pos:]) + s.pos += n + i += n + } + return len(in), nil +} + +var _ io.Closer = (*Writer)(nil) +var _ io.Closer = (*Reader)(nil) +var _ io.Reader = (*Reader)(nil) +var _ io.Writer = (*Writer)(nil) diff --git a/internal/pkg/sharedpipe/shared_pipe_bench_test.go b/internal/pkg/sharedpipe/shared_pipe_bench_test.go new file mode 100644 index 0000000000..3633e1fdf4 --- /dev/null +++ b/internal/pkg/sharedpipe/shared_pipe_bench_test.go @@ -0,0 +1,248 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package sharedpipe + +import ( + "bytes" + "io" + "math/rand" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const sendMessageSize = 8 * 1000 * 1000 +const readerCount = 10 + +func TestStability(t *testing.T) { + for range 10 { + testStability(t) + } +} + +// #nosec +func testStability(t *testing.T) { + var writer = NewWriter() + var readers []io.Reader + + for range 1000 { + readers = append(readers, writer.Transpile()) + } + var buff [4]byte + for range rand.Intn(1000) { + _, _ = writer.Write([]byte("ping")) + for i := range rand.Intn(10) { + _, _ = readers[i].Read(buff[:]) + } + } + _ = writer.Close() + for _, r := range readers { + _, err := io.ReadAll(r) + require.NoError(t, err) + } +} + +func TestBasicWriteRead(t *testing.T) { + writer := NewWriter() + defer func() { _ = writer.Close() }() + reader := writer.Transpile() + + data := []byte("Hello, World!") + n, err := writer.Write(data) + if err != nil { + t.Fatalf("Unexpected error on Write: %v", err) + } + if n != len(data) { + t.Fatalf("Expected to write %d bytes, wrote %d", len(data), n) + } + + readBuf := make([]byte, len(data)) + n, err = reader.Read(readBuf) + if err != nil && err != io.EOF { + t.Fatalf("Unexpected error on Read: %v", err) + } + if !bytes.Equal(readBuf[:n], data) { + t.Fatalf("Expected to read %q, got %q", data, readBuf[:n]) + } +} + +func TestConcurrentWriteRead(t *testing.T) { + writer := NewWriter() + defer func() { _ = writer.Close() }() + reader := writer.Transpile() + + var wg sync.WaitGroup + data := []byte("Hello, Concurrent World!") + readBuf := make([]byte, len(data)) + + wg.Add(2) + + go func() { + defer wg.Done() + _, err := writer.Write(data) + if err != nil { + t.Errorf("Unexpected error on Write: %v", err) + } + }() + + // Reader goroutine + go func() { + defer wg.Done() + n, err := reader.Read(readBuf) + if err != nil && err != io.EOF { + t.Errorf("Unexpected error on Read: %v", err) + } + if n != len(data) { + t.Errorf("Expected to read %d bytes, read %d", len(data), n) + } + if !bytes.Equal(readBuf[:n], data) { + t.Errorf("Expected to read %q, got %q", data, readBuf[:n]) + } + }() + + wg.Wait() +} + +func TestWriterClose(t *testing.T) { + writer := NewWriter() + reader := writer.Transpile() + + _, _ = writer.Write([]byte("Hello")) + _ = writer.Close() + + readBuf := make([]byte, 5) + n, err := reader.Read(readBuf) + if err != nil && err != io.EOF { + t.Fatalf("Unexpected error on Read: %v", err) + } + if n != 5 { + t.Fatalf("Expected to read 5 bytes, read %d", n) + } + + n, err = reader.Read(readBuf) + if err != io.EOF { + t.Fatalf("Expected EOF after writer close, got %v", err) + } + if n != 0 { + t.Fatalf("Expected to read 0 bytes after EOF, read %d", n) + } +} + +// Test reading from an empty writer. +func TestReadFromEmptyWriter(t *testing.T) { + writer := NewWriter() + _ = writer.Close() + reader := writer.Transpile() + + readBuf := make([]byte, 5) + _, err := reader.Read(readBuf) + require.ErrorIs(t, io.EOF, err) +} + +func Test_PipeWait(t *testing.T) { + var writer = NewWriter() + var reader = writer.Transpile() + var buff [4]byte + go func() { + time.Sleep(time.Second / 10) + _, _ = writer.Write([]byte("test")) + }() + _, _ = reader.Read(buff[:]) + require.Equal(t, "test", string(buff[:])) +} + +func Test_Consistent(t *testing.T) { + var writer = NewWriter() + var readers []io.Reader + + for range readerCount { + readers = append(readers, writer.Transpile()) + } + + _, _ = writer.Write([]byte("Hello")) + _, _ = writer.Write([]byte(" ")) + _, _ = writer.Write([]byte("World!")) + _ = writer.Close() + + var res strings.Builder + + for i := range readerCount { + for { + var buffer = make([]byte, 2) + _, err := readers[i].Read(buffer) + if err == io.EOF { + break + } + _, _ = res.WriteString(string(buffer)) + } + require.Equal(t, "Hello World!", res.String()) + res.Reset() + } +} + +// Benchmark_DefaultPipe-8 (4 b) 61956 19177 ns/op 48 B/op 1 allocs/op +// Benchmark_DefaultPipe-8 (8 mb) 22 49187741 ns/op 118 B/op 1 allocs/op +func Benchmark_DefaultPipe(b *testing.B) { + var data [sendMessageSize]byte + var buffer = make([]byte, len(data)) + var readers []io.Reader + var writers []io.Writer + + for i := 0; i < readerCount; i++ { + r, w := io.Pipe() + readers = append(readers, r) + writers = append(writers, w) + } + + b.ReportAllocs() + b.ResetTimer() + + for range b.N { + go func() { + for i := 0; i < readerCount; i++ { + _, _ = writers[i].Write(data[:]) + } + }() + for i := 0; i < readerCount; i++ { + _, _ = readers[i].Read(buffer) + } + } +} + +// Benchmark_SharedPipe-8 (4 b) 161847 8131 ns/op 160 B/op 2 allocs/op +// Benchmark_SharedPipe-8 (8 mb) 69 15880031 ns/op 160 B/op 2 allocs/op +func Benchmark_SharedPipe(b *testing.B) { + var data [sendMessageSize]byte + var buffer = make([]byte, len(data)) + var writer = NewWriter() + var readers []io.Reader + + for range readerCount { + readers = append(readers, writer.Transpile()) + } + + b.ReportAllocs() + b.ResetTimer() + + for range b.N { + _, _ = writer.Write(data[:]) + for i := 0; i < readerCount; i++ { + _, _ = readers[i].Read(buffer) + } + } +} diff --git a/internal/pkg/transcoding/command.go b/internal/pkg/transcoding/command.go new file mode 100644 index 0000000000..f43b5cf594 --- /dev/null +++ b/internal/pkg/transcoding/command.go @@ -0,0 +1,163 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package transcoding + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + + "github.com/pkg/errors" + + "github.com/huly-stream/internal/pkg/log" + "go.uber.org/zap" +) + +// Options represents configuration for the ffmpeg command +type Options struct { + OuputDir string + Resolutions []string + Threads int + UploadID string +} + +func measure(options *Options) int64 { + var res int64 + for _, resolution := range options.Resolutions { + var w, h int + var parts = strings.Split(resolution, ":") + + if len(parts) > 1 { + w, _ = strconv.Atoi(parts[0]) + w = max(w, 320) + h, _ = strconv.Atoi(parts[1]) + h = max(h, 240) + + res += int64(w) * int64(h) + } + } + + return max(res, 320*240) +} + +func newFfmpegCommand(ctx context.Context, in io.Reader, options *Options) (*exec.Cmd, error) { + if options == nil { + return nil, errors.New("options should not be nil") + } + if ctx == nil { + return nil, errors.New("ctx should not be nil") + } + + var logger = log.FromContext(ctx).With(zap.String("func", "NewFFMpegCommand")) + var args []string + + if options.Resolutions == nil { + logger.Debug("resolutions were not provided, building audio command...") + args = BuildAudioCommand(options) + } else { + logger.Debug("building video command...") + args = BuildVideoCommand(options) + } + + logger.Debug("prepared command: ", zap.Strings("args", args)) + + var result = exec.CommandContext(ctx, "ffmpeg", args...) + result.Stderr = os.Stdout + result.Stdout = os.Stdout + result.Stdin = in + + return result, nil +} + +func buildCommonComamnd(opts *Options) []string { + return []string{ + "-threads", fmt.Sprint(opts.Threads), + "-i", "pipe:0", + } +} + +// BuildAudioCommand returns flags for getting the audio from the input +func BuildAudioCommand(opts *Options) []string { + var commonPart = buildCommonComamnd(opts) + + return append(commonPart, + "-vn", "-acodec", + "copy", filepath.Join(opts.OuputDir, opts.UploadID), + ) +} + +// BuildVideoCommand returns flags for ffmpeg for video transcoding +func BuildVideoCommand(opts *Options) []string { + var result = buildCommonComamnd(opts) + + for _, res := range opts.Resolutions { + var prefix string + var w, h int + var parts = strings.Split(res, ":") + + if len(parts) > 1 { + w, _ = strconv.Atoi(parts[0]) + h, _ = strconv.Atoi(parts[1]) + } + w = max(w, 640) + h = max(h, 480) + prefix = ResolutionFromPixels(w * h) + + result = append(result, + "-vf", fmt.Sprintf("scale=%d:%d", w, h), + "-c:v", + "libx264", + "-preset", "veryfast", + "-crf", "23", + "-g", "60", + "-hls_time", "5", + "-hls_list_size", "0", + "-hls_segment_filename", filepath.Join(opts.OuputDir, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", prefix)), + filepath.Join(opts.OuputDir, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, prefix))) + } + + return result +} + +var resolutions = []struct { + pixels int + label string +}{ + {pixels: 640 * 480, label: "320p"}, + {pixels: 1280 * 720, label: "480p"}, + {pixels: 1920 * 1080, label: "720p"}, + {pixels: 2560 * 1440, label: "1k"}, + {pixels: 3840 * 2160, label: "2k"}, + {pixels: 5120 * 2160, label: "4k"}, + {pixels: 7680 * 4320, label: "5k"}, +} + +// ResolutionFromPixels converts pixel count to short string +func ResolutionFromPixels(pixels int) string { + idx := sort.Search(len(resolutions), func(i int) bool { + return pixels < resolutions[i].pixels + }) + if idx == len(resolutions) { + return "8k" + } + return resolutions[idx].label +} diff --git a/internal/pkg/transcoding/command_test.go b/internal/pkg/transcoding/command_test.go new file mode 100644 index 0000000000..14a7db515d --- /dev/null +++ b/internal/pkg/transcoding/command_test.go @@ -0,0 +1,62 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package transcoding_test + +import ( + "runtime" + "strings" + "testing" + + "github.com/huly-stream/internal/pkg/transcoding" + "github.com/stretchr/testify/require" +) + +func Test_BuildVideoCommand_Basic(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + var simpleHlsCommand = transcoding.BuildVideoCommand(&transcoding.Options{ + OuputDir: "test", + UploadID: "1", + Threads: 4, + Resolutions: []string{"1280:720"}, + }) + + const expected = `-threads 4 -i pipe:0 -vf scale=1280:720 -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_list_size 0 -hls_segment_filename test/1_%03d_720p.ts test/1_720p_master.m3u8` + + require.Contains(t, expected, strings.Join(simpleHlsCommand, " ")) +} + +func TestResolutionFromPixels(t *testing.T) { + tests := []struct { + pixels int + expected string + }{ + {pixels: 320 * 240, expected: "320p"}, + {pixels: 640 * 480, expected: "480p"}, + {pixels: 1280 * 720, expected: "720p"}, + {pixels: 1920 * 1080, expected: "1k"}, + {pixels: 2560 * 1440, expected: "2k"}, + {pixels: 3840 * 2160, expected: "4k"}, + {pixels: 5120 * 2160, expected: "5k"}, + {pixels: 9000 * 4000, expected: "8k"}, + } + + for _, tt := range tests { + t.Run(tt.expected, func(t *testing.T) { + result := transcoding.ResolutionFromPixels(tt.pixels) + require.Equal(t, tt.expected, result, "ResolutionFromPixels(%d)", tt.pixels) + }) + } +} diff --git a/internal/pkg/transcoding/limiter.go b/internal/pkg/transcoding/limiter.go new file mode 100644 index 0000000000..f1fdf51d9f --- /dev/null +++ b/internal/pkg/transcoding/limiter.go @@ -0,0 +1,78 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package transcoding + +import "sync/atomic" + +// Limiter is a simple CAS data structure for managing resources. +type Limiter struct { + capacity int64 + maxCapacity int64 +} + +// NewLimiter creates a new limiter with the given initial capacity. +func NewLimiter(capacity int64) *Limiter { + return &Limiter{ + capacity: capacity, + maxCapacity: capacity, + } +} + +// TryConsume attempts to consume the specified amount of capacity. +// Returns true if successful, false otherwise. +func (l *Limiter) TryConsume(amount int64) bool { + if amount <= 0 { + return false + } + + for { + current := atomic.LoadInt64(&l.capacity) + if current < amount { + return false + } + updated := current - amount + if atomic.CompareAndSwapInt64(&l.capacity, current, updated) { + return true + } + } +} + +// ReturnCapacity adds the specified amount back to the limiter's capacity. +// Does not exceed the maximum capacity. +func (l *Limiter) ReturnCapacity(amount int64) { + if amount <= 0 { + return + } + + for { + current := atomic.LoadInt64(&l.capacity) + updated := current + amount + if updated > l.maxCapacity { + updated = l.maxCapacity + } + if atomic.CompareAndSwapInt64(&l.capacity, current, updated) { + break + } + } +} + +// GetCapacity retrieves the current capacity for debugging or monitoring purposes. +func (l *Limiter) GetCapacity() int64 { + return atomic.LoadInt64(&l.capacity) +} + +// GetMaxCapacity retrieves the maximum capacity. +func (l *Limiter) GetMaxCapacity() int64 { + return l.maxCapacity +} diff --git a/internal/pkg/transcoding/limiter_test.go b/internal/pkg/transcoding/limiter_test.go new file mode 100644 index 0000000000..df7db82ba0 --- /dev/null +++ b/internal/pkg/transcoding/limiter_test.go @@ -0,0 +1,88 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +package transcoding_test + +import ( + "sync" + "sync/atomic" + "testing" + + "github.com/huly-stream/internal/pkg/transcoding" + "github.com/stretchr/testify/require" +) + +func TestLimiter(t *testing.T) { + limiter := transcoding.NewLimiter(10) + + t.Run("Initial capacity", func(t *testing.T) { + require.Equal(t, int64(10), limiter.GetCapacity()) + }) + + t.Run("Successful consume", func(t *testing.T) { + success := limiter.TryConsume(5) + require.True(t, success) + require.Equal(t, int64(5), limiter.GetCapacity()) + }) + + t.Run("Failed consume", func(t *testing.T) { + success := limiter.TryConsume(10) + require.False(t, success) + require.Equal(t, int64(5), limiter.GetCapacity()) + }) + + t.Run("Return capacity", func(t *testing.T) { + limiter.ReturnCapacity(3) + require.Equal(t, int64(8), limiter.GetCapacity()) + }) + + t.Run("Exceeding max capacity", func(t *testing.T) { + limiter.ReturnCapacity(10) + require.Equal(t, int64(10), limiter.GetCapacity()) + }) +} + +func TestLimiterConcurrency(t *testing.T) { + limiter := transcoding.NewLimiter(10) + var wg sync.WaitGroup + + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + limiter.TryConsume(2) + }() + } + + wg.Wait() + require.LessOrEqual(t, limiter.GetCapacity(), int64(0)) +} + +func TestLimiterCAS(t *testing.T) { + limiter := transcoding.NewLimiter(10) + var successful int64 + var wg sync.WaitGroup + + for i := 0; i < 1000; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if limiter.TryConsume(1) { + atomic.AddInt64(&successful, 1) + } + }() + } + wg.Wait() + + require.Equal(t, int64(10), successful) + require.Equal(t, int64(0), limiter.GetCapacity()) +} diff --git a/internal/pkg/transcoding/scheduler.go b/internal/pkg/transcoding/scheduler.go new file mode 100644 index 0000000000..25872a3047 --- /dev/null +++ b/internal/pkg/transcoding/scheduler.go @@ -0,0 +1,129 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package transcoding + +import ( + "context" + "strings" + "sync" + + "github.com/pkg/errors" + + "github.com/google/uuid" + "github.com/huly-stream/internal/pkg/config" + "github.com/huly-stream/internal/pkg/log" + "github.com/huly-stream/internal/pkg/sharedpipe" + "github.com/huly-stream/internal/pkg/uploader" + "github.com/tus/tusd/v2/pkg/handler" + "go.uber.org/zap" +) + +// Scheduler represents manager for worker. It creates a new worker for clients and manages its life cycle. +type Scheduler struct { + conf *config.Config + + limiter *Limiter + + mainContext context.Context + logger *zap.Logger + workers sync.Map +} + +// NewScheduler creates a new scheduler for transcode operations. +func NewScheduler(ctx context.Context, c *config.Config) *Scheduler { + return &Scheduler{ + conf: c, + limiter: NewLimiter(c.MaxCapacity), + mainContext: ctx, + logger: log.FromContext(ctx).With(zap.String("Scheduler", c.OutputDir)), + } +} + +// NewUpload creates a new worker with passed parameters +func (s *Scheduler) NewUpload(ctx context.Context, info handler.FileInfo) (handler.Upload, error) { + if info.ID == "" { + info.ID = uuid.NewString() + } + + s.logger.Debug("NewUpload", zap.String("ID", info.ID)) + + var result = &Worker{ + done: make(chan struct{}), + writer: sharedpipe.NewWriter(), + info: info, + logger: log.FromContext(s.mainContext).With(zap.String("Worker", info.ID)), + } + + var resolutions = strings.Split(info.MetaData["resolutions"], ",") + + var commandOptions = Options{ + OuputDir: s.conf.OutputDir, + Threads: s.conf.MaxThreads, + UploadID: info.ID, + Resolutions: resolutions, + } + + result.cost = measure(&commandOptions) + + if !s.limiter.TryConsume(result.cost) { + s.logger.Error("run out of resources") + return nil, errors.New("run out of resources") + } + + if s.conf.EndpointURL != nil { + s.logger.Debug("found endpoint url in the config, starting uploader...") + var contentUploader, err = uploader.New(s.mainContext, *s.conf, info.ID, info.MetaData) + if err != nil { + return nil, err + } + result.contentUploader = contentUploader + go func() { + var serverErr = result.contentUploader.Serve() + result.logger.Debug("content uploader has finished", zap.Error(serverErr)) + }() + } + + s.workers.Store(result.info.ID, result) + s.logger.Sugar().Debugf("New Upload: info %v", result.info) + if err := result.start(s.mainContext, &commandOptions); err != nil { + return nil, err + } + return result, nil +} + +// GetUpload returns current a worker based on upload id +func (s *Scheduler) GetUpload(ctx context.Context, id string) (upload handler.Upload, err error) { + if v, ok := s.workers.Load(id); ok { + s.logger.Debug("GetUpload: found worker by id", zap.String("id", id)) + return v.(*Worker), nil + } + s.logger.Debug("GetUpload: worker not found", zap.String("id", id)) + return nil, errors.New("bad id") +} + +// AsTerminatableUpload returns tusd handler.TerminatableUpload +func (s *Scheduler) AsTerminatableUpload(upload handler.Upload) handler.TerminatableUpload { + var worker = upload.(*Worker) + s.logger.Debug("AsTerminatableUpload, trying to return capacity", zap.Int64("cost", worker.cost)) + s.limiter.ReturnCapacity(worker.cost) + return worker +} + +// AsLengthDeclarableUpload returns tusd handler.LengthDeclarableUpload +func (s *Scheduler) AsLengthDeclarableUpload(upload handler.Upload) handler.LengthDeclarableUpload { + s.logger.Debug("AsLengthDeclarableUpload") + return upload.(*Worker) +} diff --git a/internal/pkg/transcoding/worker.go b/internal/pkg/transcoding/worker.go new file mode 100644 index 0000000000..60baccc465 --- /dev/null +++ b/internal/pkg/transcoding/worker.go @@ -0,0 +1,125 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package transcoding provides objects and functions for video trnascoding +package transcoding + +import ( + "context" + "io" + + "github.com/pkg/errors" + + "github.com/huly-stream/internal/pkg/sharedpipe" + "github.com/huly-stream/internal/pkg/uploader" + "github.com/tus/tusd/v2/pkg/handler" + "go.uber.org/zap" +) + +// Worker manages client's input and transcodes it based on the passsed configuration +type Worker struct { + contentUploader uploader.Uploader + logger *zap.Logger + info handler.FileInfo + writer *sharedpipe.Writer + reader *sharedpipe.Reader + cost int64 + done chan struct{} +} + +// WriteChunk calls when client sends a chunk of raw data +func (w *Worker) WriteChunk(ctx context.Context, _ int64, src io.Reader) (int64, error) { + w.logger.Debug("Write Chunk start", zap.Int64("offset", w.info.Offset)) + var bytes, err = io.ReadAll(src) + _, _ = w.writer.Write(bytes) + var n = int64(len(bytes)) + w.info.Offset += n + w.logger.Debug("Write Chunk end", zap.Int64("offset", w.info.Offset), zap.Error(err)) + return n, err +} + +// DeclareLength sets length of the video input +func (w *Worker) DeclareLength(ctx context.Context, length int64) error { + w.info.Size = length + w.info.SizeIsDeferred = false + w.logger.Debug("DeclareLength", zap.Int64("size", length), zap.Bool("SizeIsDeferred", w.info.SizeIsDeferred)) + return nil +} + +// GetInfo returns info about transcoing status +func (w *Worker) GetInfo(ctx context.Context) (handler.FileInfo, error) { + w.logger.Debug("GetInfo is executed") + return w.info, nil +} + +// GetReader returns worker's bytes stream +func (w *Worker) GetReader(ctx context.Context) (io.ReadCloser, error) { + w.logger.Debug("GetReader is executed, creating current reader...") + return w.reader, nil +} + +// Terminate calls when upload has failed +func (w *Worker) Terminate(ctx context.Context) error { + w.logger.Debug("Terminating...") + if w.contentUploader != nil { + go func() { + <-w.done + w.contentUploader.Rollback() + }() + } + return w.writer.Close() +} + +// ConcatUploads calls when upload resumed after fail +func (w *Worker) ConcatUploads(ctx context.Context, partialUploads []handler.Upload) error { + w.logger.Debug("ConcatUploads was executed, it's not implemented") + // + // TODO: load raw source from the Buckup bucket, terminate all workers with same ID and start process again. + // + return errors.New("not implemented") +} + +// FinishUpload calls when upload finished without errors on the client side +func (w *Worker) FinishUpload(ctx context.Context) error { + w.logger.Debug("finishing upload...") + if w.contentUploader != nil { + go func() { + <-w.done + w.contentUploader.Terminate() + }() + } + return w.writer.Close() +} + +// AsConcatableUpload returns tusd handler.ConcatableUpload +func (s *Scheduler) AsConcatableUpload(upload handler.Upload) handler.ConcatableUpload { + s.logger.Debug("AsConcatableUpload is executed") + return upload.(*Worker) +} + +func (w *Worker) start(ctx context.Context, options *Options) error { + w.reader = w.writer.Transpile() + var cmd, err = newFfmpegCommand(ctx, w.reader, options) + if err != nil { + return err + } + go func() { + defer close(w.done) + if runErr := cmd.Run(); runErr != nil { + w.logger.Error("transoding provider is exited with error", zap.Error(err)) + } else { + w.logger.Debug("transoding provider has finished without errors") + } + }() + return nil +} diff --git a/internal/pkg/uploader/datalake.go b/internal/pkg/uploader/datalake.go new file mode 100644 index 0000000000..6869cc9f99 --- /dev/null +++ b/internal/pkg/uploader/datalake.go @@ -0,0 +1,124 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package uploader + +import ( + "bytes" + "context" + "io" + "mime/multipart" + "os" + "path/filepath" + + "github.com/huly-stream/internal/pkg/log" + "github.com/pkg/errors" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) + +// DatalakeStorage represents datalake storage +type DatalakeStorage struct { + baseURL string + workspace string + token string +} + +// NewDatalakeStorage creates a new datalake client +func NewDatalakeStorage(baseURL, workspace, token string) Storage { + return &DatalakeStorage{ + baseURL: "https://" + baseURL, + token: token, + workspace: workspace, + } +} + +// UploadFile uploads file to the datalake +func (d *DatalakeStorage) UploadFile(ctx context.Context, fileName string) error { + // #nosec + file, err := os.Open(fileName) + if err != nil { + return err + } + defer func() { + _ = file.Close() + }() + + var _, objectKey = filepath.Split(fileName) + var logger = log.FromContext(ctx).With(zap.String("datalake upload", d.workspace), zap.String("fileName", fileName)) + + logger.Debug("start uploading") + + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + + part, err := writer.CreateFormFile("file", objectKey) + if err != nil { + return errors.Wrapf(err, "failed to create form file") + } + + _, err = io.Copy(part, file) + if err != nil { + return errors.Wrapf(err, "failed to copy file data") + } + + _ = writer.Close() + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + + res := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(res) + + req.SetRequestURI(d.baseURL + "/upload/form-data/" + d.workspace) + req.Header.SetMethod(fasthttp.MethodPost) + req.Header.Add("Authorization", "Bearer "+d.token) + req.Header.SetContentType(writer.FormDataContentType()) + req.SetBody(body.Bytes()) + + client := fasthttp.Client{} + if err := client.Do(req, res); err != nil { + return errors.Wrapf(err, "upload failed") + } + + logger.Debug("file uploaded") + + return nil +} + +// DeleteFile deletes file from the datalake +func (d *DatalakeStorage) DeleteFile(ctx context.Context, fileName string) error { + var logger = log.FromContext(ctx).With(zap.String("datalake delete", d.workspace), zap.String("fileName", fileName)) + logger.Debug("start deleting") + + var _, objectKey = filepath.Split(fileName) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + + res := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(res) + + req.SetRequestURI(d.baseURL + "/blob/" + d.workspace + "/" + objectKey) + req.Header.SetMethod(fasthttp.MethodDelete) + req.Header.Add("Authorization", "Bearer "+d.token) + + client := fasthttp.Client{} + if err := client.Do(req, res); err != nil { + return errors.Wrapf(err, "delete failed") + } + + logger.Debug("file deleted") + + return nil +} diff --git a/internal/pkg/uploader/options.go b/internal/pkg/uploader/options.go new file mode 100644 index 0000000000..076e2c70af --- /dev/null +++ b/internal/pkg/uploader/options.go @@ -0,0 +1,19 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package uploader + +type options struct{} + +// Option provides option for storages +type Option func(*options) diff --git a/internal/pkg/uploader/postpone.go b/internal/pkg/uploader/postpone.go new file mode 100644 index 0000000000..61b22b79b4 --- /dev/null +++ b/internal/pkg/uploader/postpone.go @@ -0,0 +1,40 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package uploader + +import ( + "context" + "time" +) + +func (u *uploader) postpone(id string, action func()) { + var ctx, cancel = context.WithCancel(context.Background()) + var startCh = time.After(u.postponeDuration) + + if v, ok := u.contexts.Load(id); ok { + (*v.(*context.CancelFunc))() + } + u.contexts.Store(id, &cancel) + + go func() { + defer cancel() + select { + case <-ctx.Done(): + return + case <-startCh: + action() + u.contexts.CompareAndDelete(id, &cancel) + } + }() +} diff --git a/internal/pkg/uploader/postpone_test.go b/internal/pkg/uploader/postpone_test.go new file mode 100644 index 0000000000..89522b4cec --- /dev/null +++ b/internal/pkg/uploader/postpone_test.go @@ -0,0 +1,44 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package uploader + +import ( + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func Test_Postpone(t *testing.T) { + var u = uploader{ + postponeDuration: time.Second / 4, + } + var counter atomic.Int32 + u.postpone("1", func() { counter.Add(1) }) + time.Sleep(time.Second / 8) + u.postpone("1", func() { counter.Add(1) }) + time.Sleep(time.Second / 2) + require.Equal(t, int32(1), counter.Load()) + time.Sleep(time.Second / 2) + require.Equal(t, int32(1), counter.Load()) +} + +func Test_WithoutPostpone(t *testing.T) { + var counter atomic.Int32 + var u uploader + u.postpone("1", func() { counter.Add(1) }) + time.Sleep(time.Second / 10) + require.Equal(t, int32(1), counter.Load()) +} diff --git a/internal/pkg/uploader/s3.go b/internal/pkg/uploader/s3.go new file mode 100644 index 0000000000..ba688618f9 --- /dev/null +++ b/internal/pkg/uploader/s3.go @@ -0,0 +1,143 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package uploader + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + + "os" + "path/filepath" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + "github.com/huly-stream/internal/pkg/log" + "go.uber.org/zap" +) + +// S3Storage represents S3 storage +type S3Storage struct { + client *s3.Client + bucketName string +} + +// NewS3 creates a new S3 storage +func NewS3(ctx context.Context, endpoint string) Storage { + var accessKeyID = os.Getenv("AWS_ACCESS_KEY_ID") + var accessKeySecret = os.Getenv("AWS_SECRET_ACCESS_KEY") + var bucketName = os.Getenv("AWS_BUCKET_NAME") + + cfg, err := config.LoadDefaultConfig(ctx, + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyID, accessKeySecret, "")), + config.WithRegion("auto"), + ) + if err != nil { + panic(err.Error()) + } + + var s3Client = s3.NewFromConfig(cfg, func(o *s3.Options) { + endpoint = "https://" + endpoint + o.BaseEndpoint = &endpoint + }) + + return &S3Storage{ + client: s3Client, + bucketName: bucketName, + } +} + +func getContentType(objectKey string) string { + if strings.HasSuffix(objectKey, ".txt") { + return "txt" + } + if strings.HasSuffix(objectKey, ".ts") { + return "video/mp2t" + } + if strings.HasSuffix(objectKey, ".m3u8") { + return "application/x-mpegurl" + } + return "application/octet-stream" +} + +// DeleteFile deletes file from the s3 storage +func (u *S3Storage) DeleteFile(ctx context.Context, fileName string) error { + var _, objectKey = filepath.Split(fileName) + var logger = log.FromContext(ctx).With(zap.String("s3 delete", u.bucketName), zap.String("fileName", fileName)) + + logger.Debug("start deleting") + input := &s3.DeleteObjectInput{ + Bucket: aws.String(u.bucketName), + Key: aws.String(objectKey), + } + + _, err := u.client.DeleteObject(ctx, input) + if err != nil { + return fmt.Errorf("failed to delete file from S3: %w", err) + } + logger.Debug("file deleted") + return nil +} + +// UploadFile uploads file to the s3 storage +func (u *S3Storage) UploadFile(ctx context.Context, fileName string) error { + var _, objectKey = filepath.Split(fileName) + var logger = log.FromContext(ctx).With(zap.String("s3 upload", u.bucketName), zap.String("fileName", fileName)) + logger.Debug("start upload file") + + // #nosec + var file, err = os.Open(fileName) + + if err != nil { + logger.Error("can not open file", zap.Error(err)) + return err + } + + defer func() { + _ = file.Close() + }() + _, err = u.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(u.bucketName), + Key: aws.String(objectKey), + Body: file, + ContentType: aws.String(getContentType(objectKey)), + }) + + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "EntityTooLarge" { + logger.Error("Error while uploading object. The object is too large." + + "To upload objects larger than 5GB, use the S3 console (160GB max)" + + "or the multipart upload API (5TB max).") + } else { + logger.Error("Couldn't upload file", zap.Error(err)) + } + return apiErr + } + + err = s3.NewObjectExistsWaiter(u.client).Wait( + ctx, &s3.HeadObjectInput{Bucket: aws.String(u.bucketName), Key: aws.String(objectKey)}, time.Minute) + if err != nil { + logger.Debug("Failed attempt to wait for object to exist.") + } + + logger.Debug("file has uploaded") + return err +} diff --git a/internal/pkg/uploader/uploader.go b/internal/pkg/uploader/uploader.go new file mode 100644 index 0000000000..86735191a4 --- /dev/null +++ b/internal/pkg/uploader/uploader.go @@ -0,0 +1,219 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package uploader provides objects and functions to work with uploading and monitoring files +package uploader + +import ( + "context" + "net/url" + "os" + "strings" + "sync" + "time" + + "github.com/pkg/errors" + + "github.com/fsnotify/fsnotify" + "github.com/huly-stream/internal/pkg/config" + "github.com/huly-stream/internal/pkg/log" + "go.uber.org/zap" +) + +type uploader struct { + ctx context.Context + cancel context.CancelFunc + baseDir string + uploadID string + masterFiles sync.Map + postponeDuration time.Duration + sentFiles sync.Map + storage Storage + contexts sync.Map + retryCount int + removeLocalContentOnUpload bool + eventBufferCount uint + isMasterFileFunc func(s string) bool +} + +// Rollback deletes all delivered files and also deletes all local content by uploadID +func (u *uploader) Rollback() { + log.FromContext(u.ctx).Debug("cancel") + defer u.cancel() + u.sentFiles.Range(func(key, value any) bool { + log.FromContext(u.ctx).Debug("deleting remote file", zap.String("key", key.(string))) + for range u.retryCount { + var err = u.storage.DeleteFile(u.ctx, key.(string)) + if err == nil { + break + } + log.FromContext(u.ctx).Debug("can not delete file", zap.Error(err)) + } + return true + }) + if !u.removeLocalContentOnUpload { + return + } + u.sentFiles.Range(func(key, value any) bool { + log.FromContext(u.ctx).Debug("deleting local file", zap.String("key", key.(string))) + _ = os.Remove(key.(string)) + return true + }) +} + +func (u *uploader) Terminate() { + log.FromContext(u.ctx).Debug("terminate") + defer u.cancel() + u.masterFiles.Range(func(key, value any) bool { + log.FromContext(u.ctx).Debug("uploading master file", zap.String("key", key.(string))) + for range u.retryCount { + var uploadErr = u.storage.UploadFile(u.ctx, key.(string)) + if uploadErr == nil { + break + } + log.FromContext(u.ctx).Debug("can not upload file", zap.Error(uploadErr)) + } + return true + }) + if !u.removeLocalContentOnUpload { + return + } + u.masterFiles.Range(func(key, value any) bool { + log.FromContext(u.ctx).Debug("deleting local master file", zap.String("key", key.(string))) + _ = os.Remove(key.(string)) + return true + }) + u.sentFiles.Range(func(key, value any) bool { + log.FromContext(u.ctx).Debug("deleting local file", zap.String("key", key.(string))) + _ = os.Remove(key.(string)) + return true + }) +} + +func (u *uploader) Serve() error { + var logger = log.FromContext(u.ctx) + logger = logger.With(zap.String("uploader", u.uploadID), zap.String("dir", u.baseDir)) + var watcher, err = fsnotify.NewBufferedWatcher(u.eventBufferCount) + if err != nil { + logger.Error("can not start watcher") + return err + } + if err := watcher.Add(u.baseDir); err != nil { + return err + } + defer func() { + _ = watcher.Close() + }() + + logger.Debug("uploader initialized and started to watch") + + for { + select { + case <-u.ctx.Done(): + logger.Debug("done") + return u.ctx.Err() + case event, ok := <-watcher.Events: + if !strings.Contains(event.Name, u.uploadID) { + continue + } + if !ok { + return u.ctx.Err() + } + if u.isMasterFileFunc(event.Name) { + u.masterFiles.Store(event.Name, struct{}{}) + logger.Debug("found master file", zap.String("eventName", event.Name)) + continue + } + u.postpone(event.Name, func() { + logger.Debug("started to upload", zap.String("eventName", event.Name)) + for range u.retryCount { + var uploadErr = u.storage.UploadFile(u.ctx, event.Name) + if uploadErr == nil { + break + } + logger.Error("can not upload file", zap.Error(uploadErr)) + } + logger.Debug("added to sentFiles", zap.String("eventName", event.Name)) + u.sentFiles.Store(event.Name, struct{}{}) + }) + case err, ok := <-watcher.Errors: + if !ok { + return u.ctx.Err() + } + logger.Error("get an error", zap.Error(err)) + } + } +} + +// Uploader manages content delivering +type Uploader interface { + Terminate() + Rollback() + Serve() error +} + +// Storage represents file-based storage +type Storage interface { + UploadFile(ctx context.Context, fileName string) error + DeleteFile(ctx context.Context, fileName string) error +} + +// New creates a new instance of Uplaoder +func New(ctx context.Context, conf config.Config, uploadID string, metadata map[string]string) (Uploader, error) { + var uploaderCtx, uploaderCancel = context.WithCancel(ctx) + var storage Storage + var err error + + if conf.EndpointURL != nil { + storage, err = NewStorageByURL(ctx, conf.EndpointURL, metadata) + if err != nil { + uploaderCancel() + return nil, err + } + } + + return &uploader{ + ctx: uploaderCtx, + cancel: uploaderCancel, + uploadID: uploadID, + removeLocalContentOnUpload: conf.RemoveContentOnUpload, + postponeDuration: time.Second * 2, + storage: storage, + retryCount: 5, + baseDir: conf.OutputDir, + eventBufferCount: 100, + isMasterFileFunc: func(s string) bool { + return strings.HasSuffix(s, "m3u8") + }, + }, nil +} + +// NewStorageByURL creates a new storage basd on the type from the url scheme, for example "datalake://my-datalake-endpoint" +func NewStorageByURL(ctx context.Context, u *url.URL, headers map[string]string) (Storage, error) { + switch u.Scheme { + case "tus": + return nil, errors.New("not imlemented yet") + case "datalake": + if headers["workspace"] == "" { + return nil, errors.New("missed workspace in the client's metadata") + } + if headers["token"] == "" { + return nil, errors.New("missed auth token in the client's metadata") + } + return NewDatalakeStorage(u.Hostname(), headers["workspace"], headers["token"]), nil + case "s3": + return NewS3(ctx, u.Hostname()), nil + default: + return nil, errors.New("unknown scheme") + } +} From 1b8be96319b735d0196738772a6ff65d68f7a29e Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Tue, 4 Feb 2025 05:15:28 +0300 Subject: [PATCH 03/52] apply review comments Signed-off-by: denis-tingaikin --- .github/workflows/main.yaml | 2 -- README.md | 16 +++++++++------- internal/pkg/uploader/datalake.go | 10 ++++++++-- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index a288973049..635804cf49 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -29,8 +29,6 @@ jobs: matrix: os: - ubuntu - - windows - - macos runs-on: ${{ matrix.os }}-latest steps: - name: Check out code diff --git a/README.md b/README.md index 8eb361d442..f977167dba 100644 --- a/README.md +++ b/README.md @@ -20,13 +20,15 @@ The Huly Stream high-performance HTTP-based transcoding service. Huly-stream is - `webm` ### Output Options -- **TUS Upload**: Resumable file uploads via TUS protocol. -- **s3 Upload**: Direct upload to Amazon S3. -- **datalake Upload**: Integration for data lake storage systems. - **Supported Output Formats**: - `aac` - `hls` +### Upload options +- **TUS Upload**: Resumable file uploads via TUS protocol. +- **s3 Upload**: Direct upload to S3 storage. +- **datalake Upload**: Upload to datalake storage. + ### Key Functionalities - **Live transcoing with minimal upload time**: Transcoding results are going to be avaible after stream completion. - **Transcoding Cancelation**: Cancel or pause ongoing transcoding in real-time. @@ -74,13 +76,13 @@ STREAM_REMOVE_CONTENT_ON_UPLOAD True or False true STREAM_UPLOAD_RAW_CONTENT True or False false uploads content in raw quality to the endpoint if true ``` -### Metadata: +### Metadata -**resolutions** if passed, set the resolution for the output, for example, 'resolutions: 1920:1080, 1280:720.' +**resolutions:** if passed, set the resolution for the output, for example, 'resolutions: 1920:1080, 1280:720.' -**token** must be provided to be authorized in the Huly's datalake service. +**token:** must be provided to be authorized in the Huly's datalake service. -**workspace** required for uploading content. +**workspace:** required for uploading content to the datalake storage. diff --git a/internal/pkg/uploader/datalake.go b/internal/pkg/uploader/datalake.go index 6869cc9f99..43ba535efd 100644 --- a/internal/pkg/uploader/datalake.go +++ b/internal/pkg/uploader/datalake.go @@ -18,6 +18,7 @@ import ( "context" "io" "mime/multipart" + "net/url" "os" "path/filepath" @@ -54,7 +55,7 @@ func (d *DatalakeStorage) UploadFile(ctx context.Context, fileName string) error _ = file.Close() }() - var _, objectKey = filepath.Split(fileName) + var objectKey = getObjectKey(fileName) var logger = log.FromContext(ctx).With(zap.String("datalake upload", d.workspace), zap.String("fileName", fileName)) logger.Debug("start uploading") @@ -101,7 +102,7 @@ func (d *DatalakeStorage) DeleteFile(ctx context.Context, fileName string) error var logger = log.FromContext(ctx).With(zap.String("datalake delete", d.workspace), zap.String("fileName", fileName)) logger.Debug("start deleting") - var _, objectKey = filepath.Split(fileName) + var objectKey = getObjectKey(fileName) req := fasthttp.AcquireRequest() defer fasthttp.ReleaseRequest(req) @@ -122,3 +123,8 @@ func (d *DatalakeStorage) DeleteFile(ctx context.Context, fileName string) error return nil } + +func getObjectKey(s string) string { + var _, objectKey = filepath.Split(s) + return url.QueryEscape(objectKey) +} From ac22070f89801603b4ffb6dd745bd1250c079ca5 Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Fri, 21 Feb 2025 05:03:12 +0300 Subject: [PATCH 04/52] apply fixes after testing Signed-off-by: denis-tingaikin --- .github/workflows/docker-push.yaml | 2 +- cmd/huly-stream/main.go | 11 +- go.mod | 44 +++---- go.sum | 96 +++++++-------- internal/pkg/config/config.go | 27 +++-- internal/pkg/manifest/hls.go | 122 +++++-------------- internal/pkg/manifest/hls_test.go | 146 +++++------------------ internal/pkg/resconv/resconv.go | 129 ++++++++++++++++++++ internal/pkg/resconv/resconv_test.go | 120 +++++++++++++++++++ internal/pkg/transcoding/command.go | 111 +++++------------ internal/pkg/transcoding/command_test.go | 51 +++----- internal/pkg/transcoding/scheduler.go | 98 ++++++++++----- internal/pkg/transcoding/worker.go | 56 +++++++-- internal/pkg/uploader/datalake.go | 4 +- internal/pkg/uploader/s3.go | 3 + internal/pkg/uploader/uploader.go | 123 ++++++++++--------- 16 files changed, 625 insertions(+), 518 deletions(-) create mode 100644 internal/pkg/resconv/resconv.go create mode 100644 internal/pkg/resconv/resconv_test.go diff --git a/.github/workflows/docker-push.yaml b/.github/workflows/docker-push.yaml index 0308ce654b..ad67bcee7c 100644 --- a/.github/workflows/docker-push.yaml +++ b/.github/workflows/docker-push.yaml @@ -41,4 +41,4 @@ jobs: context: . platforms: linux/amd64,linux/arm64 push: true - tags: ${{ inputs.version }} + tags: hardcoreeng/huly-stream:${{ inputs.version }} diff --git a/cmd/huly-stream/main.go b/cmd/huly-stream/main.go index 2730cd8b35..f660a2511f 100644 --- a/cmd/huly-stream/main.go +++ b/cmd/huly-stream/main.go @@ -32,7 +32,7 @@ import ( tusd "github.com/tus/tusd/v2/pkg/handler" ) -const basePath = "/transcoding" +const basePath = "/recording" func main() { var ctx, cancel = signal.NotifyContext( @@ -43,16 +43,13 @@ func main() { syscall.SIGQUIT, ) defer cancel() - ctx = log.WithLoggerFields(ctx) var logger = log.FromContext(ctx) var conf = must(config.FromEnv()) - logger.Sugar().Debugf("provided config is %v", conf) mustNoError(os.MkdirAll(conf.OutputDir, os.ModePerm)) - if conf.PprofEnabled { go pprof.ListenAndServe(ctx, "localhost:6060") } @@ -71,10 +68,12 @@ func main() { Logger: slog.New(slog.NewTextHandler(discardTextHandler{}, nil)), })) - http.Handle("/transcoding/", http.StripPrefix("/transcoding/", handler)) - http.Handle("/transcoding", http.StripPrefix("/transcoding", handler)) + http.Handle("/recording/", http.StripPrefix("/recording/", handler)) + http.Handle("/recording", http.StripPrefix("/recording", handler)) go func() { + logger.Info("started to listen") + defer logger.Info("server has finished") // #nosec var err = http.ListenAndServe(conf.ServeURL, nil) if err != nil { diff --git a/go.mod b/go.mod index 89419cb728..3775572bbf 100644 --- a/go.mod +++ b/go.mod @@ -3,42 +3,42 @@ module github.com/huly-stream go 1.23.2 require ( - github.com/aws/aws-sdk-go-v2 v1.32.3 - github.com/aws/aws-sdk-go-v2/config v1.28.1 - github.com/aws/aws-sdk-go-v2/credentials v1.17.42 - github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2 - github.com/aws/smithy-go v1.22.0 + github.com/aws/aws-sdk-go-v2 v1.36.1 + github.com/aws/aws-sdk-go-v2/config v1.29.6 + github.com/aws/aws-sdk-go-v2/credentials v1.17.59 + github.com/aws/aws-sdk-go-v2/service/s3 v1.77.0 + github.com/aws/smithy-go v1.22.3 github.com/fsnotify/fsnotify v1.8.0 github.com/google/uuid v1.6.0 github.com/kelseyhightower/envconfig v1.4.0 github.com/pkg/errors v0.9.1 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/tus/tusd/v2 v2.6.0 - github.com/valyala/fasthttp v1.58.0 + github.com/valyala/fasthttp v1.59.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df + golang.org/x/exp v0.0.0-20250215185904-eff6e970281f ) require ( github.com/andybalholm/brotli v1.1.1 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.9 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/klauspost/compress v1.17.11 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/sys v0.27.0 // indirect + golang.org/x/sys v0.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 11e9c85d56..1ffa95ca7c 100644 --- a/go.sum +++ b/go.sum @@ -2,42 +2,42 @@ github.com/Acconut/go-httptest-recorder v1.0.0 h1:TAv2dfnqp/l+SUvIaMAUK4GeN4+wqb github.com/Acconut/go-httptest-recorder v1.0.0/go.mod h1:CwQyhTH1kq/gLyWiRieo7c0uokpu3PXeyF/nZjUNtmM= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/aws/aws-sdk-go-v2 v1.32.3 h1:T0dRlFBKcdaUPGNtkBSwHZxrtis8CQU17UpNBZYd0wk= -github.com/aws/aws-sdk-go-v2 v1.32.3/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= -github.com/aws/aws-sdk-go-v2/config v1.28.1 h1:oxIvOUXy8x0U3fR//0eq+RdCKimWI900+SV+10xsCBw= -github.com/aws/aws-sdk-go-v2/config v1.28.1/go.mod h1:bRQcttQJiARbd5JZxw6wG0yIK3eLeSCPdg6uqmmlIiI= -github.com/aws/aws-sdk-go-v2/credentials v1.17.42 h1:sBP0RPjBU4neGpIYyx8mkU2QqLPl5u9cmdTWVzIpHkM= -github.com/aws/aws-sdk-go-v2/credentials v1.17.42/go.mod h1:FwZBfU530dJ26rv9saAbxa9Ej3eF/AK0OAY86k13n4M= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 h1:68jFVtt3NulEzojFesM/WVarlFpCaXLKaBxDpzkQ9OQ= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18/go.mod h1:Fjnn5jQVIo6VyedMc0/EhPpfNlPl7dHV916O6B+49aE= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 h1:Jw50LwEkVjuVzE1NzkhNKkBf9cRN7MtE1F/b2cOKTUM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22/go.mod h1:Y/SmAyPcOTmpeVaWSzSKiILfXTVJwrGmYZhcRbhWuEY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 h1:981MHwBaRZM7+9QSR6XamDzF/o7ouUGxFzr+nVSIhrs= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22/go.mod h1:1RA1+aBEfn+CAB/Mh0MB6LsdCYCnjZm7tKXtnk499ZQ= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 h1:yV+hCAHZZYJQcwAaszoBNwLbPItHvApxT0kVIw6jRgs= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22/go.mod h1:kbR1TL8llqB1eGnVbybcA4/wgScxdylOdyAd51yxPdw= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 h1:kT6BcZsmMtNkP/iYMcRG+mIEA/IbeiUimXtGmqF39y0= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3/go.mod h1:Z8uGua2k4PPaGOYn66pK02rhMrot3Xk3tpBuUFPomZU= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 h1:qcxX0JYlgWH3hpPUnd6U0ikcl6LLA9sLkXE2w1fpMvY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3/go.mod h1:cLSNEmI45soc+Ef8K/L+8sEA3A3pYFEYf5B5UI+6bH4= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 h1:ZC7Y/XgKUxwqcdhO5LE8P6oGP1eh6xlQReWNKfhvJno= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3/go.mod h1:WqfO7M9l9yUAw0HcHaikwRd/H6gzYdz7vjejCA5e2oY= -github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2 h1:p9TNFL8bFUMd+38YIpTAXpoxyz0MxC7FlbFEH4P4E1U= -github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2/go.mod h1:fNjyo0Coen9QTwQLWeV6WO2Nytwiu+cCcWaTdKCAqqE= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 h1:UTpsIf0loCIWEbrqdLb+0RxnTXfWh2vhw4nQmFi4nPc= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.3/go.mod h1:FZ9j3PFHHAR+w0BSEjK955w5YD2UwB/l/H0yAK3MJvI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 h1:2YCmIXv3tmiItw0LlYf6v7gEHebLY45kBEnPezbUKyU= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3/go.mod h1:u19stRyNPxGhj6dRm+Cdgu6N75qnbW7+QN0q0dsAk58= -github.com/aws/aws-sdk-go-v2/service/sts v1.32.3 h1:wVnQ6tigGsRqSWDEEyH6lSAJ9OyFUsSnbaUWChuSGzs= -github.com/aws/aws-sdk-go-v2/service/sts v1.32.3/go.mod h1:VZa9yTFyj4o10YGsmDO4gbQJUvvhY72fhumT8W4LqsE= -github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= -github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/aws/aws-sdk-go-v2 v1.36.1 h1:iTDl5U6oAhkNPba0e1t1hrwAo02ZMqbrGq4k5JBWM5E= +github.com/aws/aws-sdk-go-v2 v1.36.1/go.mod h1:5PMILGVKiW32oDzjj6RU52yrNrDPUHcbZQYr1sM7qmM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.9 h1:VZPDrbzdsU1ZxhyWrvROqLY0nxFWgMCAzhn/nYz3X48= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.9/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg= +github.com/aws/aws-sdk-go-v2/config v1.29.6 h1:fqgqEKK5HaZVWLQoLiC9Q+xDlSp+1LYidp6ybGE2OGg= +github.com/aws/aws-sdk-go-v2/config v1.29.6/go.mod h1:Ft+WLODzDQmCTHDvqAH1JfC2xxbZ0MxpZAcJqmE1LTQ= +github.com/aws/aws-sdk-go-v2/credentials v1.17.59 h1:9btwmrt//Q6JcSdgJOLI98sdr5p7tssS9yAsGe8aKP4= +github.com/aws/aws-sdk-go-v2/credentials v1.17.59/go.mod h1:NM8fM6ovI3zak23UISdWidyZuI1ghNe2xjzUZAyT+08= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 h1:KwsodFKVQTlI5EyhRSugALzsV6mG/SGrdjlMXSZSdso= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28/go.mod h1:EY3APf9MzygVhKuPXAc5H+MkGb8k/DOSQjWS0LgkKqI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 h1:BjUcr3X3K0wZPGFg2bxOWW3VPN8rkE3/61zhP+IHviA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32/go.mod h1:80+OGC/bgzzFFTUmcuwD0lb4YutwQeKLFpmt6hoWapU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 h1:m1GeXHVMJsRsUAqG6HjZWx9dj7F5TR+cF1bjyfYyBd4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32/go.mod h1:IitoQxGfaKdVLNg0hD8/DXmAqNy0H4K2H2Sf91ti8sI= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 h1:Pg9URiobXy85kgFev3og2CuOZ8JZUBENF+dcgWBaYNk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 h1:OIHj/nAhVzIXGzbAE+4XmZ8FPvro3THr6NlqErJc3wY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32/go.mod h1:LiBEsDo34OJXqdDlRGsilhlIiXR7DL+6Cx2f4p1EgzI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.0 h1:kT2WeWcFySdYpPgyqJMSUE7781Qucjtn6wBvrgm9P+M= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.0/go.mod h1:WYH1ABybY7JK9TITPnk6ZlP7gQB8psI4c9qDmMsnLSA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 h1:SYVGSFQHlchIcy6e7x12bsrxClCXSP5et8cqVhL8cuw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13/go.mod h1:kizuDaLX37bG5WZaoxGPQR/LNFXpxp0vsUnqfkWXfNE= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 h1:OBsrtam3rk8NfBEq7OLOMm5HtQ9Yyw32X4UQMya/wjw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13/go.mod h1:3U4gFA5pmoCOja7aq4nSaIAGbaOHv2Yl2ug018cmC+Q= +github.com/aws/aws-sdk-go-v2/service/s3 v1.77.0 h1:RCOi1rDmLqOICym/6UeS2cqKED4T4m966w2rl1HfL+g= +github.com/aws/aws-sdk-go-v2/service/s3 v1.77.0/go.mod h1:VC4EKSHqT3nzOcU955VWHMGsQ+w67wfAUBSjC8NOo8U= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 h1:/eE3DogBjYlvlbhd2ssWyeuovWunHLxfgw3s/OJa4GQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.15/go.mod h1:2PCJYpi7EKeA5SkStAmZlF6fi0uUABuhtF8ILHjGc3Y= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 h1:M/zwXiL2iXUrHputuXgmO94TVNmcenPHxgLXLutodKE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14/go.mod h1:RVwIw3y/IqxC2YEXSIkAzRDdEU1iRabDPaYjpGCbCGQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 h1:TzeR06UCMUq+KA3bDkujxK1GVGy+G8qQN/QVYzGLkQE= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.14/go.mod h1:dspXf/oYWGWo6DEvj98wpaTeqt5+DMidZD0A9BYTizc= +github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= +github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= @@ -54,14 +54,14 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tus/tusd/v2 v2.6.0 h1:Je243QDKnFTvm/WkLH2bd1oQ+7trolrflRWyuI0PdWI= github.com/tus/tusd/v2 v2.6.0/go.mod h1:1Eb1lBoSRBfYJ/mQfFVjyw8ZdNMdBqW17vgQKl3Ah9g= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.58.0 h1:GGB2dWxSbEprU9j0iMJHgdKYJVDyjrOwF9RE59PbRuE= -github.com/valyala/fasthttp v1.58.0/go.mod h1:SYXvHHaFp7QZHGKSHmoMipInhrI5StHrhDTYVEjK/Kw= +github.com/valyala/fasthttp v1.59.0 h1:Qu0qYHfXvPk1mSLNqcFtEk6DpxgA26hy6bmydotDpRI= +github.com/valyala/fasthttp v1.59.0/go.mod h1:GTxNb9Bc6r2a9D0TWNSPwDz78UxnTGBViY3xZNEqyYU= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -70,14 +70,14 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= -golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= -golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/exp v0.0.0-20250215185904-eff6e970281f h1:oFMYAjX0867ZD2jcNiLBrI9BdpmEkvPyi5YrBGXbamg= +golang.org/x/exp v0.0.0-20250215185904-eff6e970281f/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk= +golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= +golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 1ef90f2192..9c296309c6 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -16,23 +16,24 @@ package config import ( "net/url" + "time" "github.com/kelseyhightower/envconfig" ) // Config represents configuration for the huly-stream application. type Config struct { - SecretToken string `split_words:"true" desc:"secret token for authorize requests"` - LogLevel string `split_words:"true" default:"debug" desc:"sets log level for the application"` - PprofEnabled bool `default:"false" split_words:"true" desc:"starts profile server on localhost:6060 if true"` - Insecure bool `default:"false" desc:"ignores authorization check if true"` - ServeURL string `split_words:"true" desc:"app listen url" default:"0.0.0.0:1080"` - EndpointURL *url.URL `split_words:"true" desc:"S3 or Datalake endpoint, example: s3://my-ip-address, datalake://my-ip-address"` - MaxCapacity int64 `split_words:"true" default:"6220800" desc:"represents the amount of maximum possible capacity for the transcoding. The default value is 1920 * 1080 * 3."` - MaxThreads int `split_words:"true" default:"4" desc:"means upper bound for the transcoing provider."` - OutputDir string `split_words:"true" default:"/tmp/transcoing/" desc:"path to the directory with transcoding result."` - RemoveContentOnUpload bool `split_words:"true" default:"true" desc:"deletes all content when content delivered if true"` - UploadRawContent bool `split_words:"true" default:"false" desc:"uploads content in raw quality to the endpoint if true"` + SecretToken string `split_words:"true" desc:"secret token for authorize requests"` + LogLevel string `split_words:"true" default:"debug" desc:"sets log level for the application"` + PprofEnabled bool `default:"false" split_words:"true" desc:"starts profile server on localhost:6060 if true"` + Insecure bool `default:"false" desc:"ignores authorization check if true"` + ServeURL string `split_words:"true" desc:"app listen url" default:"0.0.0.0:1080"` + EndpointURL *url.URL `split_words:"true" desc:"S3 or Datalake endpoint, example: s3://my-ip-address, datalake://my-ip-address"` + AuthURL *url.URL `split_words:"true" desc:"url to auth the upload"` + MaxCapacity int64 `split_words:"true" default:"6220800" desc:"represents the amount of maximum possible capacity for the transcoding. The default value is 1920 * 1080 * 3."` + MaxThreads int `split_words:"true" default:"4" desc:"means upper bound for the transcoing provider."` + OutputDir string `split_words:"true" default:"/tmp/transcoing/" desc:"path to the directory with transcoding result."` + Timeout time.Duration `default:"5m" desc:"timeout for the upload"` } // FromEnv creates new Config from env @@ -47,5 +48,9 @@ func FromEnv() (*Config, error) { return nil, err } + if *result.EndpointURL == (url.URL{}) { + result.EndpointURL = nil + } + return &result, nil } diff --git a/internal/pkg/manifest/hls.go b/internal/pkg/manifest/hls.go index 712e97f072..d8dd7a5361 100644 --- a/internal/pkg/manifest/hls.go +++ b/internal/pkg/manifest/hls.go @@ -15,111 +15,45 @@ package manifest import ( - "bufio" "fmt" - "strconv" + "os" + "path/filepath" "strings" + + "github.com/huly-stream/internal/pkg/resconv" ) -// HLSManifest represents an HLS manifest file -// with metadata about the playlist and its segments. -type HLSManifest struct { - Version int - TargetDuration int - SequenceNumber int - Segments []Segment - EndList bool -} +// GenerateHLSPlaylist generates master file for master files for resolution levels +func GenerateHLSPlaylist(levels []string, outputPath, uploadID string) error { + p := filepath.Join(outputPath, uploadID, fmt.Sprintf("%v_master.m3u8", uploadID)) + d := filepath.Dir(p) + _ = os.MkdirAll(d, os.ModePerm) + // #nosec + file, err := os.Create(p) + if err != nil { + return err + } + defer func() { _ = file.Close() }() -// Segment represents a media segment in the HLS manifest. -type Segment struct { - URI string - Duration float64 - Title string -} - -// ToM3U8 serializes the HLSManifest to an M3U8 file format. -func (m *HLSManifest) ToM3U8() string { - var builder strings.Builder - - builder.WriteString("#EXTM3U\n") - builder.WriteString(fmt.Sprintf("#EXT-X-VERSION:%d\n", m.Version)) - builder.WriteString(fmt.Sprintf("#EXT-X-TARGETDURATION:%d\n", m.TargetDuration)) - builder.WriteString(fmt.Sprintf("#EXT-X-MEDIA-SEQUENCE:%d\n", m.SequenceNumber)) - - for _, segment := range m.Segments { - if segment.Title != "" { - builder.WriteString(fmt.Sprintf("#EXTINF:%.2f,%s\n", segment.Duration, segment.Title)) - } else { - builder.WriteString(fmt.Sprintf("#EXTINF:%.2f,\n", segment.Duration)) - } - builder.WriteString(fmt.Sprintf("%s\n", segment.URI)) + _, err = file.WriteString("#EXTM3U\n") + if err != nil { + return err } - if m.EndList { - builder.WriteString("#EXT-X-ENDLIST\n") - } + for _, res := range levels { + var bandwidth = resconv.Bandwidth(res) + var resolution = strings.ReplaceAll(resconv.Resolution(res), ":", "x") - return builder.String() -} - -// FromM3U8 converts raw input to the hls master file -// nolint -func FromM3U8(data string) (*HLSManifest, error) { - scanner := bufio.NewScanner(strings.NewReader(data)) - manifest := &HLSManifest{} - var currentSegment *Segment - - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" { - continue + _, err = file.WriteString(fmt.Sprintf("#EXT-X-STREAM-INF:BANDWIDTH=%d,RESOLUTION=%v\n", bandwidth, resolution)) + if err != nil { + return err } - if strings.HasPrefix(line, "#EXTM3U") { - continue - } - if strings.HasPrefix(line, "#EXT-X-VERSION:") { - version, err := strconv.Atoi(strings.TrimPrefix(line, "#EXT-X-VERSION:")) - if err != nil { - return nil, err - } - manifest.Version = version - } else if strings.HasPrefix(line, "#EXT-X-TARGETDURATION:") { - targetDuration, err := strconv.Atoi(strings.TrimPrefix(line, "#EXT-X-TARGETDURATION:")) - if err != nil { - return nil, err - } - manifest.TargetDuration = targetDuration - } else if strings.HasPrefix(line, "#EXT-X-MEDIA-SEQUENCE:") { - sequenceNumber, err := strconv.Atoi(strings.TrimPrefix(line, "#EXT-X-MEDIA-SEQUENCE:")) - if err != nil { - return nil, err - } - manifest.SequenceNumber = sequenceNumber - } else if strings.HasPrefix(line, "#EXTINF:") { - parts := strings.SplitN(strings.TrimPrefix(line, "#EXTINF:"), ",", 2) - duration, err := strconv.ParseFloat(parts[0], 64) - if err != nil { - return nil, err - } - title := "" - if len(parts) > 1 { - title = parts[1] - } - currentSegment = &Segment{Duration: duration, Title: title} - } else if strings.HasPrefix(line, "#EXT-X-ENDLIST") { - manifest.EndList = true - } else if currentSegment != nil { - currentSegment.URI = line - manifest.Segments = append(manifest.Segments, *currentSegment) - currentSegment = nil + _, err = file.WriteString(fmt.Sprintf("%s_%s_master.m3u8\n", uploadID, res)) + if err != nil { + return err } } - if err := scanner.Err(); err != nil { - return nil, err - } - - return manifest, nil + return nil } diff --git a/internal/pkg/manifest/hls_test.go b/internal/pkg/manifest/hls_test.go index 15f3dbe022..624e3a27a9 100644 --- a/internal/pkg/manifest/hls_test.go +++ b/internal/pkg/manifest/hls_test.go @@ -14,130 +14,38 @@ package manifest_test import ( + "os" + "path/filepath" "testing" "github.com/huly-stream/internal/pkg/manifest" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestToM3U8(t *testing.T) { - tests := []struct { - name string - manifest manifest.HLSManifest - expected string - }{ - { - name: "simple manifest", - manifest: manifest.HLSManifest{ - Version: 3, - TargetDuration: 10, - SequenceNumber: 1, - Segments: []manifest.Segment{ - {URI: "segment1.ts", Duration: 9.5, Title: "Segment 1"}, - {URI: "segment2.ts", Duration: 9.0, Title: "Segment 2"}, - }, - EndList: true, - }, - expected: `#EXTM3U -#EXT-X-VERSION:3 -#EXT-X-TARGETDURATION:10 -#EXT-X-MEDIA-SEQUENCE:1 -#EXTINF:9.50,Segment 1 -segment1.ts -#EXTINF:9.00,Segment 2 -segment2.ts -#EXT-X-ENDLIST -`, - }, - { - name: "empty manifest", - manifest: manifest.HLSManifest{ - Version: 3, - TargetDuration: 10, - SequenceNumber: 1, - Segments: []manifest.Segment{}, - EndList: false, - }, - expected: `#EXTM3U -#EXT-X-VERSION:3 -#EXT-X-TARGETDURATION:10 -#EXT-X-MEDIA-SEQUENCE:1 -`, - }, +func TestGenerateHLSPlaylist(t *testing.T) { + resolutions := []string{"320p", "480p", "720p", "1080p", "4k", "8k"} + uploadID := "test123" + + err := manifest.GenerateHLSPlaylist(resolutions, "", uploadID) + require.NoError(t, err) + + outputPath := filepath.Join(uploadID, uploadID+"_master.m3u8") + + _, err = os.Stat(outputPath) + require.NoError(t, err, "Master playlist file should exist") + + // #nosec + data, err := os.ReadFile(outputPath) + require.NoError(t, err, "Error reading the generated file") + + playlistContent := string(data) + + require.Contains(t, playlistContent, "#EXTM3U", "File must start with #EXTM3U") + + for _, res := range resolutions { + expectedLine := uploadID + "_" + res + "_master.m3u8" + require.Contains(t, playlistContent, expectedLine, "Missing expected reference: "+expectedLine) } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - actual := tt.manifest.ToM3U8() - assert.Equal(t, tt.expected, actual) - }) - } -} - -func TestFromM3U8(t *testing.T) { - tests := []struct { - name string - data string - expected manifest.HLSManifest - err bool - }{ - { - name: "valid manifest", - data: `#EXTM3U -#EXT-X-VERSION:3 -#EXT-X-TARGETDURATION:10 -#EXT-X-MEDIA-SEQUENCE:1 -#EXTINF:9.50,Segment 1 -segment1.ts -#EXTINF:9.00,Segment 2 -segment2.ts -#EXT-X-ENDLIST -`, - expected: manifest.HLSManifest{ - Version: 3, - TargetDuration: 10, - SequenceNumber: 1, - Segments: []manifest.Segment{ - {URI: "segment1.ts", Duration: 9.5, Title: "Segment 1"}, - {URI: "segment2.ts", Duration: 9.0, Title: "Segment 2"}, - }, - EndList: true, - }, - err: false, - }, - // { - // name: "missing target duration", - // data: `#EXTM3U - // #EXT-X-VERSION:3 - // #EXT-X-MEDIA-SEQUENCE:1 - // #EXTINF:9.50,Segment 1 - // segment1.ts - // `, - // expected: manifest.HLSManifest{}, - // err: true, - // }, - { - name: "empty file", - data: "", - expected: manifest.HLSManifest{ - Version: 0, - TargetDuration: 0, - SequenceNumber: 0, - EndList: false, - }, - err: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - actual, err := manifest.FromM3U8(tt.data) - if tt.err { - assert.Error(t, err) - } else { - assert.NoError(t, err) - assert.Equal(t, tt.expected, *actual) - } - }) - } + _ = os.RemoveAll(uploadID) } diff --git a/internal/pkg/resconv/resconv.go b/internal/pkg/resconv/resconv.go new file mode 100644 index 0000000000..4e33a8db44 --- /dev/null +++ b/internal/pkg/resconv/resconv.go @@ -0,0 +1,129 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package resconv implements conversions to and from string representations of video resolutions. +package resconv + +import ( + "sort" + "strconv" + "strings" +) + +const defaultLevel = "320p" + +var prefixes = []struct { + pixels int + label string +}{ + {pixels: 640 * 480, label: "320p"}, + {pixels: 1280 * 720, label: "480p"}, + {pixels: 1920 * 1080, label: "720p"}, + {pixels: 2560 * 1440, label: "1080p"}, + {pixels: 3840 * 2160, label: "2k"}, + {pixels: 5120 * 2880, label: "4k"}, + {pixels: 7680 * 4320, label: "5k"}, +} + +var bandwidthMap = map[string]int{ + "320p": 300000, + "360p": 500000, + "480p": 2000000, + "720p": 5000000, + "1080p": 8000000, + "1440p": 16000000, + "4k": 25000000, + "8k": 50000000, +} + +var resolutions = map[string]string{ + "320p": "480:240", + "480p": "640:480", + "720p": "1280:720", + "1080p": "1920:1080", + "2k": "2048:1080", + "4k": "3840:2160", + "5k": "5120:2880", + "8k": "7680:4320", +} + +// SubLevels returns sublevels for the resolution +func SubLevels(resolution string) (res []string) { + var pixels = Pixels(resolution) + var idx = sort.Search(len(prefixes), func(i int) bool { + return pixels < prefixes[i].pixels + }) + if idx < 2 { + return res + } + + idx-- + idx = min(idx, 3) + + for idx >= 1 { + res = append(res, prefixes[idx].label) + idx-- + if len(res) == 2 { + break + } + } + + return res +} + +// Resolution returns default resolution based on the level +func Resolution(level string) string { + if v, ok := resolutions[level]; ok { + return v + } + return Resolution(defaultLevel) +} + +// Level converts the resolution to short prefix +func Level(resolution string) string { + var pixels = Pixels(resolution) + idx := sort.Search(len(prefixes), func(i int) bool { + return pixels < prefixes[i].pixels + }) + if idx == len(prefixes) { + return "8k" + } + + return prefixes[idx].label +} + +// Pixels returns amount of pixels for the resolution +func Pixels(resolution string) int { + var parts = strings.Split(resolution, ":") + var w, h = 420, 240 + + if len(parts) > 1 { + var _w, _ = strconv.Atoi(parts[0]) + var _h, _ = strconv.Atoi(parts[1]) + w = max(w, _w) + h = max(h, _h) + } + + return w * h +} + +// Bandwidth returns default bandwidth for the resolution +func Bandwidth(resolution string) int { + if v, ok := bandwidthMap[resolution]; ok { + return v + } + + return bandwidthMap[defaultLevel] +} diff --git a/internal/pkg/resconv/resconv_test.go b/internal/pkg/resconv/resconv_test.go new file mode 100644 index 0000000000..4139e21370 --- /dev/null +++ b/internal/pkg/resconv/resconv_test.go @@ -0,0 +1,120 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package resconv_test + +import ( + "testing" + + "github.com/huly-stream/internal/pkg/resconv" + "github.com/stretchr/testify/require" +) + +func Test_Resconv_ShouldReturnCorrectPrefix(t *testing.T) { + tests := []struct { + res string + expected string + }{ + {res: "320:240", expected: "320p"}, + {res: "640:480", expected: "480p"}, + {res: "1280:720", expected: "720p"}, + {res: "1920:1080", expected: "1080p"}, + {res: "2560:1440", expected: "2k"}, + {res: "3840:2160", expected: "4k"}, + {res: "5120:2880", expected: "5k"}, + {res: "9000:4000", expected: "8k"}, + } + + for _, tt := range tests { + t.Run(tt.expected, func(t *testing.T) { + result := resconv.Level(tt.res) + require.Equal(t, tt.expected, result, "ResolutionFromPixels(%d)", tt.res) + }) + } +} + +func Test_Resconv_ShouldReturnCorrectPrefixes(t *testing.T) { + tests := []struct { + name string + res string + expected []string + }{ + { + name: "pixels below smallest resolution", + res: "640:479", + expected: nil, + }, + { + name: "pixels equal to smallest resolution", + res: "640:480", + expected: nil, + }, + { + name: "pixels just above smallest resolution", + res: "641:480", + expected: nil, + }, + { + name: "pixels equal to 720p", + res: "1280:720", + expected: []string{"480p"}, + }, + { + name: "pixels just above 480p", + res: "1280:721", + expected: []string{"480p"}, + }, + { + name: "pixels equal to 1k", + res: "1920:1080", + expected: []string{"720p", "480p"}, + }, + { + name: "pixels just above 1k", + res: "1920:1081", + expected: []string{"720p", "480p"}, + }, + { + name: "pixels equal to 1k", + res: "2560:1440", + expected: []string{"1080p", "720p"}, + }, + { + name: "pixels equal to 2k", + res: "3840:2160", + expected: []string{"1080p", "720p"}, + }, + { + name: "pixels equal to 4k", + res: "5120:2160", + expected: []string{"1080p", "720p"}, + }, + { + name: "pixels equal to 5k", + res: "7680:4320", + expected: []string{"1080p", "720p"}, + }, + { + name: "pixels above largest resolution", + res: "7681:4320", + expected: []string{"1080p", "720p"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := resconv.SubLevels(tt.res) + require.Equal(t, tt.expected, result, "SubResolutionsFromPixels(%d) returned unexpected result", tt.res) + }) + } +} diff --git a/internal/pkg/transcoding/command.go b/internal/pkg/transcoding/command.go index f43b5cf594..0065fa3196 100644 --- a/internal/pkg/transcoding/command.go +++ b/internal/pkg/transcoding/command.go @@ -22,61 +22,29 @@ import ( "os" "os/exec" "path/filepath" - "sort" - "strconv" - "strings" "github.com/pkg/errors" "github.com/huly-stream/internal/pkg/log" + "github.com/huly-stream/internal/pkg/resconv" "go.uber.org/zap" ) // Options represents configuration for the ffmpeg command type Options struct { - OuputDir string - Resolutions []string - Threads int - UploadID string + OuputDir string + ScalingLevels []string + Level string + Threads int + UploadID string } -func measure(options *Options) int64 { - var res int64 - for _, resolution := range options.Resolutions { - var w, h int - var parts = strings.Split(resolution, ":") - - if len(parts) > 1 { - w, _ = strconv.Atoi(parts[0]) - w = max(w, 320) - h, _ = strconv.Atoi(parts[1]) - h = max(h, 240) - - res += int64(w) * int64(h) - } - } - - return max(res, 320*240) -} - -func newFfmpegCommand(ctx context.Context, in io.Reader, options *Options) (*exec.Cmd, error) { - if options == nil { - return nil, errors.New("options should not be nil") - } +func newFfmpegCommand(ctx context.Context, in io.Reader, args []string) (*exec.Cmd, error) { if ctx == nil { return nil, errors.New("ctx should not be nil") } - var logger = log.FromContext(ctx).With(zap.String("func", "NewFFMpegCommand")) - var args []string - - if options.Resolutions == nil { - logger.Debug("resolutions were not provided, building audio command...") - args = BuildAudioCommand(options) - } else { - logger.Debug("building video command...") - args = BuildVideoCommand(options) - } + var logger = log.FromContext(ctx).With(zap.String("func", "newFFMpegCommand")) logger.Debug("prepared command: ", zap.Strings("args", args)) @@ -90,6 +58,7 @@ func newFfmpegCommand(ctx context.Context, in io.Reader, options *Options) (*exe func buildCommonComamnd(opts *Options) []string { return []string{ + "-nostdin", "-threads", fmt.Sprint(opts.Threads), "-i", "pipe:0", } @@ -105,25 +74,25 @@ func BuildAudioCommand(opts *Options) []string { ) } -// BuildVideoCommand returns flags for ffmpeg for video transcoding -func BuildVideoCommand(opts *Options) []string { +// BuildRawVideoCommand returns an extremely lightweight ffmpeg command for converting raw video without extra cost. +func BuildRawVideoCommand(opts *Options) []string { + return append(buildCommonComamnd(opts), + "-c:v", + "copy", + "-fps_mode", + "vfr", + "-hls_time", "5", + "-hls_list_size", "0", + "-hls_segment_filename", filepath.Join(opts.OuputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", opts.Level)), + filepath.Join(opts.OuputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, opts.Level))) +} + +// BuildScalingVideoCommand returns flags for ffmpeg for video scaling +func BuildScalingVideoCommand(opts *Options) []string { var result = buildCommonComamnd(opts) - - for _, res := range opts.Resolutions { - var prefix string - var w, h int - var parts = strings.Split(res, ":") - - if len(parts) > 1 { - w, _ = strconv.Atoi(parts[0]) - h, _ = strconv.Atoi(parts[1]) - } - w = max(w, 640) - h = max(h, 480) - prefix = ResolutionFromPixels(w * h) - + for _, level := range opts.ScalingLevels { result = append(result, - "-vf", fmt.Sprintf("scale=%d:%d", w, h), + "-vf", "scale="+resconv.Resolution(level), "-c:v", "libx264", "-preset", "veryfast", @@ -131,33 +100,9 @@ func BuildVideoCommand(opts *Options) []string { "-g", "60", "-hls_time", "5", "-hls_list_size", "0", - "-hls_segment_filename", filepath.Join(opts.OuputDir, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", prefix)), - filepath.Join(opts.OuputDir, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, prefix))) + "-hls_segment_filename", filepath.Join(opts.OuputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", level)), + filepath.Join(opts.OuputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, level))) } return result } - -var resolutions = []struct { - pixels int - label string -}{ - {pixels: 640 * 480, label: "320p"}, - {pixels: 1280 * 720, label: "480p"}, - {pixels: 1920 * 1080, label: "720p"}, - {pixels: 2560 * 1440, label: "1k"}, - {pixels: 3840 * 2160, label: "2k"}, - {pixels: 5120 * 2160, label: "4k"}, - {pixels: 7680 * 4320, label: "5k"}, -} - -// ResolutionFromPixels converts pixel count to short string -func ResolutionFromPixels(pixels int) string { - idx := sort.Search(len(resolutions), func(i int) bool { - return pixels < resolutions[i].pixels - }) - if idx == len(resolutions) { - return "8k" - } - return resolutions[idx].label -} diff --git a/internal/pkg/transcoding/command_test.go b/internal/pkg/transcoding/command_test.go index 14a7db515d..719c38467a 100644 --- a/internal/pkg/transcoding/command_test.go +++ b/internal/pkg/transcoding/command_test.go @@ -14,49 +14,36 @@ package transcoding_test import ( - "runtime" "strings" "testing" + "github.com/huly-stream/internal/pkg/resconv" "github.com/huly-stream/internal/pkg/transcoding" "github.com/stretchr/testify/require" ) -func Test_BuildVideoCommand_Basic(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip() - } - var simpleHlsCommand = transcoding.BuildVideoCommand(&transcoding.Options{ - OuputDir: "test", - UploadID: "1", - Threads: 4, - Resolutions: []string{"1280:720"}, +func Test_BuildVideoCommand_Scaling(t *testing.T) { + var scaleCommand = transcoding.BuildScalingVideoCommand(&transcoding.Options{ + OuputDir: "test", + UploadID: "1", + Threads: 4, + ScalingLevels: []string{"720p", "480p"}, }) - const expected = `-threads 4 -i pipe:0 -vf scale=1280:720 -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_list_size 0 -hls_segment_filename test/1_%03d_720p.ts test/1_720p_master.m3u8` + const expected = `-nostdin -threads 4 -i pipe:0 -vf scale=1280:720 -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -vf scale=640:480 -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` - require.Contains(t, expected, strings.Join(simpleHlsCommand, " ")) + require.Contains(t, expected, strings.Join(scaleCommand, " ")) } -func TestResolutionFromPixels(t *testing.T) { - tests := []struct { - pixels int - expected string - }{ - {pixels: 320 * 240, expected: "320p"}, - {pixels: 640 * 480, expected: "480p"}, - {pixels: 1280 * 720, expected: "720p"}, - {pixels: 1920 * 1080, expected: "1k"}, - {pixels: 2560 * 1440, expected: "2k"}, - {pixels: 3840 * 2160, expected: "4k"}, - {pixels: 5120 * 2160, expected: "5k"}, - {pixels: 9000 * 4000, expected: "8k"}, - } +func Test_BuildVideoCommand_Raw(t *testing.T) { + var rawCommand = transcoding.BuildRawVideoCommand(&transcoding.Options{ + OuputDir: "test", + UploadID: "1", + Threads: 4, + Level: resconv.Level("651:490"), + }) - for _, tt := range tests { - t.Run(tt.expected, func(t *testing.T) { - result := transcoding.ResolutionFromPixels(tt.pixels) - require.Equal(t, tt.expected, result, "ResolutionFromPixels(%d)", tt.pixels) - }) - } + const expected = `-nostdin -threads 4 -i pipe:0 -c:v copy -fps_mode vfr -hls_time 5 -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + + require.Contains(t, expected, strings.Join(rawCommand, " ")) } diff --git a/internal/pkg/transcoding/scheduler.go b/internal/pkg/transcoding/scheduler.go index 25872a3047..080aeedbb5 100644 --- a/internal/pkg/transcoding/scheduler.go +++ b/internal/pkg/transcoding/scheduler.go @@ -17,14 +17,15 @@ package transcoding import ( "context" - "strings" "sync" + "time" "github.com/pkg/errors" "github.com/google/uuid" "github.com/huly-stream/internal/pkg/config" "github.com/huly-stream/internal/pkg/log" + "github.com/huly-stream/internal/pkg/resconv" "github.com/huly-stream/internal/pkg/sharedpipe" "github.com/huly-stream/internal/pkg/uploader" "github.com/tus/tusd/v2/pkg/handler" @@ -40,6 +41,7 @@ type Scheduler struct { mainContext context.Context logger *zap.Logger workers sync.Map + cancels sync.Map } // NewScheduler creates a new scheduler for transcode operations. @@ -57,58 +59,95 @@ func (s *Scheduler) NewUpload(ctx context.Context, info handler.FileInfo) (handl if info.ID == "" { info.ID = uuid.NewString() } - + s.logger.Sugar().Debugf("upload: %v", info) s.logger.Debug("NewUpload", zap.String("ID", info.ID)) - var result = &Worker{ - done: make(chan struct{}), + var worker = &Worker{ writer: sharedpipe.NewWriter(), info: info, - logger: log.FromContext(s.mainContext).With(zap.String("Worker", info.ID)), + logger: log.FromContext(s.mainContext).With(zap.String("worker", info.ID)), + done: make(chan struct{}), } - var resolutions = strings.Split(info.MetaData["resolutions"], ",") + var scaling = resconv.SubLevels(info.MetaData["resolution"]) + var level = resconv.Level(info.MetaData["resolution"]) + var cost int64 + + for _, scale := range scaling { + cost += int64(resconv.Pixels(resconv.Resolution(scale))) + } + + if !s.limiter.TryConsume(cost) { + s.logger.Debug("run out of resources for scaling") + scaling = nil + } var commandOptions = Options{ - OuputDir: s.conf.OutputDir, - Threads: s.conf.MaxThreads, - UploadID: info.ID, - Resolutions: resolutions, - } - - result.cost = measure(&commandOptions) - - if !s.limiter.TryConsume(result.cost) { - s.logger.Error("run out of resources") - return nil, errors.New("run out of resources") + OuputDir: s.conf.OutputDir, + Threads: s.conf.MaxThreads, + UploadID: info.ID, + Level: level, + ScalingLevels: scaling, } if s.conf.EndpointURL != nil { - s.logger.Debug("found endpoint url in the config, starting uploader...") - var contentUploader, err = uploader.New(s.mainContext, *s.conf, info.ID, info.MetaData) + s.logger.Sugar().Debugf("initializing uploader for %v", info) + var contentUploader, err = uploader.New(s.mainContext, s.conf.OutputDir, s.conf.EndpointURL, info) if err != nil { + s.logger.Error("can not create uploader", zap.Error(err)) return nil, err } - result.contentUploader = contentUploader + + worker.contentUploader = contentUploader go func() { - var serverErr = result.contentUploader.Serve() - result.logger.Debug("content uploader has finished", zap.Error(serverErr)) + var serverErr = worker.contentUploader.Serve() + worker.logger.Debug("content uploader has finished", zap.Error(serverErr)) }() } - - s.workers.Store(result.info.ID, result) - s.logger.Sugar().Debugf("New Upload: info %v", result.info) - if err := result.start(s.mainContext, &commandOptions); err != nil { + s.workers.Store(worker.info.ID, worker) + if err := worker.start(s.mainContext, &commandOptions); err != nil { return nil, err } - return result, nil + + go func() { + worker.wg.Wait() + s.limiter.ReturnCapacity(cost) + s.logger.Debug("returned capacity", zap.Int64("capacity", cost)) + close(worker.done) + }() + + s.logger.Debug("NewUpload", zap.String("done", info.ID)) + return worker, nil } // GetUpload returns current a worker based on upload id func (s *Scheduler) GetUpload(ctx context.Context, id string) (upload handler.Upload, err error) { if v, ok := s.workers.Load(id); ok { s.logger.Debug("GetUpload: found worker by id", zap.String("id", id)) - return v.(*Worker), nil + var w = v.(*Worker) + var cancelCtx, cancel = context.WithCancel(context.Background()) + if v, ok := s.cancels.Load(id); ok { + v.(context.CancelFunc)() + } + s.cancels.Store(id, cancel) + go func() { + select { + case <-w.done: + w.logger.Debug("upload timeout just canceled") + s.cancels.Delete(id) + return + case <-cancelCtx.Done(): + w.logger.Debug("upload refreshed") + return + case <-time.After(s.conf.Timeout): + w.logger.Debug("upload timeout") + s.cancels.Delete(id) + var terminateCtx, terminateCancel = context.WithTimeout(context.Background(), s.conf.Timeout) + defer terminateCancel() + _ = w.Terminate(terminateCtx) + } + }() + return w, nil } s.logger.Debug("GetUpload: worker not found", zap.String("id", id)) return nil, errors.New("bad id") @@ -117,8 +156,7 @@ func (s *Scheduler) GetUpload(ctx context.Context, id string) (upload handler.Up // AsTerminatableUpload returns tusd handler.TerminatableUpload func (s *Scheduler) AsTerminatableUpload(upload handler.Upload) handler.TerminatableUpload { var worker = upload.(*Worker) - s.logger.Debug("AsTerminatableUpload, trying to return capacity", zap.Int64("cost", worker.cost)) - s.limiter.ReturnCapacity(worker.cost) + s.logger.Debug("AsTerminatableUpload") return worker } diff --git a/internal/pkg/transcoding/worker.go b/internal/pkg/transcoding/worker.go index 60baccc465..1235bfcf02 100644 --- a/internal/pkg/transcoding/worker.go +++ b/internal/pkg/transcoding/worker.go @@ -17,9 +17,11 @@ package transcoding import ( "context" "io" + "sync" "github.com/pkg/errors" + "github.com/huly-stream/internal/pkg/manifest" "github.com/huly-stream/internal/pkg/sharedpipe" "github.com/huly-stream/internal/pkg/uploader" "github.com/tus/tusd/v2/pkg/handler" @@ -33,8 +35,9 @@ type Worker struct { info handler.FileInfo writer *sharedpipe.Writer reader *sharedpipe.Reader - cost int64 - done chan struct{} + + wg sync.WaitGroup + done chan struct{} } // WriteChunk calls when client sends a chunk of raw data @@ -73,7 +76,7 @@ func (w *Worker) Terminate(ctx context.Context) error { w.logger.Debug("Terminating...") if w.contentUploader != nil { go func() { - <-w.done + w.wg.Wait() w.contentUploader.Rollback() }() } @@ -94,7 +97,7 @@ func (w *Worker) FinishUpload(ctx context.Context) error { w.logger.Debug("finishing upload...") if w.contentUploader != nil { go func() { - <-w.done + w.wg.Wait() w.contentUploader.Terminate() }() } @@ -108,18 +111,49 @@ func (s *Scheduler) AsConcatableUpload(upload handler.Upload) handler.Concatable } func (w *Worker) start(ctx context.Context, options *Options) error { + defer w.logger.Debug("start done") w.reader = w.writer.Transpile() - var cmd, err = newFfmpegCommand(ctx, w.reader, options) - if err != nil { + + if err := manifest.GenerateHLSPlaylist(append(options.ScalingLevels, options.Level), options.OuputDir, options.UploadID); err != nil { return err } + + w.wg.Add(1) go func() { - defer close(w.done) - if runErr := cmd.Run(); runErr != nil { - w.logger.Error("transoding provider is exited with error", zap.Error(err)) - } else { - w.logger.Debug("transoding provider has finished without errors") + defer w.wg.Done() + var logger = w.logger.With(zap.String("command", "raw")) + defer logger.Debug("done") + + var args = BuildRawVideoCommand(options) + var convertSourceCommand, err = newFfmpegCommand(ctx, w.reader, args) + if err != nil { + logger.Debug("can not start", zap.Error(err)) + } + err = convertSourceCommand.Run() + if err != nil { + logger.Debug("finished with error", zap.Error(err)) } }() + + if len(options.ScalingLevels) > 0 { + w.wg.Add(1) + var scalingCommandReader = w.writer.Transpile() + go func() { + defer w.wg.Done() + var logger = w.logger.With(zap.String("command", "scaling")) + defer logger.Debug("done") + + var args = BuildScalingVideoCommand(options) + var convertSourceCommand, err = newFfmpegCommand(ctx, scalingCommandReader, args) + if err != nil { + logger.Debug("can not start", zap.Error(err)) + } + err = convertSourceCommand.Run() + if err != nil { + logger.Debug("finished with error", zap.Error(err)) + } + }() + } + return nil } diff --git a/internal/pkg/uploader/datalake.go b/internal/pkg/uploader/datalake.go index 43ba535efd..dfff697012 100644 --- a/internal/pkg/uploader/datalake.go +++ b/internal/pkg/uploader/datalake.go @@ -18,7 +18,6 @@ import ( "context" "io" "mime/multipart" - "net/url" "os" "path/filepath" @@ -116,6 +115,7 @@ func (d *DatalakeStorage) DeleteFile(ctx context.Context, fileName string) error client := fasthttp.Client{} if err := client.Do(req, res); err != nil { + logger.Error("failed to del", zap.Error(err)) return errors.Wrapf(err, "delete failed") } @@ -126,5 +126,5 @@ func (d *DatalakeStorage) DeleteFile(ctx context.Context, fileName string) error func getObjectKey(s string) string { var _, objectKey = filepath.Split(s) - return url.QueryEscape(objectKey) + return objectKey } diff --git a/internal/pkg/uploader/s3.go b/internal/pkg/uploader/s3.go index ba688618f9..faf15e22d0 100644 --- a/internal/pkg/uploader/s3.go +++ b/internal/pkg/uploader/s3.go @@ -37,6 +37,7 @@ import ( type S3Storage struct { client *s3.Client bucketName string + logger *zap.Logger } // NewS3 creates a new S3 storage @@ -44,6 +45,7 @@ func NewS3(ctx context.Context, endpoint string) Storage { var accessKeyID = os.Getenv("AWS_ACCESS_KEY_ID") var accessKeySecret = os.Getenv("AWS_SECRET_ACCESS_KEY") var bucketName = os.Getenv("AWS_BUCKET_NAME") + var logger = log.FromContext(ctx).With(zap.String("s3", "storage")) cfg, err := config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyID, accessKeySecret, "")), @@ -61,6 +63,7 @@ func NewS3(ctx context.Context, endpoint string) Storage { return &S3Storage{ client: s3Client, bucketName: bucketName, + logger: logger, } } diff --git a/internal/pkg/uploader/uploader.go b/internal/pkg/uploader/uploader.go index 86735191a4..f61a6396ee 100644 --- a/internal/pkg/uploader/uploader.go +++ b/internal/pkg/uploader/uploader.go @@ -18,52 +18,54 @@ import ( "context" "net/url" "os" + "path/filepath" "strings" "sync" "time" "github.com/pkg/errors" + "github.com/tus/tusd/v2/pkg/handler" "github.com/fsnotify/fsnotify" - "github.com/huly-stream/internal/pkg/config" "github.com/huly-stream/internal/pkg/log" "go.uber.org/zap" ) type uploader struct { - ctx context.Context - cancel context.CancelFunc - baseDir string - uploadID string - masterFiles sync.Map - postponeDuration time.Duration - sentFiles sync.Map - storage Storage - contexts sync.Map - retryCount int - removeLocalContentOnUpload bool - eventBufferCount uint - isMasterFileFunc func(s string) bool + ctx context.Context + cancel context.CancelFunc + baseDir string + uploadID string + masterFiles sync.Map + postponeDuration time.Duration + sentFiles sync.Map + storage Storage + contexts sync.Map + retryCount int + eventBufferCount uint + isMasterFileFunc func(s string) bool +} + +func (u *uploader) retry(action func() error) { + for range u.retryCount { + if err := action(); err == nil { + return + } + } } // Rollback deletes all delivered files and also deletes all local content by uploadID func (u *uploader) Rollback() { - log.FromContext(u.ctx).Debug("cancel") + logger := log.FromContext(u.ctx).With(zap.String("uploader", "Rollback")) + logger.Debug("starting") defer u.cancel() + u.sentFiles.Range(func(key, value any) bool { - log.FromContext(u.ctx).Debug("deleting remote file", zap.String("key", key.(string))) - for range u.retryCount { - var err = u.storage.DeleteFile(u.ctx, key.(string)) - if err == nil { - break - } - log.FromContext(u.ctx).Debug("can not delete file", zap.Error(err)) - } + logger.Debug("deleting remote file", zap.String("key", key.(string))) + u.retry(func() error { return u.storage.DeleteFile(u.ctx, key.(string)) }) return true }) - if !u.removeLocalContentOnUpload { - return - } + u.sentFiles.Range(func(key, value any) bool { log.FromContext(u.ctx).Debug("deleting local file", zap.String("key", key.(string))) _ = os.Remove(key.(string)) @@ -72,50 +74,47 @@ func (u *uploader) Rollback() { } func (u *uploader) Terminate() { - log.FromContext(u.ctx).Debug("terminate") + logger := log.FromContext(u.ctx).With(zap.String("uploader", "Terminate")) + logger.Debug("starting") defer u.cancel() + u.masterFiles.Range(func(key, value any) bool { log.FromContext(u.ctx).Debug("uploading master file", zap.String("key", key.(string))) - for range u.retryCount { - var uploadErr = u.storage.UploadFile(u.ctx, key.(string)) - if uploadErr == nil { - break - } - log.FromContext(u.ctx).Debug("can not upload file", zap.Error(uploadErr)) - } + go u.retry(func() error { return u.storage.UploadFile(u.ctx, key.(string)) }) return true }) - if !u.removeLocalContentOnUpload { - return - } + u.masterFiles.Range(func(key, value any) bool { - log.FromContext(u.ctx).Debug("deleting local master file", zap.String("key", key.(string))) _ = os.Remove(key.(string)) return true }) + u.sentFiles.Range(func(key, value any) bool { - log.FromContext(u.ctx).Debug("deleting local file", zap.String("key", key.(string))) _ = os.Remove(key.(string)) return true }) } func (u *uploader) Serve() error { - var logger = log.FromContext(u.ctx) - logger = logger.With(zap.String("uploader", u.uploadID), zap.String("dir", u.baseDir)) + var logger = log.FromContext(u.ctx).With(zap.String("uploader", u.uploadID), zap.String("dir", u.baseDir)) var watcher, err = fsnotify.NewBufferedWatcher(u.eventBufferCount) + if err != nil { logger.Error("can not start watcher") return err } + + _ = os.MkdirAll(u.baseDir, os.ModePerm) + if err := watcher.Add(u.baseDir); err != nil { return err } + defer func() { _ = watcher.Close() }() - logger.Debug("uploader initialized and started to watch") + logger.Debug("the uploader has initialized and started watching") for { select { @@ -123,6 +122,9 @@ func (u *uploader) Serve() error { logger.Debug("done") return u.ctx.Err() case event, ok := <-watcher.Events: + if strings.HasSuffix(event.Name, "tmp") { + continue + } if !strings.Contains(event.Name, u.uploadID) { continue } @@ -169,29 +171,32 @@ type Storage interface { } // New creates a new instance of Uplaoder -func New(ctx context.Context, conf config.Config, uploadID string, metadata map[string]string) (Uploader, error) { - var uploaderCtx, uploaderCancel = context.WithCancel(ctx) +func New(ctx context.Context, baseDir string, endpointURL *url.URL, uploadInfo handler.FileInfo) (Uploader, error) { + var uploaderCtx, uploadCancel = context.WithCancel(context.Background()) + uploaderCtx = log.WithLoggerFields(uploaderCtx) + go func() { + <-ctx.Done() + time.Sleep(time.Minute * 2) + uploadCancel() + }() var storage Storage var err error - if conf.EndpointURL != nil { - storage, err = NewStorageByURL(ctx, conf.EndpointURL, metadata) - if err != nil { - uploaderCancel() - return nil, err - } + storage, err = NewStorageByURL(ctx, endpointURL, uploadInfo.MetaData) + if err != nil { + uploadCancel() + return nil, err } return &uploader{ - ctx: uploaderCtx, - cancel: uploaderCancel, - uploadID: uploadID, - removeLocalContentOnUpload: conf.RemoveContentOnUpload, - postponeDuration: time.Second * 2, - storage: storage, - retryCount: 5, - baseDir: conf.OutputDir, - eventBufferCount: 100, + ctx: uploaderCtx, + cancel: uploadCancel, + uploadID: uploadInfo.ID, + postponeDuration: time.Second * 2, + storage: storage, + retryCount: 5, + baseDir: filepath.Join(baseDir, uploadInfo.ID), + eventBufferCount: 100, isMasterFileFunc: func(s string) bool { return strings.HasSuffix(s, "m3u8") }, From a6e428cef0c7bd71c95ccb0821ec781016d405f1 Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Tue, 25 Feb 2025 03:03:29 +0300 Subject: [PATCH 05/52] fix: correctly handle uploader termination Signed-off-by: denis-tingaikin --- cmd/huly-stream/main.go | 3 +- internal/pkg/config/config.go | 11 +++++ internal/pkg/manifest/hls.go | 5 ++ internal/pkg/transcoding/worker.go | 2 - internal/pkg/uploader/datalake.go | 2 +- internal/pkg/uploader/postpone.go | 2 + internal/pkg/uploader/s3.go | 2 +- internal/pkg/uploader/uploader.go | 73 ++++++++++++------------------ 8 files changed, 51 insertions(+), 49 deletions(-) diff --git a/cmd/huly-stream/main.go b/cmd/huly-stream/main.go index f660a2511f..d7c5d9690b 100644 --- a/cmd/huly-stream/main.go +++ b/cmd/huly-stream/main.go @@ -49,11 +49,12 @@ func main() { var conf = must(config.FromEnv()) logger.Sugar().Debugf("provided config is %v", conf) + logger.Sugar().Info(conf.Endpoint()) + mustNoError(os.MkdirAll(conf.OutputDir, os.ModePerm)) if conf.PprofEnabled { go pprof.ListenAndServe(ctx, "localhost:6060") } - scheduler := transcoding.NewScheduler(ctx, conf) tusComposer := tusd.NewStoreComposer() diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 9c296309c6..4eb601749e 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -54,3 +54,14 @@ func FromEnv() (*Config, error) { return &result, nil } + +func (c *Config) Endpoint() *url.URL { + var scheme = "https" + if c.Insecure { + scheme = "http" + } + return &url.URL{ + Scheme: scheme, + Host: c.EndpointURL.Host, + } +} diff --git a/internal/pkg/manifest/hls.go b/internal/pkg/manifest/hls.go index d8dd7a5361..c301e4f4f5 100644 --- a/internal/pkg/manifest/hls.go +++ b/internal/pkg/manifest/hls.go @@ -53,6 +53,11 @@ func GenerateHLSPlaylist(levels []string, outputPath, uploadID string) error { if err != nil { return err } + + _, err = file.WriteString("#EXT-X-ENDLIST") + if err != nil { + return err + } } return nil diff --git a/internal/pkg/transcoding/worker.go b/internal/pkg/transcoding/worker.go index 1235bfcf02..c2aaca3d34 100644 --- a/internal/pkg/transcoding/worker.go +++ b/internal/pkg/transcoding/worker.go @@ -113,11 +113,9 @@ func (s *Scheduler) AsConcatableUpload(upload handler.Upload) handler.Concatable func (w *Worker) start(ctx context.Context, options *Options) error { defer w.logger.Debug("start done") w.reader = w.writer.Transpile() - if err := manifest.GenerateHLSPlaylist(append(options.ScalingLevels, options.Level), options.OuputDir, options.UploadID); err != nil { return err } - w.wg.Add(1) go func() { defer w.wg.Done() diff --git a/internal/pkg/uploader/datalake.go b/internal/pkg/uploader/datalake.go index dfff697012..097bf0a6b5 100644 --- a/internal/pkg/uploader/datalake.go +++ b/internal/pkg/uploader/datalake.go @@ -37,7 +37,7 @@ type DatalakeStorage struct { // NewDatalakeStorage creates a new datalake client func NewDatalakeStorage(baseURL, workspace, token string) Storage { return &DatalakeStorage{ - baseURL: "https://" + baseURL, + baseURL: baseURL, token: token, workspace: workspace, } diff --git a/internal/pkg/uploader/postpone.go b/internal/pkg/uploader/postpone.go index 61b22b79b4..3486996ec1 100644 --- a/internal/pkg/uploader/postpone.go +++ b/internal/pkg/uploader/postpone.go @@ -19,6 +19,7 @@ import ( ) func (u *uploader) postpone(id string, action func()) { + u.wg.Add(1) var ctx, cancel = context.WithCancel(context.Background()) var startCh = time.After(u.postponeDuration) @@ -28,6 +29,7 @@ func (u *uploader) postpone(id string, action func()) { u.contexts.Store(id, &cancel) go func() { + u.wg.Done() defer cancel() select { case <-ctx.Done(): diff --git a/internal/pkg/uploader/s3.go b/internal/pkg/uploader/s3.go index faf15e22d0..016e6fc87c 100644 --- a/internal/pkg/uploader/s3.go +++ b/internal/pkg/uploader/s3.go @@ -56,8 +56,8 @@ func NewS3(ctx context.Context, endpoint string) Storage { } var s3Client = s3.NewFromConfig(cfg, func(o *s3.Options) { - endpoint = "https://" + endpoint o.BaseEndpoint = &endpoint + o.UsePathStyle = true }) return &S3Storage{ diff --git a/internal/pkg/uploader/uploader.go b/internal/pkg/uploader/uploader.go index f61a6396ee..a2c76811cd 100644 --- a/internal/pkg/uploader/uploader.go +++ b/internal/pkg/uploader/uploader.go @@ -27,23 +27,24 @@ import ( "github.com/tus/tusd/v2/pkg/handler" "github.com/fsnotify/fsnotify" + "github.com/huly-stream/internal/pkg/config" "github.com/huly-stream/internal/pkg/log" "go.uber.org/zap" ) type uploader struct { + done chan struct{} + wg sync.WaitGroup ctx context.Context cancel context.CancelFunc baseDir string uploadID string - masterFiles sync.Map postponeDuration time.Duration sentFiles sync.Map storage Storage contexts sync.Map retryCount int eventBufferCount uint - isMasterFileFunc func(s string) bool } func (u *uploader) retry(action func() error) { @@ -60,13 +61,11 @@ func (u *uploader) Rollback() { logger.Debug("starting") defer u.cancel() + u.wg.Wait() + u.sentFiles.Range(func(key, value any) bool { logger.Debug("deleting remote file", zap.String("key", key.(string))) u.retry(func() error { return u.storage.DeleteFile(u.ctx, key.(string)) }) - return true - }) - - u.sentFiles.Range(func(key, value any) bool { log.FromContext(u.ctx).Debug("deleting local file", zap.String("key", key.(string))) _ = os.Remove(key.(string)) return true @@ -78,16 +77,7 @@ func (u *uploader) Terminate() { logger.Debug("starting") defer u.cancel() - u.masterFiles.Range(func(key, value any) bool { - log.FromContext(u.ctx).Debug("uploading master file", zap.String("key", key.(string))) - go u.retry(func() error { return u.storage.UploadFile(u.ctx, key.(string)) }) - return true - }) - - u.masterFiles.Range(func(key, value any) bool { - _ = os.Remove(key.(string)) - return true - }) + u.wg.Wait() u.sentFiles.Range(func(key, value any) bool { _ = os.Remove(key.(string)) @@ -98,6 +88,7 @@ func (u *uploader) Terminate() { func (u *uploader) Serve() error { var logger = log.FromContext(u.ctx).With(zap.String("uploader", u.uploadID), zap.String("dir", u.baseDir)) var watcher, err = fsnotify.NewBufferedWatcher(u.eventBufferCount) + defer close(u.done) if err != nil { logger.Error("can not start watcher") @@ -105,6 +96,16 @@ func (u *uploader) Serve() error { } _ = os.MkdirAll(u.baseDir, os.ModePerm) + initFiles, _ := os.ReadDir(u.baseDir) + for _, f := range initFiles { + var name = filepath.Join(u.baseDir, f.Name()) + u.postpone(name, func() { + logger.Debug("started uploading", zap.String("eventName", name)) + u.retry(func() error { return u.storage.UploadFile(u.ctx, name) }) + logger.Debug("added to sentFiles", zap.String("eventName", name)) + u.sentFiles.Store(name, struct{}{}) + }) + } if err := watcher.Add(u.baseDir); err != nil { return err @@ -114,37 +115,26 @@ func (u *uploader) Serve() error { _ = watcher.Close() }() - logger.Debug("the uploader has initialized and started watching") + logger.Debug("uploader has initialized and started watching") + defer logger.Debug("done") for { select { case <-u.ctx.Done(): - logger.Debug("done") return u.ctx.Err() case event, ok := <-watcher.Events: + if !ok { + return u.ctx.Err() + } if strings.HasSuffix(event.Name, "tmp") { continue } if !strings.Contains(event.Name, u.uploadID) { continue } - if !ok { - return u.ctx.Err() - } - if u.isMasterFileFunc(event.Name) { - u.masterFiles.Store(event.Name, struct{}{}) - logger.Debug("found master file", zap.String("eventName", event.Name)) - continue - } u.postpone(event.Name, func() { - logger.Debug("started to upload", zap.String("eventName", event.Name)) - for range u.retryCount { - var uploadErr = u.storage.UploadFile(u.ctx, event.Name) - if uploadErr == nil { - break - } - logger.Error("can not upload file", zap.Error(uploadErr)) - } + logger.Debug("started uploading", zap.String("eventName", event.Name)) + u.retry(func() error { return u.storage.UploadFile(u.ctx, event.Name) }) logger.Debug("added to sentFiles", zap.String("eventName", event.Name)) u.sentFiles.Store(event.Name, struct{}{}) }) @@ -174,11 +164,7 @@ type Storage interface { func New(ctx context.Context, baseDir string, endpointURL *url.URL, uploadInfo handler.FileInfo) (Uploader, error) { var uploaderCtx, uploadCancel = context.WithCancel(context.Background()) uploaderCtx = log.WithLoggerFields(uploaderCtx) - go func() { - <-ctx.Done() - time.Sleep(time.Minute * 2) - uploadCancel() - }() + var storage Storage var err error @@ -191,20 +177,19 @@ func New(ctx context.Context, baseDir string, endpointURL *url.URL, uploadInfo h return &uploader{ ctx: uploaderCtx, cancel: uploadCancel, + done: make(chan struct{}), uploadID: uploadInfo.ID, postponeDuration: time.Second * 2, storage: storage, retryCount: 5, baseDir: filepath.Join(baseDir, uploadInfo.ID), eventBufferCount: 100, - isMasterFileFunc: func(s string) bool { - return strings.HasSuffix(s, "m3u8") - }, }, nil } // NewStorageByURL creates a new storage basd on the type from the url scheme, for example "datalake://my-datalake-endpoint" func NewStorageByURL(ctx context.Context, u *url.URL, headers map[string]string) (Storage, error) { + c, _ := config.FromEnv() switch u.Scheme { case "tus": return nil, errors.New("not imlemented yet") @@ -215,9 +200,9 @@ func NewStorageByURL(ctx context.Context, u *url.URL, headers map[string]string) if headers["token"] == "" { return nil, errors.New("missed auth token in the client's metadata") } - return NewDatalakeStorage(u.Hostname(), headers["workspace"], headers["token"]), nil + return NewDatalakeStorage(c.Endpoint().String(), headers["workspace"], headers["token"]), nil case "s3": - return NewS3(ctx, u.Hostname()), nil + return NewS3(ctx, c.Endpoint().String()), nil default: return nil, errors.New("unknown scheme") } From f25bd82709b1f94be948948564fc4aeb33756c25 Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Tue, 25 Feb 2025 03:06:03 +0300 Subject: [PATCH 06/52] fix linter Signed-off-by: denis-tingaikin --- internal/pkg/config/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 4eb601749e..057a4b0a6e 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -55,6 +55,7 @@ func FromEnv() (*Config, error) { return &result, nil } +// Endpoint returns upload address func (c *Config) Endpoint() *url.URL { var scheme = "https" if c.Insecure { From 4c38713a66174e9686e41dd1391601f1c72d80e7 Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Sat, 1 Mar 2025 15:58:55 +0300 Subject: [PATCH 07/52] fix issues with recordings with 1440p and above Signed-off-by: denis-tingaikin --- internal/pkg/config/config.go | 2 +- internal/pkg/manifest/hls.go | 5 -- internal/pkg/resconv/resconv.go | 27 +++++----- internal/pkg/resconv/resconv_test.go | 9 ++-- internal/pkg/uploader/postpone.go | 20 ++++--- internal/pkg/uploader/postpone_test.go | 7 +-- internal/pkg/uploader/s3.go | 6 +-- internal/pkg/uploader/uploader.go | 75 ++++++++++++++------------ 8 files changed, 76 insertions(+), 75 deletions(-) diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 057a4b0a6e..6a748d5bae 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -28,7 +28,7 @@ type Config struct { PprofEnabled bool `default:"false" split_words:"true" desc:"starts profile server on localhost:6060 if true"` Insecure bool `default:"false" desc:"ignores authorization check if true"` ServeURL string `split_words:"true" desc:"app listen url" default:"0.0.0.0:1080"` - EndpointURL *url.URL `split_words:"true" desc:"S3 or Datalake endpoint, example: s3://my-ip-address, datalake://my-ip-address"` + EndpointURL *url.URL `split_words:"true" default:"s3://127.0.0.1:9000" desc:"S3 or Datalake endpoint, example: s3://my-ip-address, datalake://my-ip-address"` AuthURL *url.URL `split_words:"true" desc:"url to auth the upload"` MaxCapacity int64 `split_words:"true" default:"6220800" desc:"represents the amount of maximum possible capacity for the transcoding. The default value is 1920 * 1080 * 3."` MaxThreads int `split_words:"true" default:"4" desc:"means upper bound for the transcoing provider."` diff --git a/internal/pkg/manifest/hls.go b/internal/pkg/manifest/hls.go index c301e4f4f5..d8dd7a5361 100644 --- a/internal/pkg/manifest/hls.go +++ b/internal/pkg/manifest/hls.go @@ -53,11 +53,6 @@ func GenerateHLSPlaylist(levels []string, outputPath, uploadID string) error { if err != nil { return err } - - _, err = file.WriteString("#EXT-X-ENDLIST") - if err != nil { - return err - } } return nil diff --git a/internal/pkg/resconv/resconv.go b/internal/pkg/resconv/resconv.go index 4e33a8db44..1c00db3d23 100644 --- a/internal/pkg/resconv/resconv.go +++ b/internal/pkg/resconv/resconv.go @@ -28,35 +28,32 @@ var prefixes = []struct { pixels int label string }{ - {pixels: 640 * 480, label: "320p"}, + {pixels: 640 * 360, label: "360p"}, {pixels: 1280 * 720, label: "480p"}, {pixels: 1920 * 1080, label: "720p"}, {pixels: 2560 * 1440, label: "1080p"}, - {pixels: 3840 * 2160, label: "2k"}, - {pixels: 5120 * 2880, label: "4k"}, - {pixels: 7680 * 4320, label: "5k"}, + {pixels: 3840 * 2160, label: "1440p"}, + {pixels: 7680 * 4320, label: "2160p"}, } var bandwidthMap = map[string]int{ - "320p": 300000, "360p": 500000, "480p": 2000000, "720p": 5000000, "1080p": 8000000, "1440p": 16000000, - "4k": 25000000, - "8k": 50000000, + "2160p": 25000000, + "4320p": 50000000, } var resolutions = map[string]string{ - "320p": "480:240", + "360p": "640:360", "480p": "640:480", "720p": "1280:720", "1080p": "1920:1080", - "2k": "2048:1080", - "4k": "3840:2160", - "5k": "5120:2880", - "8k": "7680:4320", + "1440p": "2560:1440", + "2160p": "3840:2160", + "4320p": "7680:4320", } // SubLevels returns sublevels for the resolution @@ -85,8 +82,8 @@ func SubLevels(resolution string) (res []string) { // Resolution returns default resolution based on the level func Resolution(level string) string { - if v, ok := resolutions[level]; ok { - return v + if res, ok := resolutions[level]; ok { + return res } return Resolution(defaultLevel) } @@ -98,7 +95,7 @@ func Level(resolution string) string { return pixels < prefixes[i].pixels }) if idx == len(prefixes) { - return "8k" + return "4320p" } return prefixes[idx].label diff --git a/internal/pkg/resconv/resconv_test.go b/internal/pkg/resconv/resconv_test.go index 4139e21370..f1dc27b800 100644 --- a/internal/pkg/resconv/resconv_test.go +++ b/internal/pkg/resconv/resconv_test.go @@ -29,10 +29,11 @@ func Test_Resconv_ShouldReturnCorrectPrefix(t *testing.T) { {res: "640:480", expected: "480p"}, {res: "1280:720", expected: "720p"}, {res: "1920:1080", expected: "1080p"}, - {res: "2560:1440", expected: "2k"}, - {res: "3840:2160", expected: "4k"}, - {res: "5120:2880", expected: "5k"}, - {res: "9000:4000", expected: "8k"}, + {res: "2880:1800", expected: "1440p"}, + {res: "2560:1440", expected: "1440p"}, + {res: "3840:2160", expected: "2160p"}, + {res: "5120:2880", expected: "2160p"}, + {res: "9000:4000", expected: "4320p"}, } for _, tt := range tests { diff --git a/internal/pkg/uploader/postpone.go b/internal/pkg/uploader/postpone.go index 3486996ec1..4cfaa81b37 100644 --- a/internal/pkg/uploader/postpone.go +++ b/internal/pkg/uploader/postpone.go @@ -16,27 +16,33 @@ package uploader import ( "context" "time" + + "github.com/huly-stream/internal/pkg/log" + "go.uber.org/zap" ) -func (u *uploader) postpone(id string, action func()) { - u.wg.Add(1) +func (u *uploader) postpone(fileName string, action func(ctx context.Context)) { + u.waitJobs.Add(1) var ctx, cancel = context.WithCancel(context.Background()) + ctx = log.WithLoggerFields(ctx, zap.String("pospone", "action")) var startCh = time.After(u.postponeDuration) - if v, ok := u.contexts.Load(id); ok { + if v, ok := u.contexts.Load(fileName); ok { (*v.(*context.CancelFunc))() } - u.contexts.Store(id, &cancel) + u.contexts.Store(fileName, &cancel) go func() { - u.wg.Done() + u.waitJobs.Done() defer cancel() select { case <-ctx.Done(): return case <-startCh: - action() - u.contexts.CompareAndDelete(id, &cancel) + action(ctx) + if u.ctx.Err() == nil { + u.contexts.CompareAndDelete(fileName, &cancel) + } } }() } diff --git a/internal/pkg/uploader/postpone_test.go b/internal/pkg/uploader/postpone_test.go index 89522b4cec..68758451e7 100644 --- a/internal/pkg/uploader/postpone_test.go +++ b/internal/pkg/uploader/postpone_test.go @@ -14,6 +14,7 @@ package uploader import ( + "context" "sync/atomic" "testing" "time" @@ -26,9 +27,9 @@ func Test_Postpone(t *testing.T) { postponeDuration: time.Second / 4, } var counter atomic.Int32 - u.postpone("1", func() { counter.Add(1) }) + u.postpone("1", func(context.Context) { counter.Add(1) }) time.Sleep(time.Second / 8) - u.postpone("1", func() { counter.Add(1) }) + u.postpone("1", func(context.Context) { counter.Add(1) }) time.Sleep(time.Second / 2) require.Equal(t, int32(1), counter.Load()) time.Sleep(time.Second / 2) @@ -38,7 +39,7 @@ func Test_Postpone(t *testing.T) { func Test_WithoutPostpone(t *testing.T) { var counter atomic.Int32 var u uploader - u.postpone("1", func() { counter.Add(1) }) + u.postpone("1", func(context.Context) { counter.Add(1) }) time.Sleep(time.Second / 10) require.Equal(t, int32(1), counter.Load()) } diff --git a/internal/pkg/uploader/s3.go b/internal/pkg/uploader/s3.go index 016e6fc87c..d211e0347f 100644 --- a/internal/pkg/uploader/s3.go +++ b/internal/pkg/uploader/s3.go @@ -41,10 +41,9 @@ type S3Storage struct { } // NewS3 creates a new S3 storage -func NewS3(ctx context.Context, endpoint string) Storage { +func NewS3(ctx context.Context, endpoint, bucketName string) Storage { var accessKeyID = os.Getenv("AWS_ACCESS_KEY_ID") var accessKeySecret = os.Getenv("AWS_SECRET_ACCESS_KEY") - var bucketName = os.Getenv("AWS_BUCKET_NAME") var logger = log.FromContext(ctx).With(zap.String("s3", "storage")) cfg, err := config.LoadDefaultConfig(ctx, @@ -68,9 +67,6 @@ func NewS3(ctx context.Context, endpoint string) Storage { } func getContentType(objectKey string) string { - if strings.HasSuffix(objectKey, ".txt") { - return "txt" - } if strings.HasSuffix(objectKey, ".ts") { return "video/mp2t" } diff --git a/internal/pkg/uploader/uploader.go b/internal/pkg/uploader/uploader.go index a2c76811cd..8d04618f57 100644 --- a/internal/pkg/uploader/uploader.go +++ b/internal/pkg/uploader/uploader.go @@ -34,7 +34,7 @@ import ( type uploader struct { done chan struct{} - wg sync.WaitGroup + waitJobs sync.WaitGroup ctx context.Context cancel context.CancelFunc baseDir string @@ -48,10 +48,13 @@ type uploader struct { } func (u *uploader) retry(action func() error) { + var delay = time.Millisecond * 50 for range u.retryCount { if err := action(); err == nil { return } + time.Sleep(delay) + delay *= 2 } } @@ -59,30 +62,41 @@ func (u *uploader) retry(action func() error) { func (u *uploader) Rollback() { logger := log.FromContext(u.ctx).With(zap.String("uploader", "Rollback")) logger.Debug("starting") - defer u.cancel() + defer logger.Debug("done") - u.wg.Wait() - - u.sentFiles.Range(func(key, value any) bool { - logger.Debug("deleting remote file", zap.String("key", key.(string))) - u.retry(func() error { return u.storage.DeleteFile(u.ctx, key.(string)) }) - log.FromContext(u.ctx).Debug("deleting local file", zap.String("key", key.(string))) - _ = os.Remove(key.(string)) - return true + u.postpone("", func(ctx context.Context) { + u.sentFiles.Range(func(key, value any) bool { + logger.Debug("deleting remote file", zap.String("key", key.(string))) + u.retry(func() error { return u.storage.DeleteFile(ctx, key.(string)) }) + return true + }) }) + + u.Terminate() } +// Terminate deletes func (u *uploader) Terminate() { logger := log.FromContext(u.ctx).With(zap.String("uploader", "Terminate")) logger.Debug("starting") - defer u.cancel() - u.wg.Wait() + go func() { + defer logger.Debug("done") + u.waitJobs.Wait() + u.cancel() + }() +} - u.sentFiles.Range(func(key, value any) bool { - _ = os.Remove(key.(string)) - return true +func (u *uploader) uploadAndDelte(fileName string) { + u.postpone(fileName+"-del", func(context.Context) {}) + u.postpone(fileName, func(ctx context.Context) { + u.retry(func() error { return u.storage.UploadFile(ctx, fileName) }) + u.postpone(fileName+"-del", func(context.Context) { + _ = os.Remove(fileName) + }) }) + + u.sentFiles.Store(fileName, struct{}{}) } func (u *uploader) Serve() error { @@ -98,13 +112,8 @@ func (u *uploader) Serve() error { _ = os.MkdirAll(u.baseDir, os.ModePerm) initFiles, _ := os.ReadDir(u.baseDir) for _, f := range initFiles { - var name = filepath.Join(u.baseDir, f.Name()) - u.postpone(name, func() { - logger.Debug("started uploading", zap.String("eventName", name)) - u.retry(func() error { return u.storage.UploadFile(u.ctx, name) }) - logger.Debug("added to sentFiles", zap.String("eventName", name)) - u.sentFiles.Store(name, struct{}{}) - }) + var filePath = filepath.Join(u.baseDir, f.Name()) + u.uploadAndDelte(filePath) } if err := watcher.Add(u.baseDir); err != nil { @@ -132,12 +141,7 @@ func (u *uploader) Serve() error { if !strings.Contains(event.Name, u.uploadID) { continue } - u.postpone(event.Name, func() { - logger.Debug("started uploading", zap.String("eventName", event.Name)) - u.retry(func() error { return u.storage.UploadFile(u.ctx, event.Name) }) - logger.Debug("added to sentFiles", zap.String("eventName", event.Name)) - u.sentFiles.Store(event.Name, struct{}{}) - }) + u.uploadAndDelte(event.Name) case err, ok := <-watcher.Errors: if !ok { return u.ctx.Err() @@ -168,7 +172,7 @@ func New(ctx context.Context, baseDir string, endpointURL *url.URL, uploadInfo h var storage Storage var err error - storage, err = NewStorageByURL(ctx, endpointURL, uploadInfo.MetaData) + storage, err = NewStorageByURL(uploaderCtx, endpointURL, uploadInfo.MetaData) if err != nil { uploadCancel() return nil, err @@ -181,7 +185,7 @@ func New(ctx context.Context, baseDir string, endpointURL *url.URL, uploadInfo h uploadID: uploadInfo.ID, postponeDuration: time.Second * 2, storage: storage, - retryCount: 5, + retryCount: 10, baseDir: filepath.Join(baseDir, uploadInfo.ID), eventBufferCount: 100, }, nil @@ -189,20 +193,21 @@ func New(ctx context.Context, baseDir string, endpointURL *url.URL, uploadInfo h // NewStorageByURL creates a new storage basd on the type from the url scheme, for example "datalake://my-datalake-endpoint" func NewStorageByURL(ctx context.Context, u *url.URL, headers map[string]string) (Storage, error) { + var workspace = headers["workspace"] + if workspace == "" { + return nil, errors.New("missed workspace in the client's metadata") + } c, _ := config.FromEnv() switch u.Scheme { case "tus": return nil, errors.New("not imlemented yet") case "datalake": - if headers["workspace"] == "" { - return nil, errors.New("missed workspace in the client's metadata") - } if headers["token"] == "" { return nil, errors.New("missed auth token in the client's metadata") } - return NewDatalakeStorage(c.Endpoint().String(), headers["workspace"], headers["token"]), nil + return NewDatalakeStorage(c.Endpoint().String(), workspace, headers["token"]), nil case "s3": - return NewS3(ctx, c.Endpoint().String()), nil + return NewS3(ctx, c.Endpoint().String(), workspace), nil default: return nil, errors.New("unknown scheme") } From ce86585484c47142544b4b3b838820911a748bf1 Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Mon, 3 Mar 2025 06:39:23 +0300 Subject: [PATCH 08/52] fix tests Signed-off-by: denis-tingaikin --- internal/pkg/resconv/resconv.go | 2 +- internal/pkg/resconv/resconv_test.go | 3 ++- internal/pkg/uploader/postpone.go | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/pkg/resconv/resconv.go b/internal/pkg/resconv/resconv.go index 1c00db3d23..3ed4a0b3c2 100644 --- a/internal/pkg/resconv/resconv.go +++ b/internal/pkg/resconv/resconv.go @@ -22,7 +22,7 @@ import ( "strings" ) -const defaultLevel = "320p" +const defaultLevel = "360p" var prefixes = []struct { pixels int diff --git a/internal/pkg/resconv/resconv_test.go b/internal/pkg/resconv/resconv_test.go index f1dc27b800..547fab0582 100644 --- a/internal/pkg/resconv/resconv_test.go +++ b/internal/pkg/resconv/resconv_test.go @@ -25,7 +25,8 @@ func Test_Resconv_ShouldReturnCorrectPrefix(t *testing.T) { res string expected string }{ - {res: "320:240", expected: "320p"}, + {res: "320:240", expected: "360p"}, + {res: "320:360", expected: "360p"}, {res: "640:480", expected: "480p"}, {res: "1280:720", expected: "720p"}, {res: "1920:1080", expected: "1080p"}, diff --git a/internal/pkg/uploader/postpone.go b/internal/pkg/uploader/postpone.go index 4cfaa81b37..29b308dc5e 100644 --- a/internal/pkg/uploader/postpone.go +++ b/internal/pkg/uploader/postpone.go @@ -40,7 +40,7 @@ func (u *uploader) postpone(fileName string, action func(ctx context.Context)) { return case <-startCh: action(ctx) - if u.ctx.Err() == nil { + if ctx.Err() == nil { u.contexts.CompareAndDelete(fileName, &cancel) } } From 5342edacf638d28ebb19259594268db1af535396 Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Sun, 16 Mar 2025 01:00:35 +0300 Subject: [PATCH 09/52] add possible to schedule transcoding tasks Signed-off-by: denis-tingaikin --- .github/workflows/docker-push.yaml | 4 +- .golangci.yaml | 4 +- Dockerfile | 18 +- README.md | 100 +++-- cmd/huly-stream/main.go | 104 ----- cmd/stream/main.go | 71 ++++ go.mod | 12 +- go.sum | 18 +- internal/pkg/api/v1/recording/handler.go | 82 ++++ internal/pkg/api/v1/transcoding/handler.go | 66 +++ internal/pkg/config/config.go | 28 +- internal/pkg/log/zap.go | 4 +- internal/pkg/manifest/hls.go | 2 +- internal/pkg/manifest/hls_test.go | 2 +- internal/pkg/pprof/pprof.go | 3 +- internal/pkg/resconv/resconv_test.go | 2 +- .../pkg/sharedpipe/shared_pipe_bench_test.go | 18 +- internal/pkg/storage/datalake.go | 254 ++++++++++++ internal/pkg/{uploader => storage}/s3.go | 89 ++-- internal/pkg/storage/storage.go | 55 +++ internal/pkg/token/token.go | 77 ++++ .../postpone_test.go => token/token_test.go} | 37 +- .../{transcoding => transcoder}/command.go | 14 +- .../command_test.go | 16 +- internal/pkg/transcoder/coordinator.go | 190 +++++++++ internal/pkg/transcoder/queue.go | 234 +++++++++++ .../worker.go => transcoder/stream.go} | 63 +-- internal/pkg/transcoding/limiter.go | 78 ---- internal/pkg/transcoding/limiter_test.go | 88 ---- internal/pkg/transcoding/scheduler.go | 167 -------- internal/pkg/uploader/datalake.go | 130 ------ internal/pkg/uploader/options.go | 14 +- internal/pkg/uploader/postpone.go | 48 --- internal/pkg/uploader/uploader.go | 381 +++++++++++------- 34 files changed, 1523 insertions(+), 950 deletions(-) delete mode 100644 cmd/huly-stream/main.go create mode 100644 cmd/stream/main.go create mode 100644 internal/pkg/api/v1/recording/handler.go create mode 100644 internal/pkg/api/v1/transcoding/handler.go create mode 100644 internal/pkg/storage/datalake.go rename internal/pkg/{uploader => storage}/s3.go (61%) create mode 100644 internal/pkg/storage/storage.go create mode 100644 internal/pkg/token/token.go rename internal/pkg/{uploader/postpone_test.go => token/token_test.go} (50%) rename internal/pkg/{transcoding => transcoder}/command.go (93%) rename internal/pkg/{transcoding => transcoder}/command_test.go (56%) create mode 100644 internal/pkg/transcoder/coordinator.go create mode 100644 internal/pkg/transcoder/queue.go rename internal/pkg/{transcoding/worker.go => transcoder/stream.go} (72%) delete mode 100644 internal/pkg/transcoding/limiter.go delete mode 100644 internal/pkg/transcoding/limiter_test.go delete mode 100644 internal/pkg/transcoding/scheduler.go delete mode 100644 internal/pkg/uploader/datalake.go delete mode 100644 internal/pkg/uploader/postpone.go diff --git a/.github/workflows/docker-push.yaml b/.github/workflows/docker-push.yaml index ad67bcee7c..47baea039f 100644 --- a/.github/workflows/docker-push.yaml +++ b/.github/workflows/docker-push.yaml @@ -29,7 +29,7 @@ jobs: id: metaci uses: docker/metadata-action@v3 with: - images: hardcoreeng/huly-stream + images: hardcoreeng/stream:${{ inputs.version }} tags: | type=ref,event=pr type=sha,prefix= @@ -41,4 +41,4 @@ jobs: context: . platforms: linux/amd64,linux/arm64 push: true - tags: hardcoreeng/huly-stream:${{ inputs.version }} + tags: hardcoreeng/stream:${{ inputs.version }} diff --git a/.golangci.yaml b/.golangci.yaml index 0099487e6f..a5335cf208 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -61,8 +61,8 @@ linters-settings: dupl: threshold: 150 funlen: - lines: 120 - statements: 60 + lines: 140 + statements: 80 goconst: min-len: 2 min-occurrences: 2 diff --git a/Dockerfile b/Dockerfile index 0b4dca5d82..4304349055 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM --platform=$BUILDPLATFORM golang:1.23.5 AS builder +FROM --platform=linux/amd64 golang:1.24.1 AS builder ENV GO111MODULE=on ENV CGO_ENABLED=0 ENV GOBIN=/bin @@ -19,18 +19,18 @@ ARG BUILDARCH=amd64 COPY . ./ -RUN set -xe && GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /go/bin/huly-stream ./cmd/huly-stream +RUN set -xe && GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /go/bin/stream ./cmd/stream -FROM alpine +FROM --platform=linux/amd64 alpine RUN set -xe && apk add --no-cache ffmpeg RUN apk add --no-cache ca-certificates jq bash \ - && addgroup -g 1000 huly-stream \ - && adduser -u 1000 -G huly-stream -s /bin/sh -D huly-stream \ - && chown huly-stream:huly-stream /. -COPY --from=builder /go/bin/huly-stream /huly-stream + && addgroup -g 1000 stream \ + && adduser -u 1000 -G stream -s /bin/sh -D stream \ + && chown stream:stream /. +COPY --from=builder /go/bin/stream /stream EXPOSE 1080 -USER huly-stream +USER stream -ENTRYPOINT ["/huly-stream"] \ No newline at end of file +ENTRYPOINT ["/stream"] \ No newline at end of file diff --git a/README.md b/README.md index f977167dba..529b764313 100644 --- a/README.md +++ b/README.md @@ -1,91 +1,96 @@ -# Huly Stream +# Stream [![X (formerly Twitter) Follow](https://img.shields.io/twitter/follow/huly_io?style=for-the-badge)](https://x.com/huly_io) ![GitHub License](https://img.shields.io/github/license/hcengineering/platform?style=for-the-badge) ## About -The Huly Stream high-performance HTTP-based transcoding service. Huly-stream is built around the **TUS protocol**, enabling reliable, resumable file uploads and downloads. Designed for seamless and consistent media processing,it supports advanced transcoding features with robust integration options. +The Stream is high-performance HTTP-based transcoding service. *Stream* supports **TUS protocol**, enabling reliable, +resumable transcodings. Designed for seamless and consistent media processing,it supports advanced transcoding features +with robust integration options. --- ## Features ### TUS Protocol Support -- **Resumable transcoding**: Leveraging the TUS protocol, Huly-stream ensures reliable and efficient stream processing. -### Input Support +- **Resumable transcoding**: Leveraging the TUS protocol, *Stream* ensures reliable and efficient transcoding bucket + processing. + +#### Input Support + - **Supported Input Formats**: - - `mp4` - - `webm` + - `mp4` + - `webm` + +#### Output Options -### Output Options - **Supported Output Formats**: - - `aac` - - `hls` + - `hls` + +#### Upload options -### Upload options -- **TUS Upload**: Resumable file uploads via TUS protocol. - **s3 Upload**: Direct upload to S3 storage. - **datalake Upload**: Upload to datalake storage. -### Key Functionalities +#### Key Functionalities + - **Live transcoing with minimal upload time**: Transcoding results are going to be avaible after stream completion. - **Transcoding Cancelation**: Cancel or pause ongoing transcoding in real-time. - **Transcoding Resumption**: Resume incomplete transcoding tasks efficiently. ---- +### Transcoding scheduling ## Installation ### Prerequisites + - [Go](https://golang.org/dl/) (v1.23+ recommended) - [ffmpeg](https://www.ffmpeg.org/download.html) (ensure it’s installed and available in your system's PATH) ### Steps 1. Install dependencies: - ```bash - go mod tidy - ``` + +```bash +go mod tidy +``` 2. Build the service: - ```bash - docker build . -t hcengineering/huly-stream:latest - ``` +```bash +docker build . -t hcengineering/sream:latest +``` --- ## Configuraiton ### App env configuraiton + The following environment variables can be used: + ``` -KEY TYPE DEFAULT REQUIRED DESCRIPTION -STREAM_SECRET_TOKEN String secret token for authorize requests -STREAM_LOG_LEVEL String debug sets log level for the application -STREAM_PPROF_ENABLED True or False false starts profile server on localhost:6060 if true -STREAM_INSECURE True or False false ignores authorization check if true -STREAM_SERVE_URL String 0.0.0.0:1080 app listen url -STREAM_ENDPOINT_URL URL S3 or Datalake endpoint, example: s3://my-ip-address, datalake://my-ip-address -STREAM_MAX_CAPACITY Integer 6220800 represents the amount of maximum possible capacity for the transcoding. The default value is 1920 * 1080 * 3. -STREAM_MAX_THREADS Integer 4 means upper bound for the transcoing provider. -STREAM_OUTPUT_DIR String /tmp/transcoing/ path to the directory with transcoding result. -STREAM_REMOVE_CONTENT_ON_UPLOAD True or False true deletes all content when content delivered if true -STREAM_UPLOAD_RAW_CONTENT True or False false uploads content in raw quality to the endpoint if true +KEY TYPE DEFAULT STREAM_LOG_LEVEL String debug sets log level for the application +STREAM_SERVER_SECRET String server secret required to generate and verify tokens +STREAM_PPROF_ENABLED True or False true starts profile server on localhost:6060 if true +STREAM_INSECURE True or False false ignores authorization check if true +STREAM_SERVE_URL String 0.0.0.0:1080 listen on url +STREAM_ENDPOINT_URL URL s3://127.0.0.1:9000 S3 or Datalake endpoint, example: s3://my-ip-address, datalake://my-ip-address +STREAM_MAX_PARALLEL_SCALING_COUNT Integer 2 how much parallel scaling can be processed +STREAM_MAX_THREAD_COUNT Integer 4 max number of threads for transcoder +STREAM_OUTPUT_DIR String /tmp/transcoing/ path to the directory with tra ``` ### Metadata -**resolutions:** if passed, set the resolution for the output, for example, 'resolutions: 1920:1080, 1280:720.' +**resolution:** if passed, set the resolution for the output, for example, 'resolution: 1920:1080'. **token:** must be provided to be authorized in the Huly's datalake service. **workspace:** required for uploading content to the datalake storage. - - #### S3 Env configuration if you're working with S3 storage type, these envs must be provided: @@ -94,16 +99,35 @@ if you're working with S3 storage type, these envs must be provided: ## Usage -The service exposes an HTTP API. Below are some examples of how to interact with it. +The service exposes an HTTP API. + +Below are some examples of how to interact with it. + +### Trnascode via TUS -### Upload a File for Transcoding via TUS ```bash -curl -X POST http://localhost:1080/transcoing \ +curl -X POST http://localhost:1080/recording \ -H "Tus-Resumable: 1.0.0" \ -H "Upload-Length: " \ --data-binary @path/to/your/file.mp4 ``` +Note: tus client is required, to play with a service locally you can use tus-js-client example +with [video](https://github.com/tus/tus-js-client/blob/main/demos/browser/video.html) + +### Schedule a transcoding + +```bash +curl -X POST http://localhost:1080/transcoding \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer " \ + -d '{ + "source": "", + "format": "hls", + "workspace": "test" + }' +``` + ## Contributing We welcome contributions! To get started: @@ -120,4 +144,4 @@ This project is licensed under the [MIT License](LICENSE). --- -Enjoy seamless transcoding with huly-stream! 🚀 \ No newline at end of file +Enjoy seamless transcoding with *Stream*! 🚀 \ No newline at end of file diff --git a/cmd/huly-stream/main.go b/cmd/huly-stream/main.go deleted file mode 100644 index d7c5d9690b..0000000000 --- a/cmd/huly-stream/main.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright © 2025 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package main provides huly-stream entry point function -package main - -import ( - "context" - "net/http" - - "os" - "os/signal" - "syscall" - - "go.uber.org/zap" - "golang.org/x/exp/slog" - - "github.com/huly-stream/internal/pkg/config" - "github.com/huly-stream/internal/pkg/log" - "github.com/huly-stream/internal/pkg/pprof" - "github.com/huly-stream/internal/pkg/transcoding" - tusd "github.com/tus/tusd/v2/pkg/handler" -) - -const basePath = "/recording" - -func main() { - var ctx, cancel = signal.NotifyContext( - context.Background(), - os.Interrupt, - syscall.SIGHUP, - syscall.SIGTERM, - syscall.SIGQUIT, - ) - defer cancel() - ctx = log.WithLoggerFields(ctx) - - var logger = log.FromContext(ctx) - var conf = must(config.FromEnv()) - logger.Sugar().Debugf("provided config is %v", conf) - - logger.Sugar().Info(conf.Endpoint()) - - mustNoError(os.MkdirAll(conf.OutputDir, os.ModePerm)) - if conf.PprofEnabled { - go pprof.ListenAndServe(ctx, "localhost:6060") - } - scheduler := transcoding.NewScheduler(ctx, conf) - - tusComposer := tusd.NewStoreComposer() - tusComposer.UseCore(scheduler) - tusComposer.UseTerminater(scheduler) - tusComposer.UseConcater(scheduler) - tusComposer.UseLengthDeferrer(scheduler) - - var handler = must(tusd.NewHandler(tusd.Config{ - BasePath: basePath, - StoreComposer: tusComposer, - Logger: slog.New(slog.NewTextHandler(discardTextHandler{}, nil)), - })) - - http.Handle("/recording/", http.StripPrefix("/recording/", handler)) - http.Handle("/recording", http.StripPrefix("/recording", handler)) - - go func() { - logger.Info("started to listen") - defer logger.Info("server has finished") - // #nosec - var err = http.ListenAndServe(conf.ServeURL, nil) - if err != nil { - cancel() - logger.Debug("unable to listen", zap.Error(err)) - } - }() - - <-ctx.Done() -} - -type discardTextHandler struct{} - -func (discardTextHandler) Write([]byte) (int, error) { - return 0, nil -} - -func mustNoError(err error) { - if err != nil { - panic(err.Error()) - } -} - -func must[T any](val T, err error) T { - mustNoError(err) - return val -} diff --git a/cmd/stream/main.go b/cmd/stream/main.go new file mode 100644 index 0000000000..9b34801c02 --- /dev/null +++ b/cmd/stream/main.go @@ -0,0 +1,71 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main provides huly-stream entry point function +package main + +import ( + "context" + "net/http" + + "os" + "os/signal" + "syscall" + + "go.uber.org/zap" + + "github.com/hcengineering/stream/internal/pkg/api/v1/recording" + "github.com/hcengineering/stream/internal/pkg/api/v1/transcoding" + "github.com/hcengineering/stream/internal/pkg/config" + "github.com/hcengineering/stream/internal/pkg/log" +) + +func main() { + var ctx, cancel = signal.NotifyContext( + context.Background(), + os.Interrupt, + syscall.SIGHUP, + syscall.SIGTERM, + syscall.SIGQUIT, + ) + defer cancel() + ctx = log.WithFields(ctx) + + var logger = log.FromContext(ctx) + var cfg, err = config.FromEnv() + if err != nil { + panic(err.Error()) + } + logger.Sugar().Debugf("parsed config is %v", cfg) + + var recordingHandler = recording.NewHandler(ctx, cfg) + var transcodingHandler = transcoding.NewHandler(ctx, cfg) + + http.Handle("/recording/", http.StripPrefix("/recording/", recordingHandler)) + http.Handle("/recording", http.StripPrefix("/recording", recordingHandler)) + http.Handle("/transcoding", http.StripPrefix("/transcoding", transcodingHandler)) + + go func() { + logger.Info("server started serving", zap.String("ServeURL", cfg.ServeURL)) + defer logger.Info("server finished") + + // #nosec + var err = http.ListenAndServe(cfg.ServeURL, nil) + if err != nil { + cancel() + logger.Debug("unable to listen", zap.Error(err)) + } + }() + + <-ctx.Done() +} diff --git a/go.mod b/go.mod index 3775572bbf..0671d43070 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,13 @@ -module github.com/huly-stream +module github.com/hcengineering/stream -go 1.23.2 +go 1.24.1 require ( github.com/aws/aws-sdk-go-v2 v1.36.1 github.com/aws/aws-sdk-go-v2/config v1.29.6 github.com/aws/aws-sdk-go-v2/credentials v1.17.59 github.com/aws/aws-sdk-go-v2/service/s3 v1.77.0 - github.com/aws/smithy-go v1.22.3 - github.com/fsnotify/fsnotify v1.8.0 + github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.6.0 github.com/kelseyhightower/envconfig v1.4.0 github.com/pkg/errors v0.9.1 @@ -17,6 +16,8 @@ require ( github.com/valyala/fasthttp v1.59.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20250215185904-eff6e970281f + gopkg.in/vansante/go-ffprobe.v2 v2.2.1 + k8s.io/utils v0.0.0-20241210054802-24370beab758 ) require ( @@ -34,11 +35,12 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect + github.com/aws/smithy-go v1.22.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/klauspost/compress v1.17.11 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/sys v0.30.0 // indirect + golang.org/x/net v0.37.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 1ffa95ca7c..6ef7c10d82 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,8 @@ github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -72,13 +72,15 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/exp v0.0.0-20250215185904-eff6e970281f h1:oFMYAjX0867ZD2jcNiLBrI9BdpmEkvPyi5YrBGXbamg= golang.org/x/exp v0.0.0-20250215185904-eff6e970281f/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk= -golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= -golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= +golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/vansante/go-ffprobe.v2 v2.2.1 h1:sFV08OT1eZ1yroLCZVClIVd9YySgCh9eGjBWO0oRayI= +gopkg.in/vansante/go-ffprobe.v2 v2.2.1/go.mod h1:qF0AlAjk7Nqzqf3y333Ly+KxN3cKF2JqA3JT5ZheUGE= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0= +k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= diff --git a/internal/pkg/api/v1/recording/handler.go b/internal/pkg/api/v1/recording/handler.go new file mode 100644 index 0000000000..4282261f76 --- /dev/null +++ b/internal/pkg/api/v1/recording/handler.go @@ -0,0 +1,82 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package recording provides recording http handler. +package recording + +import ( + "context" + + "golang.org/x/exp/slog" + + "net/http" + "sync" + + "github.com/hcengineering/stream/internal/pkg/config" + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/transcoder" + "go.uber.org/zap" + + tusd "github.com/tus/tusd/v2/pkg/handler" +) + +type recordingHandler struct { + logger *zap.Logger + once sync.Once + cfg *config.Config + ctx context.Context + tusHandler http.Handler +} + +func (h *recordingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if !h.cfg.Insecure { + r.Header.Set("X-Forwarded-Proto", "https") + } + h.once.Do(h.initialize) + h.tusHandler.ServeHTTP(w, r) +} + +// NewHandler creates a new recording http handler, requires context and config. +func NewHandler(ctx context.Context, cfg *config.Config) http.Handler { + return &recordingHandler{ + logger: log.FromContext(ctx).With(zap.String("handler", "recording")), + cfg: cfg, + ctx: ctx, + } +} + +func (h *recordingHandler) initialize() { + scheduler := transcoder.NewStreamCoordinator(h.ctx, h.cfg) + + tusComposer := tusd.NewStoreComposer() + tusComposer.UseCore(scheduler) + tusComposer.UseTerminater(scheduler) + tusComposer.UseConcater(scheduler) + tusComposer.UseLengthDeferrer(scheduler) + + var tusHandler, err = tusd.NewHandler(tusd.Config{ + BasePath: "/recording", + StoreComposer: tusComposer, + RespectForwardedHeaders: true, + DisableDownload: true, + Cors: &tusd.DefaultCorsConfig, + NetworkTimeout: h.cfg.Timeout, + Logger: slog.Default(), + }) + + if err != nil { + panic(err.Error()) + } + + h.tusHandler = tusHandler +} diff --git a/internal/pkg/api/v1/transcoding/handler.go b/internal/pkg/api/v1/transcoding/handler.go new file mode 100644 index 0000000000..0391ea74da --- /dev/null +++ b/internal/pkg/api/v1/transcoding/handler.go @@ -0,0 +1,66 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package transcoding provides transcoding http handler. +package transcoding + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/hcengineering/stream/internal/pkg/config" + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/transcoder" + "go.uber.org/zap" +) + +type trascodeHandler struct { + taskQueue *transcoder.Scheduler + logger *zap.Logger +} + +func (t *trascodeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "" { + w.WriteHeader(http.StatusBadRequest) + _, _ = fmt.Fprint(w, "uri is not allowed") + return + } + + if r.Header.Get("Authorization") == "" { + w.WriteHeader(http.StatusUnauthorized) + _, _ = fmt.Fprint(w, "missed Authorization header") + return + } + + var decoder = json.NewDecoder(r.Body) + var task transcoder.Task + + if err := decoder.Decode(&task); err != nil { + w.WriteHeader(http.StatusBadRequest) + _, _ = fmt.Fprint(w, "can not decode request body") + return + } + + t.taskQueue.Schedule(&task) + w.WriteHeader(http.StatusOK) +} + +// NewHandler creates a new trnascoding http handler, requires context and config. +func NewHandler(ctx context.Context, cfg *config.Config) http.Handler { + return &trascodeHandler{ + taskQueue: transcoder.NewScheduler(ctx, cfg), + logger: log.FromContext(ctx).With(zap.String("handler", "transcoding")), + } +} diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 6a748d5bae..23fee3afc7 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -18,22 +18,24 @@ import ( "net/url" "time" + "github.com/pkg/errors" + "github.com/kelseyhightower/envconfig" ) // Config represents configuration for the huly-stream application. type Config struct { - SecretToken string `split_words:"true" desc:"secret token for authorize requests"` - LogLevel string `split_words:"true" default:"debug" desc:"sets log level for the application"` - PprofEnabled bool `default:"false" split_words:"true" desc:"starts profile server on localhost:6060 if true"` - Insecure bool `default:"false" desc:"ignores authorization check if true"` - ServeURL string `split_words:"true" desc:"app listen url" default:"0.0.0.0:1080"` - EndpointURL *url.URL `split_words:"true" default:"s3://127.0.0.1:9000" desc:"S3 or Datalake endpoint, example: s3://my-ip-address, datalake://my-ip-address"` - AuthURL *url.URL `split_words:"true" desc:"url to auth the upload"` - MaxCapacity int64 `split_words:"true" default:"6220800" desc:"represents the amount of maximum possible capacity for the transcoding. The default value is 1920 * 1080 * 3."` - MaxThreads int `split_words:"true" default:"4" desc:"means upper bound for the transcoing provider."` - OutputDir string `split_words:"true" default:"/tmp/transcoing/" desc:"path to the directory with transcoding result."` - Timeout time.Duration `default:"5m" desc:"timeout for the upload"` + LogLevel string `split_words:"true" default:"debug" desc:"sets log level for the application"` + ServerSecret string `split_words:"true" default:"" desc:"server secret required to generate and verify tokens"` + PprofEnabled bool `split_words:"true" default:"true" desc:"starts profile server on localhost:6060 if true"` + Insecure bool `split_words:"true" default:"false" desc:"ignores authorization check if true"` + ServeURL string `split_words:"true" desc:"listen on url" default:"0.0.0.0:1080"` + EndpointURL *url.URL `split_words:"true" default:"s3://127.0.0.1:9000" desc:"S3 or Datalake endpoint, example: s3://my-ip-address, datalake://my-ip-address"` + MaxParallelScalingCount int `split_words:"true" default:"2" desc:"how much parallel scaling can be processed"` + MaxThreadCount int `split_words:"true" default:"4" desc:"max number of threads for transcoder"` + + OutputDir string `split_words:"true" default:"/tmp/transcoing/" desc:"path to the directory with transcoding result."` + Timeout time.Duration `default:"5m" desc:"timeout for the upload"` } // FromEnv creates new Config from env @@ -52,6 +54,10 @@ func FromEnv() (*Config, error) { result.EndpointURL = nil } + if !result.Insecure && result.ServerSecret == "" { + return nil, errors.New("server secret must be provided for secure confgiuration") + } + return &result, nil } diff --git a/internal/pkg/log/zap.go b/internal/pkg/log/zap.go index 72e03ac379..b85fcfbe5b 100644 --- a/internal/pkg/log/zap.go +++ b/internal/pkg/log/zap.go @@ -22,8 +22,8 @@ import ( type contextKey struct{} -// WithLoggerFields createsa new context with zap.Logger and passed fields -func WithLoggerFields(ctx context.Context, fields ...zap.Field) context.Context { +// WithFields createsa new context with zap.Logger and passed fields +func WithFields(ctx context.Context, fields ...zap.Field) context.Context { var logger = FromContext(ctx) if logger == nil { var err error diff --git a/internal/pkg/manifest/hls.go b/internal/pkg/manifest/hls.go index d8dd7a5361..dcb421fe50 100644 --- a/internal/pkg/manifest/hls.go +++ b/internal/pkg/manifest/hls.go @@ -20,7 +20,7 @@ import ( "path/filepath" "strings" - "github.com/huly-stream/internal/pkg/resconv" + "github.com/hcengineering/stream/internal/pkg/resconv" ) // GenerateHLSPlaylist generates master file for master files for resolution levels diff --git a/internal/pkg/manifest/hls_test.go b/internal/pkg/manifest/hls_test.go index 624e3a27a9..a37c63b874 100644 --- a/internal/pkg/manifest/hls_test.go +++ b/internal/pkg/manifest/hls_test.go @@ -18,7 +18,7 @@ import ( "path/filepath" "testing" - "github.com/huly-stream/internal/pkg/manifest" + "github.com/hcengineering/stream/internal/pkg/manifest" "github.com/stretchr/testify/require" ) diff --git a/internal/pkg/pprof/pprof.go b/internal/pkg/pprof/pprof.go index e7a5eb0302..081e9603df 100644 --- a/internal/pkg/pprof/pprof.go +++ b/internal/pkg/pprof/pprof.go @@ -20,7 +20,7 @@ import ( "net/http/pprof" "time" - "github.com/huly-stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/log" "go.uber.org/zap" ) @@ -48,7 +48,6 @@ func ListenAndServe(ctx context.Context, listenOn string) { if err := server.ListenAndServe(); err != nil { log.FromContext(ctx).Debug("Failed to start profiler", zap.Error(err)) } - <-ctx.Done() _ = server.Close() } diff --git a/internal/pkg/resconv/resconv_test.go b/internal/pkg/resconv/resconv_test.go index 547fab0582..48fd291d57 100644 --- a/internal/pkg/resconv/resconv_test.go +++ b/internal/pkg/resconv/resconv_test.go @@ -16,7 +16,7 @@ package resconv_test import ( "testing" - "github.com/huly-stream/internal/pkg/resconv" + "github.com/hcengineering/stream/internal/pkg/resconv" "github.com/stretchr/testify/require" ) diff --git a/internal/pkg/sharedpipe/shared_pipe_bench_test.go b/internal/pkg/sharedpipe/shared_pipe_bench_test.go index 3633e1fdf4..cafe282dc0 100644 --- a/internal/pkg/sharedpipe/shared_pipe_bench_test.go +++ b/internal/pkg/sharedpipe/shared_pipe_bench_test.go @@ -196,36 +196,35 @@ func Test_Consistent(t *testing.T) { } // Benchmark_DefaultPipe-8 (4 b) 61956 19177 ns/op 48 B/op 1 allocs/op -// Benchmark_DefaultPipe-8 (8 mb) 22 49187741 ns/op 118 B/op 1 allocs/op +// Benchmark_DefaultPipe-8 (8 mb) 24 48471316 ns/op 257 B/op 1 allocs/op func Benchmark_DefaultPipe(b *testing.B) { var data [sendMessageSize]byte var buffer = make([]byte, len(data)) var readers []io.Reader var writers []io.Writer - for i := 0; i < readerCount; i++ { + for range readerCount { r, w := io.Pipe() readers = append(readers, r) writers = append(writers, w) } b.ReportAllocs() - b.ResetTimer() - for range b.N { + for b.Loop() { go func() { - for i := 0; i < readerCount; i++ { + for i := range readerCount { _, _ = writers[i].Write(data[:]) } }() - for i := 0; i < readerCount; i++ { + for i := range readerCount { _, _ = readers[i].Read(buffer) } } } // Benchmark_SharedPipe-8 (4 b) 161847 8131 ns/op 160 B/op 2 allocs/op -// Benchmark_SharedPipe-8 (8 mb) 69 15880031 ns/op 160 B/op 2 allocs/op +// Benchmark_SharedPipe-8 (8 mb) 75 15710658 ns/op 161 B/op 2 allocs/op func Benchmark_SharedPipe(b *testing.B) { var data [sendMessageSize]byte var buffer = make([]byte, len(data)) @@ -237,11 +236,10 @@ func Benchmark_SharedPipe(b *testing.B) { } b.ReportAllocs() - b.ResetTimer() - for range b.N { + for b.Loop() { _, _ = writer.Write(data[:]) - for i := 0; i < readerCount; i++ { + for i := range readerCount { _, _ = readers[i].Read(buffer) } } diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go new file mode 100644 index 0000000000..abe2f69459 --- /dev/null +++ b/internal/pkg/storage/datalake.go @@ -0,0 +1,254 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "os" + "path/filepath" + + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/pkg/errors" + "github.com/valyala/fasthttp" + "go.uber.org/zap" +) + +// DatalakeStorage represents datalake storage +type DatalakeStorage struct { + baseURL string + workspace string + token string + logger *zap.Logger + client fasthttp.Client +} + +// NewDatalakeStorage creates a new datalake client +func NewDatalakeStorage(ctx context.Context, baseURL, workspace, token string) Storage { + return &DatalakeStorage{ + baseURL: baseURL, + token: token, + workspace: workspace, + logger: log.FromContext(ctx).With(zap.String("storage", "datalake")), + } +} + +// PutFile uploads file to the datalake +func (d *DatalakeStorage) PutFile(ctx context.Context, fileName string) error { + // #nosec + file, err := os.Open(fileName) + if err != nil { + return err + } + defer func() { + _ = file.Close() + }() + + var objectKey = getObjectKey(fileName) + var logger = d.logger.With(zap.String("upload", d.workspace), zap.String("fileName", fileName)) + + logger.Debug("start") + + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + + part, err := writer.CreateFormFile("file", objectKey) + if err != nil { + return errors.Wrapf(err, "failed to create form file") + } + + _, err = io.Copy(part, file) + if err != nil { + return errors.Wrapf(err, "failed to copy file data") + } + + _ = writer.Close() + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + + res := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(res) + + req.SetRequestURI(d.baseURL + "/upload/form-data/" + d.workspace) + req.Header.SetMethod(fasthttp.MethodPost) + req.Header.Add("Authorization", "Bearer "+d.token) + req.Header.SetContentType(writer.FormDataContentType()) + req.SetBody(body.Bytes()) + + if err := d.client.Do(req, res); err != nil { + logger.Error("upload failed", zap.Error(err)) + return errors.Wrapf(err, "upload failed") + } + + logger.Debug("uploaded") + + return nil +} + +// DeleteFile deletes file from the datalake +func (d *DatalakeStorage) DeleteFile(ctx context.Context, fileName string) error { + var logger = d.logger.With(zap.String("delete", d.workspace), zap.String("fileName", fileName)) + logger.Debug("start") + + var objectKey = getObjectKey(fileName) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + + res := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(res) + + req.SetRequestURI(d.baseURL + "/blob/" + d.workspace + "/" + objectKey) + req.Header.SetMethod(fasthttp.MethodDelete) + req.Header.Add("Authorization", "Bearer "+d.token) + + if err := d.client.Do(req, res); err != nil { + logger.Error("delete failed", zap.Error(err)) + return errors.Wrapf(err, "delete failed") + } + + logger.Debug("deleted") + + return nil +} + +func getObjectKey(s string) string { + var _, objectKey = filepath.Split(s) + return objectKey +} + +// PatchMeta patches metadata for the object +func (d *DatalakeStorage) PatchMeta(ctx context.Context, filename string, md *Metadata) error { + var logger = d.logger.With(zap.String("patch meta", d.workspace), zap.String("fileName", filename)) + logger.Debug("start") + defer logger.Debug("finished") + + var objectKey = getObjectKey(filename) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(d.baseURL + "/meta/" + d.workspace + "/" + objectKey) + req.Header.SetMethod(fasthttp.MethodPatch) + req.Header.Add("Authorization", "Bearer "+d.token) + req.Header.SetContentType("application/json") + + b, err := json.Marshal(md) + + if err != nil { + return err + } + req.SetBody(b) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + return err + } + + if resp.StatusCode() != fasthttp.StatusOK { + var err = fmt.Errorf("unexpected status code: %d", resp.StatusCode()) + logger.Debug("bad status code", zap.Error(err)) + return err + } + + fmt.Println(string(resp.Body())) + + return nil +} + +// GetMeta gets metadata related to the object +func (d *DatalakeStorage) GetMeta(ctx context.Context, filename string) (*Metadata, error) { + var logger = d.logger.With(zap.String("get meta", d.workspace), zap.String("fileName", filename)) + logger.Debug("start") + + var objectKey = getObjectKey(filename) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(d.baseURL + "/meta/" + d.workspace + "/" + objectKey) + req.Header.SetMethod(fasthttp.MethodGet) + req.Header.Add("Authorization", "Bearer "+d.token) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + return nil, err + } + + if resp.StatusCode() != fasthttp.StatusOK { + var err = fmt.Errorf("unexpected status code: %d", resp.StatusCode()) + logger.Debug("bad status code", zap.Error(err)) + return nil, err + } + + var md Metadata + fmt.Println(string(resp.Body())) + var err = json.Unmarshal(resp.Body(), &md) + + return &md, err +} + +// GetFile gets file from the storage +func (d *DatalakeStorage) GetFile(ctx context.Context, filename, destination string) error { + var logger = d.logger.With(zap.String("get", d.workspace), zap.String("fileName", filename), zap.String("destination", destination)) + logger.Debug("start") + + var objectKey = getObjectKey(filename) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(d.baseURL + "/blob/" + d.workspace + "/" + objectKey) + req.Header.SetMethod(fasthttp.MethodGet) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + return err + } + + // Check the response status code + if resp.StatusCode() != fasthttp.StatusOK { + var err = fmt.Errorf("unexpected status code: %d", resp.StatusCode()) + logger.Debug("bad status code", zap.Error(err)) + return err + } + + // #nosec + file, err := os.Create(destination) + if err != nil { + logger.Debug("can't create a file", zap.Error(err)) + return err + } + defer func() { + _ = file.Close() + }() + if err := resp.BodyWriteTo(file); err != nil { + logger.Debug("can't write to file", zap.Error(err)) + return err + } + + logger.Debug("file downloaded successfully") + return nil +} + +var _ Storage = (*DatalakeStorage)(nil) +var _ MetaProvider = (*DatalakeStorage)(nil) diff --git a/internal/pkg/uploader/s3.go b/internal/pkg/storage/s3.go similarity index 61% rename from internal/pkg/uploader/s3.go rename to internal/pkg/storage/s3.go index d211e0347f..41da2eaaf1 100644 --- a/internal/pkg/uploader/s3.go +++ b/internal/pkg/storage/s3.go @@ -11,14 +11,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package uploader +// Package storage provdies simple storage interface for the remote storages. +package storage import ( "context" "fmt" - "github.com/pkg/errors" - "os" "path/filepath" "strings" @@ -28,8 +27,7 @@ import ( "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/smithy-go" - "github.com/huly-stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/log" "go.uber.org/zap" ) @@ -44,7 +42,6 @@ type S3Storage struct { func NewS3(ctx context.Context, endpoint, bucketName string) Storage { var accessKeyID = os.Getenv("AWS_ACCESS_KEY_ID") var accessKeySecret = os.Getenv("AWS_SECRET_ACCESS_KEY") - var logger = log.FromContext(ctx).With(zap.String("s3", "storage")) cfg, err := config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyID, accessKeySecret, "")), @@ -62,7 +59,7 @@ func NewS3(ctx context.Context, endpoint, bucketName string) Storage { return &S3Storage{ client: s3Client, bucketName: bucketName, - logger: logger, + logger: log.FromContext(ctx).With(zap.String("s3", "storage")), } } @@ -71,7 +68,7 @@ func getContentType(objectKey string) string { return "video/mp2t" } if strings.HasSuffix(objectKey, ".m3u8") { - return "application/x-mpegurl" + return "video/x-mpegurl" } return "application/octet-stream" } @@ -79,9 +76,10 @@ func getContentType(objectKey string) string { // DeleteFile deletes file from the s3 storage func (u *S3Storage) DeleteFile(ctx context.Context, fileName string) error { var _, objectKey = filepath.Split(fileName) - var logger = log.FromContext(ctx).With(zap.String("s3 delete", u.bucketName), zap.String("fileName", fileName)) + var logger = u.logger.With(zap.String("delete", u.bucketName), zap.String("fileName", fileName)) + + logger.Debug("start") - logger.Debug("start deleting") input := &s3.DeleteObjectInput{ Bucket: aws.String(u.bucketName), Key: aws.String(objectKey), @@ -91,15 +89,17 @@ func (u *S3Storage) DeleteFile(ctx context.Context, fileName string) error { if err != nil { return fmt.Errorf("failed to delete file from S3: %w", err) } - logger.Debug("file deleted") + + logger.Debug("deleted") return nil } -// UploadFile uploads file to the s3 storage -func (u *S3Storage) UploadFile(ctx context.Context, fileName string) error { +// PutFile uploads file to the s3 storage +func (u *S3Storage) PutFile(ctx context.Context, fileName string) error { var _, objectKey = filepath.Split(fileName) - var logger = log.FromContext(ctx).With(zap.String("s3 upload", u.bucketName), zap.String("fileName", fileName)) - logger.Debug("start upload file") + var logger = u.logger.With(zap.String("upload", u.bucketName), zap.String("fileName", fileName)) + + logger.Debug("start") // #nosec var file, err = os.Open(fileName) @@ -112,6 +112,7 @@ func (u *S3Storage) UploadFile(ctx context.Context, fileName string) error { defer func() { _ = file.Close() }() + _, err = u.client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(u.bucketName), Key: aws.String(objectKey), @@ -120,23 +121,57 @@ func (u *S3Storage) UploadFile(ctx context.Context, fileName string) error { }) if err != nil { - var apiErr smithy.APIError - if errors.As(err, &apiErr) && apiErr.ErrorCode() == "EntityTooLarge" { - logger.Error("Error while uploading object. The object is too large." + - "To upload objects larger than 5GB, use the S3 console (160GB max)" + - "or the multipart upload API (5TB max).") - } else { - logger.Error("Couldn't upload file", zap.Error(err)) - } - return apiErr + logger.Error("couldn't upload file", zap.Error(err)) + return err } err = s3.NewObjectExistsWaiter(u.client).Wait( ctx, &s3.HeadObjectInput{Bucket: aws.String(u.bucketName), Key: aws.String(objectKey)}, time.Minute) + if err != nil { - logger.Debug("Failed attempt to wait for object to exist.") + logger.Error("Failed attempt to wait for object to exist") + return err } - logger.Debug("file has uploaded") - return err + logger.Debug("uploaded") + return nil +} + +// GetFile gets file from the storage and stores it to destination +func (u *S3Storage) GetFile(ctx context.Context, filename, dest string) error { + var logger = u.logger.With(zap.String("get", u.bucketName), zap.String("fileName", filename), zap.String("destination", dest)) + + var result, err = u.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &u.bucketName, + Key: &filename, + }) + + if err != nil { + logger.Error("failed to get object", zap.Error(err)) + return err + } + defer func() { + _ = result.Body.Close() + }() + + // Create a local file to save the downloaded content + // #nosec + file, err := os.Create(dest) + if err != nil { + logger.Error("failed to create file", zap.Error(err)) + return err + } + defer func() { + _ = file.Close() + }() + // Copy the S3 object content to the local file + _, err = file.ReadFrom(result.Body) + if err != nil { + logger.Error("failed to write to file", zap.Error(err)) + return err + } + + logger.Debug("file downloaded successfully") + + return nil } diff --git a/internal/pkg/storage/storage.go b/internal/pkg/storage/storage.go new file mode 100644 index 0000000000..e45250284d --- /dev/null +++ b/internal/pkg/storage/storage.go @@ -0,0 +1,55 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "net/url" + + "github.com/pkg/errors" +) + +// Metadata represents file's metadata +type Metadata map[string]any + +// MetaProvider provides simple api for working with files metadata +type MetaProvider interface { + GetMeta(ctx context.Context, filename string) (*Metadata, error) + PatchMeta(ctx context.Context, filename string, value *Metadata) error +} + +// Storage represents file-based storage +type Storage interface { + PutFile(ctx context.Context, fileName string) error + DeleteFile(ctx context.Context, fileName string) error + GetFile(ctx context.Context, fileName, destination string) error +} + +// NewStorageByURL creates a new storage based on the type from the url scheme, for example "datalake://my-datalake-endpoint" +func NewStorageByURL(ctx context.Context, u *url.URL, storageType, token, worksapce string) (Storage, error) { + if worksapce == "" { + return nil, errors.New("workspace is missed") + } + switch storageType { + case "datalake": + if token == "" { + return nil, errors.New("token is missed") + } + return NewDatalakeStorage(ctx, u.String(), worksapce, token), nil + case "s3": + return NewS3(ctx, u.String(), worksapce), nil + default: + return nil, errors.New("unknown scheme") + } +} diff --git a/internal/pkg/token/token.go b/internal/pkg/token/token.go new file mode 100644 index 0000000000..1f81a76484 --- /dev/null +++ b/internal/pkg/token/token.go @@ -0,0 +1,77 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package token provides functions to work with platform tokens +package token + +import ( + "fmt" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/google/uuid" +) + +// Token represents Claims for the platform token +type Token struct { + jwt.RegisteredClaims + Account string `json:"account"` + Workspace string `json:"workspace,omitempty"` + Extra map[string]interface{} `json:"extra,omitempty"` +} + +// NewToken creates a new platform token +func NewToken(serverSecret, workspace, issuer, audience string) (string, error) { + var res = Token{ + Account: uuid.NewString(), + Workspace: workspace, + RegisteredClaims: jwt.RegisteredClaims{ + Issuer: issuer, + Audience: jwt.ClaimStrings{audience}, + ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Hour * 12)), + }, + } + return res.Encode(serverSecret) +} + +// Encode signes a token object and converts it to string +func (t *Token) Encode(serverSecret string) (string, error) { + var token = jwt.NewWithClaims(jwt.SigningMethodHS256, t) + var tokenString, err = token.SignedString([]byte(serverSecret)) + + if err != nil { + return "", err + } + + return tokenString, nil +} + +// Decode decodes a token by a passed configuration +func Decode(secretKey, tokenString string) (*Token, error) { + token, err := jwt.ParseWithClaims(tokenString, &Token{}, func(token *jwt.Token) (interface{}, error) { + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) + } + return []byte(secretKey), nil + }) + + if err != nil { + return nil, fmt.Errorf("failed to parse token: %w", err) + } + + if claims, ok := token.Claims.(*Token); ok && token.Valid { + return claims, nil + } + + return nil, fmt.Errorf("invalid token: can't parse claims") +} diff --git a/internal/pkg/uploader/postpone_test.go b/internal/pkg/token/token_test.go similarity index 50% rename from internal/pkg/uploader/postpone_test.go rename to internal/pkg/token/token_test.go index 68758451e7..93d97bdb53 100644 --- a/internal/pkg/uploader/postpone_test.go +++ b/internal/pkg/token/token_test.go @@ -11,35 +11,30 @@ // See the License for the specific language governing permissions and // limitations under the License. -package uploader +package token_test import ( - "context" - "sync/atomic" "testing" "time" + "github.com/golang-jwt/jwt/v5" + "github.com/hcengineering/stream/internal/pkg/token" "github.com/stretchr/testify/require" ) -func Test_Postpone(t *testing.T) { - var u = uploader{ - postponeDuration: time.Second / 4, - } - var counter atomic.Int32 - u.postpone("1", func(context.Context) { counter.Add(1) }) - time.Sleep(time.Second / 8) - u.postpone("1", func(context.Context) { counter.Add(1) }) - time.Sleep(time.Second / 2) - require.Equal(t, int32(1), counter.Load()) - time.Sleep(time.Second / 2) - require.Equal(t, int32(1), counter.Load()) +func Test_GenerateSimpleServiceToken(t *testing.T) { + var _, err = token.NewToken("secret", "ws", "issuer", "aud") + require.NoError(t, err) } -func Test_WithoutPostpone(t *testing.T) { - var counter atomic.Int32 - var u uploader - u.postpone("1", func(context.Context) { counter.Add(1) }) - time.Sleep(time.Second / 10) - require.Equal(t, int32(1), counter.Load()) +func Test_ParseSimpleServiceToken(t *testing.T) { + const secret = "secret" + tokenString, err := token.NewToken(secret, "ws", "issuer", "aud") + require.NoError(t, err) + tok, err := token.Decode(secret, tokenString) + require.NoError(t, err) + require.Equal(t, tok.Issuer, "issuer") + require.Equal(t, tok.Audience, jwt.ClaimStrings{"aud"}) + require.Equal(t, tok.Workspace, "ws") + require.True(t, tok.ExpiresAt.After(time.Now())) } diff --git a/internal/pkg/transcoding/command.go b/internal/pkg/transcoder/command.go similarity index 93% rename from internal/pkg/transcoding/command.go rename to internal/pkg/transcoder/command.go index 0065fa3196..9cc65d9637 100644 --- a/internal/pkg/transcoding/command.go +++ b/internal/pkg/transcoder/command.go @@ -13,7 +13,7 @@ // limitations under the License. // -package transcoding +package transcoder import ( "context" @@ -25,13 +25,13 @@ import ( "github.com/pkg/errors" - "github.com/huly-stream/internal/pkg/log" - "github.com/huly-stream/internal/pkg/resconv" + "github.com/hcengineering/stream/internal/pkg/log" "go.uber.org/zap" ) // Options represents configuration for the ffmpeg command type Options struct { + Input string OuputDir string ScalingLevels []string Level string @@ -58,9 +58,8 @@ func newFfmpegCommand(ctx context.Context, in io.Reader, args []string) (*exec.C func buildCommonComamnd(opts *Options) []string { return []string{ - "-nostdin", "-threads", fmt.Sprint(opts.Threads), - "-i", "pipe:0", + "-i", opts.Input, } } @@ -79,8 +78,6 @@ func BuildRawVideoCommand(opts *Options) []string { return append(buildCommonComamnd(opts), "-c:v", "copy", - "-fps_mode", - "vfr", "-hls_time", "5", "-hls_list_size", "0", "-hls_segment_filename", filepath.Join(opts.OuputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", opts.Level)), @@ -90,9 +87,10 @@ func BuildRawVideoCommand(opts *Options) []string { // BuildScalingVideoCommand returns flags for ffmpeg for video scaling func BuildScalingVideoCommand(opts *Options) []string { var result = buildCommonComamnd(opts) + for _, level := range opts.ScalingLevels { result = append(result, - "-vf", "scale="+resconv.Resolution(level), + "-vf", "scale=-2:"+level[:len(level)-1], "-c:v", "libx264", "-preset", "veryfast", diff --git a/internal/pkg/transcoding/command_test.go b/internal/pkg/transcoder/command_test.go similarity index 56% rename from internal/pkg/transcoding/command_test.go rename to internal/pkg/transcoder/command_test.go index 719c38467a..db382a555e 100644 --- a/internal/pkg/transcoding/command_test.go +++ b/internal/pkg/transcoder/command_test.go @@ -11,39 +11,41 @@ // See the License for the specific language governing permissions and // limitations under the License. -package transcoding_test +package transcoder_test import ( "strings" "testing" - "github.com/huly-stream/internal/pkg/resconv" - "github.com/huly-stream/internal/pkg/transcoding" + "github.com/hcengineering/stream/internal/pkg/resconv" + "github.com/hcengineering/stream/internal/pkg/transcoder" "github.com/stretchr/testify/require" ) func Test_BuildVideoCommand_Scaling(t *testing.T) { - var scaleCommand = transcoding.BuildScalingVideoCommand(&transcoding.Options{ + var scaleCommand = transcoder.BuildScalingVideoCommand(&transcoder.Options{ OuputDir: "test", + Input: "pipe:0", UploadID: "1", Threads: 4, ScalingLevels: []string{"720p", "480p"}, }) - const expected = `-nostdin -threads 4 -i pipe:0 -vf scale=1280:720 -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -vf scale=640:480 -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `-threads 4 -i pipe:0 -vf scale=-2:720 -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -vf scale=-2:480 -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(scaleCommand, " ")) } func Test_BuildVideoCommand_Raw(t *testing.T) { - var rawCommand = transcoding.BuildRawVideoCommand(&transcoding.Options{ + var rawCommand = transcoder.BuildRawVideoCommand(&transcoder.Options{ OuputDir: "test", + Input: "pipe:0", UploadID: "1", Threads: 4, Level: resconv.Level("651:490"), }) - const expected = `-nostdin -threads 4 -i pipe:0 -c:v copy -fps_mode vfr -hls_time 5 -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `"-threads 4 -i pipe:0 -c:v copy -hls_time 5 -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(rawCommand, " ")) } diff --git a/internal/pkg/transcoder/coordinator.go b/internal/pkg/transcoder/coordinator.go new file mode 100644 index 0000000000..d0705cb0ce --- /dev/null +++ b/internal/pkg/transcoder/coordinator.go @@ -0,0 +1,190 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package transcoder + +import ( + "context" + "path/filepath" + "sync" + "sync/atomic" + "time" + + "github.com/pkg/errors" + + "github.com/google/uuid" + "github.com/hcengineering/stream/internal/pkg/config" + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/resconv" + "github.com/hcengineering/stream/internal/pkg/sharedpipe" + "github.com/hcengineering/stream/internal/pkg/storage" + "github.com/hcengineering/stream/internal/pkg/uploader" + "github.com/tus/tusd/v2/pkg/handler" + "go.uber.org/zap" +) + +// StreamCoordinator represents manager for streams. It creates a new stream for a client and manages it's life cycle. +type StreamCoordinator struct { + conf *config.Config + uploadOptions uploader.Options + + activeScalling int32 + + mainContext context.Context + logger *zap.Logger + + streams sync.Map + cancels sync.Map +} + +// NewStreamCoordinator creates a new scheduler for transcode operations. +func NewStreamCoordinator(ctx context.Context, c *config.Config) *StreamCoordinator { + return &StreamCoordinator{ + conf: c, + uploadOptions: uploader.Options{ + RetryDelay: time.Millisecond * 100, + Timeout: c.Timeout, + WorkerCount: uint32(c.MaxThreadCount), + RetryCount: 5, + BufferSize: 128, + Dir: c.OutputDir, + }, + mainContext: ctx, + logger: log.FromContext(ctx).With(zap.String("Scheduler", c.OutputDir)), + } +} + +// NewUpload creates a new worker with passed parameters +func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo) (handler.Upload, error) { + if info.ID == "" { + info.ID = uuid.NewString() + } + s.logger.Sugar().Debugf("stream: %v", info) + s.logger.Debug("NewUpload", zap.String("ID", info.ID)) + + var stream = &Stream{ + writer: sharedpipe.NewWriter(), + info: info, + logger: log.FromContext(s.mainContext).With(zap.String("worker", info.ID)), + done: make(chan struct{}), + } + + var scaling = resconv.SubLevels(info.MetaData["resolution"]) + var level = resconv.Level(info.MetaData["resolution"]) + var cost int64 + + for _, scale := range scaling { + cost += int64(resconv.Pixels(resconv.Resolution(scale))) + } + + if atomic.AddInt32(&s.activeScalling, 1) > int32(s.conf.MaxParallelScalingCount) { + atomic.AddInt32(&s.activeScalling, -1) + s.logger.Debug("run out of resources for scaling") + scaling = nil + } + + var commandOptions = Options{ + Input: "pipe:0", + OuputDir: s.conf.OutputDir, + Threads: s.conf.MaxThreadCount, + UploadID: info.ID, + Level: level, + ScalingLevels: scaling, + } + + if s.conf.EndpointURL != nil { + s.logger.Sugar().Debugf("initializing uploader for %v", info) + var opts = s.uploadOptions + opts.Dir = filepath.Join(opts.Dir, info.ID) + + var storage, err = storage.NewStorageByURL(s.mainContext, s.conf.Endpoint(), s.conf.EndpointURL.Scheme, info.MetaData["token"], info.MetaData["workspace"]) + if err != nil { + s.logger.Error("can not create storage by url") + return nil, err + } + var contentUploader = uploader.New(s.mainContext, storage, opts) + + stream.contentUploader = contentUploader + } + + s.streams.Store(stream.info.ID, stream) + if err := stream.start(s.mainContext, &commandOptions); err != nil { + return nil, err + } + + go func() { + stream.commandGroup.Wait() + if scaling != nil { + atomic.AddInt32(&s.activeScalling, -1) + } + s.logger.Debug("returned capacity", zap.Int64("capacity", cost)) + close(stream.done) + }() + + s.manageTimeout(stream) + + s.logger.Debug("NewUpload", zap.String("done", info.ID)) + return stream, nil +} + +// GetUpload returns current a worker based on upload id +func (s *StreamCoordinator) GetUpload(ctx context.Context, id string) (upload handler.Upload, err error) { + if v, ok := s.streams.Load(id); ok { + s.logger.Debug("GetUpload: found worker by id", zap.String("id", id)) + var w = v.(*Stream) + s.manageTimeout(w) + return w, nil + } + s.logger.Debug("GetUpload: worker not found", zap.String("id", id)) + return nil, errors.New("bad id") +} + +// AsTerminatableUpload returns tusd handler.TerminatableUpload +func (s *StreamCoordinator) AsTerminatableUpload(upload handler.Upload) handler.TerminatableUpload { + var worker = upload.(*Stream) + s.logger.Debug("AsTerminatableUpload") + return worker +} + +// AsLengthDeclarableUpload returns tusd handler.LengthDeclarableUpload +func (s *StreamCoordinator) AsLengthDeclarableUpload(upload handler.Upload) handler.LengthDeclarableUpload { + s.logger.Debug("AsLengthDeclarableUpload") + return upload.(*Stream) +} + +func (s *StreamCoordinator) manageTimeout(w *Stream) { + var cancelCtx, cancel = context.WithCancel(context.Background()) + if v, ok := s.cancels.Load(w.info.ID); ok { + v.(context.CancelFunc)() + } + s.cancels.Store(w.info.ID, cancel) + go func() { + select { + case <-w.done: + w.logger.Debug("stream has finished") + s.cancels.Delete(w.info.ID) + return + case <-cancelCtx.Done(): + w.logger.Debug("stream timeout has refreshed") + return + case <-time.After(s.conf.Timeout): + w.logger.Error("stream timeout") + s.cancels.Delete(w.info.ID) + var terminateCtx, terminateCancel = context.WithTimeout(context.Background(), s.conf.Timeout) + defer terminateCancel() + _ = w.Terminate(terminateCtx) + } + }() +} diff --git a/internal/pkg/transcoder/queue.go b/internal/pkg/transcoder/queue.go new file mode 100644 index 0000000000..4a82ae107b --- /dev/null +++ b/internal/pkg/transcoder/queue.go @@ -0,0 +1,234 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package transcoder + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/google/uuid" + "github.com/hcengineering/stream/internal/pkg/config" + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/manifest" + "github.com/hcengineering/stream/internal/pkg/resconv" + "github.com/hcengineering/stream/internal/pkg/storage" + "github.com/hcengineering/stream/internal/pkg/token" + "github.com/hcengineering/stream/internal/pkg/uploader" + "go.uber.org/zap" + "gopkg.in/vansante/go-ffprobe.v2" +) + +// HLS represents metadata for transcoding result +type HLS struct { + Source string `json:"source"` +} + +// Task represents transcoding task +type Task struct { + ID string + Status string + Source string + Format string + Workspace string + Metadata map[string]string +} + +// Scheduler manages transcoding tasks by passed config +type Scheduler struct { + logger *zap.Logger + taskCh chan *Task + cfg *config.Config + ctx context.Context +} + +// Schedule schedules a task to transcode +func (p *Scheduler) Schedule(t *Task) { + t.ID = uuid.NewString() + t.Status = "planned" + + select { + case p.taskCh <- t: + p.logger.Sugar().Debugf("task %v is scheduled", t) + default: + p.logger.Error("task channel is full") + } +} + +// NewScheduler creates a new instance of transcoding task scheduler +func NewScheduler(ctx context.Context, cfg *config.Config) *Scheduler { + var p = &Scheduler{ + taskCh: make(chan *Task, 128), + cfg: cfg, + ctx: ctx, + logger: log.FromContext(ctx).With(zap.String("transcoding", "planner")), + } + + go p.start() + + return p +} + +func (p *Scheduler) start() { + go func() { + <-p.ctx.Done() + close(p.taskCh) + }() + + for range p.cfg.MaxParallelScalingCount { + go func() { + for task := range p.taskCh { + p.processTask(p.ctx, task) + } + }() + } +} + +func (p *Scheduler) processTask(ctx context.Context, task *Task) { + var logger = p.logger.With(zap.String("task-id", task.ID)) + + logger.Debug("start") + defer logger.Debug("finished") + + logger.Debug("phase 1: get a token") + var tokenString, err = token.NewToken(p.cfg.ServerSecret, task.Workspace, "stream", "datalake") + if err != nil { + logger.Error("can not create token", zap.Error(err)) + return + } + + logger.Debug("phase 2: preparing fs") + var destinationFolder = filepath.Join(p.cfg.OutputDir, task.ID) + var _, filename = filepath.Split(task.Source) + var sourceFilePath = filepath.Join(destinationFolder, filename) + _ = os.MkdirAll(destinationFolder, os.ModePerm) + + logger.Debug("phase 3: get the remote file") + + remoteStorage, err := storage.NewStorageByURL(ctx, p.cfg.Endpoint(), p.cfg.EndpointURL.Scheme, tokenString, task.Workspace) + + if err != nil { + logger.Error("can not create storage by url", zap.Error(err)) + _ = os.RemoveAll(destinationFolder) + return + } + + if err = remoteStorage.GetFile(ctx, task.Source, sourceFilePath); err != nil { + logger.Error("can not download a file", zap.Error(err)) + _ = os.RemoveAll(destinationFolder) + // TODO: reschedule + return + } + + logger.Debug("phase 4: prepare to transcode") + probe, err := ffprobe.ProbeURL(ctx, sourceFilePath) + if err != nil { + logger.Error("can not get probe for a file", zap.Error(err), zap.String("filepath", sourceFilePath)) + _ = os.RemoveAll(destinationFolder) + return + } + + var res = fmt.Sprintf("%v:%v", probe.FirstVideoStream().Width, probe.FirstVideoStream().Height) + var level = resconv.Level(res) + var opts = Options{ + Input: sourceFilePath, + OuputDir: p.cfg.OutputDir, + Level: level, + ScalingLevels: append(resconv.SubLevels(res), level), + UploadID: task.ID, + Threads: p.cfg.MaxThreadCount, + } + + logger.Debug("phase 5: start async upload process") + var uploader = uploader.New(ctx, remoteStorage, uploader.Options{ + Dir: destinationFolder, + WorkerCount: uint32(opts.Threads), + BufferSize: 128, + RetryCount: 5, + RetryDelay: time.Millisecond * 100, + Timeout: p.cfg.Timeout, + SourceFile: sourceFilePath, + }) + + go uploader.Start() + + logger.Debug("phase 6: start async transcode process") + var rawCommand, scaleCommand *exec.Cmd + + rawCommand, err = newFfmpegCommand(ctx, nil, BuildRawVideoCommand(&opts)) + if err != nil { + logger.Error("can not create ffmpeg command", zap.Error(err)) + go uploader.Cancel() + return + } + + scaleCommand, err = newFfmpegCommand(ctx, nil, BuildScalingVideoCommand(&opts)) + if err != nil { + logger.Error("can not create ffmpeg command", zap.Error(err)) + go uploader.Cancel() + return + } + + _ = manifest.GenerateHLSPlaylist(opts.ScalingLevels, p.cfg.OutputDir, opts.UploadID) + + if err = rawCommand.Start(); err != nil { + logger.Error("can not run raw ffmpeg command", zap.Error(err)) + go uploader.Cancel() + return + } + + if err = scaleCommand.Start(); err != nil { + logger.Error("can not run scale ffmpeg command", zap.Error(err)) + go uploader.Cancel() + return + } + + logger.Debug("phase 7: wait for the result") + if err = scaleCommand.Wait(); err != nil { + logger.Error("can not scale ", zap.Error(err)) + go uploader.Cancel() + return + } + if err = rawCommand.Wait(); err != nil { + logger.Error("can not process raw", zap.Error(err)) + go uploader.Cancel() + return + } + + logger.Debug("phase 8: schedule cleanup") + go uploader.Stop() + + logger.Debug("phase 9: try to set metadata") + var resultURL = p.cfg.Endpoint().JoinPath("blob", task.Workspace, task.ID+"_master.m3u8") + + if metaProvider, ok := remoteStorage.(storage.MetaProvider); ok { + var hls = &HLS{Source: resultURL.String()} + logger.Debug("applying metadata", zap.Stringer("url", resultURL), zap.String("source", task.Source)) + err = metaProvider.PatchMeta( + ctx, + task.Source, + &storage.Metadata{ + "hls": hls, + }, + ) + if err != nil { + logger.Error("can not patch the source file", zap.Error(err)) + } + } +} diff --git a/internal/pkg/transcoding/worker.go b/internal/pkg/transcoder/stream.go similarity index 72% rename from internal/pkg/transcoding/worker.go rename to internal/pkg/transcoder/stream.go index c2aaca3d34..d59c12c09f 100644 --- a/internal/pkg/transcoding/worker.go +++ b/internal/pkg/transcoder/stream.go @@ -11,8 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package transcoding provides objects and functions for video trnascoding -package transcoding +// Package transcoder provides types and functions for video trnascoding +package transcoder import ( "context" @@ -21,27 +21,27 @@ import ( "github.com/pkg/errors" - "github.com/huly-stream/internal/pkg/manifest" - "github.com/huly-stream/internal/pkg/sharedpipe" - "github.com/huly-stream/internal/pkg/uploader" + "github.com/hcengineering/stream/internal/pkg/manifest" + "github.com/hcengineering/stream/internal/pkg/sharedpipe" + "github.com/hcengineering/stream/internal/pkg/uploader" "github.com/tus/tusd/v2/pkg/handler" "go.uber.org/zap" ) -// Worker manages client's input and transcodes it based on the passsed configuration -type Worker struct { +// Stream manages client's input and transcodes it based on the passsed configuration +type Stream struct { contentUploader uploader.Uploader logger *zap.Logger info handler.FileInfo writer *sharedpipe.Writer reader *sharedpipe.Reader - wg sync.WaitGroup - done chan struct{} + commandGroup sync.WaitGroup + done chan struct{} } // WriteChunk calls when client sends a chunk of raw data -func (w *Worker) WriteChunk(ctx context.Context, _ int64, src io.Reader) (int64, error) { +func (w *Stream) WriteChunk(ctx context.Context, _ int64, src io.Reader) (int64, error) { w.logger.Debug("Write Chunk start", zap.Int64("offset", w.info.Offset)) var bytes, err = io.ReadAll(src) _, _ = w.writer.Write(bytes) @@ -52,7 +52,7 @@ func (w *Worker) WriteChunk(ctx context.Context, _ int64, src io.Reader) (int64, } // DeclareLength sets length of the video input -func (w *Worker) DeclareLength(ctx context.Context, length int64) error { +func (w *Stream) DeclareLength(ctx context.Context, length int64) error { w.info.Size = length w.info.SizeIsDeferred = false w.logger.Debug("DeclareLength", zap.Int64("size", length), zap.Bool("SizeIsDeferred", w.info.SizeIsDeferred)) @@ -60,65 +60,67 @@ func (w *Worker) DeclareLength(ctx context.Context, length int64) error { } // GetInfo returns info about transcoing status -func (w *Worker) GetInfo(ctx context.Context) (handler.FileInfo, error) { +func (w *Stream) GetInfo(ctx context.Context) (handler.FileInfo, error) { w.logger.Debug("GetInfo is executed") return w.info, nil } -// GetReader returns worker's bytes stream -func (w *Worker) GetReader(ctx context.Context) (io.ReadCloser, error) { +// GetReader returns Stream's bytes stream +func (w *Stream) GetReader(ctx context.Context) (io.ReadCloser, error) { w.logger.Debug("GetReader is executed, creating current reader...") return w.reader, nil } // Terminate calls when upload has failed -func (w *Worker) Terminate(ctx context.Context) error { +func (w *Stream) Terminate(ctx context.Context) error { w.logger.Debug("Terminating...") if w.contentUploader != nil { go func() { - w.wg.Wait() - w.contentUploader.Rollback() + w.commandGroup.Wait() + w.contentUploader.Cancel() }() } return w.writer.Close() } // ConcatUploads calls when upload resumed after fail -func (w *Worker) ConcatUploads(ctx context.Context, partialUploads []handler.Upload) error { +func (w *Stream) ConcatUploads(ctx context.Context, partialUploads []handler.Upload) error { w.logger.Debug("ConcatUploads was executed, it's not implemented") // - // TODO: load raw source from the Buckup bucket, terminate all workers with same ID and start process again. + // TODO: load raw source from the Buckup bucket, terminate all Streams with same ID and start process again. // return errors.New("not implemented") } // FinishUpload calls when upload finished without errors on the client side -func (w *Worker) FinishUpload(ctx context.Context) error { +func (w *Stream) FinishUpload(ctx context.Context) error { w.logger.Debug("finishing upload...") + if w.contentUploader != nil { go func() { - w.wg.Wait() - w.contentUploader.Terminate() + w.commandGroup.Wait() + w.contentUploader.Stop() }() } + return w.writer.Close() } // AsConcatableUpload returns tusd handler.ConcatableUpload -func (s *Scheduler) AsConcatableUpload(upload handler.Upload) handler.ConcatableUpload { +func (s *StreamCoordinator) AsConcatableUpload(upload handler.Upload) handler.ConcatableUpload { s.logger.Debug("AsConcatableUpload is executed") - return upload.(*Worker) + return upload.(*Stream) } -func (w *Worker) start(ctx context.Context, options *Options) error { +func (w *Stream) start(ctx context.Context, options *Options) error { defer w.logger.Debug("start done") w.reader = w.writer.Transpile() if err := manifest.GenerateHLSPlaylist(append(options.ScalingLevels, options.Level), options.OuputDir, options.UploadID); err != nil { return err } - w.wg.Add(1) + w.commandGroup.Add(1) go func() { - defer w.wg.Done() + defer w.commandGroup.Done() var logger = w.logger.With(zap.String("command", "raw")) defer logger.Debug("done") @@ -134,10 +136,11 @@ func (w *Worker) start(ctx context.Context, options *Options) error { }() if len(options.ScalingLevels) > 0 { - w.wg.Add(1) + w.commandGroup.Add(1) var scalingCommandReader = w.writer.Transpile() + go func() { - defer w.wg.Done() + defer w.commandGroup.Done() var logger = w.logger.With(zap.String("command", "scaling")) defer logger.Debug("done") @@ -153,5 +156,7 @@ func (w *Worker) start(ctx context.Context, options *Options) error { }() } + go w.contentUploader.Start() + return nil } diff --git a/internal/pkg/transcoding/limiter.go b/internal/pkg/transcoding/limiter.go deleted file mode 100644 index f1fdf51d9f..0000000000 --- a/internal/pkg/transcoding/limiter.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright © 2025 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. - -package transcoding - -import "sync/atomic" - -// Limiter is a simple CAS data structure for managing resources. -type Limiter struct { - capacity int64 - maxCapacity int64 -} - -// NewLimiter creates a new limiter with the given initial capacity. -func NewLimiter(capacity int64) *Limiter { - return &Limiter{ - capacity: capacity, - maxCapacity: capacity, - } -} - -// TryConsume attempts to consume the specified amount of capacity. -// Returns true if successful, false otherwise. -func (l *Limiter) TryConsume(amount int64) bool { - if amount <= 0 { - return false - } - - for { - current := atomic.LoadInt64(&l.capacity) - if current < amount { - return false - } - updated := current - amount - if atomic.CompareAndSwapInt64(&l.capacity, current, updated) { - return true - } - } -} - -// ReturnCapacity adds the specified amount back to the limiter's capacity. -// Does not exceed the maximum capacity. -func (l *Limiter) ReturnCapacity(amount int64) { - if amount <= 0 { - return - } - - for { - current := atomic.LoadInt64(&l.capacity) - updated := current + amount - if updated > l.maxCapacity { - updated = l.maxCapacity - } - if atomic.CompareAndSwapInt64(&l.capacity, current, updated) { - break - } - } -} - -// GetCapacity retrieves the current capacity for debugging or monitoring purposes. -func (l *Limiter) GetCapacity() int64 { - return atomic.LoadInt64(&l.capacity) -} - -// GetMaxCapacity retrieves the maximum capacity. -func (l *Limiter) GetMaxCapacity() int64 { - return l.maxCapacity -} diff --git a/internal/pkg/transcoding/limiter_test.go b/internal/pkg/transcoding/limiter_test.go deleted file mode 100644 index df7db82ba0..0000000000 --- a/internal/pkg/transcoding/limiter_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright © 2025 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -package transcoding_test - -import ( - "sync" - "sync/atomic" - "testing" - - "github.com/huly-stream/internal/pkg/transcoding" - "github.com/stretchr/testify/require" -) - -func TestLimiter(t *testing.T) { - limiter := transcoding.NewLimiter(10) - - t.Run("Initial capacity", func(t *testing.T) { - require.Equal(t, int64(10), limiter.GetCapacity()) - }) - - t.Run("Successful consume", func(t *testing.T) { - success := limiter.TryConsume(5) - require.True(t, success) - require.Equal(t, int64(5), limiter.GetCapacity()) - }) - - t.Run("Failed consume", func(t *testing.T) { - success := limiter.TryConsume(10) - require.False(t, success) - require.Equal(t, int64(5), limiter.GetCapacity()) - }) - - t.Run("Return capacity", func(t *testing.T) { - limiter.ReturnCapacity(3) - require.Equal(t, int64(8), limiter.GetCapacity()) - }) - - t.Run("Exceeding max capacity", func(t *testing.T) { - limiter.ReturnCapacity(10) - require.Equal(t, int64(10), limiter.GetCapacity()) - }) -} - -func TestLimiterConcurrency(t *testing.T) { - limiter := transcoding.NewLimiter(10) - var wg sync.WaitGroup - - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - limiter.TryConsume(2) - }() - } - - wg.Wait() - require.LessOrEqual(t, limiter.GetCapacity(), int64(0)) -} - -func TestLimiterCAS(t *testing.T) { - limiter := transcoding.NewLimiter(10) - var successful int64 - var wg sync.WaitGroup - - for i := 0; i < 1000; i++ { - wg.Add(1) - go func() { - defer wg.Done() - if limiter.TryConsume(1) { - atomic.AddInt64(&successful, 1) - } - }() - } - wg.Wait() - - require.Equal(t, int64(10), successful) - require.Equal(t, int64(0), limiter.GetCapacity()) -} diff --git a/internal/pkg/transcoding/scheduler.go b/internal/pkg/transcoding/scheduler.go deleted file mode 100644 index 080aeedbb5..0000000000 --- a/internal/pkg/transcoding/scheduler.go +++ /dev/null @@ -1,167 +0,0 @@ -// -// Copyright © 2025 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package transcoding - -import ( - "context" - "sync" - "time" - - "github.com/pkg/errors" - - "github.com/google/uuid" - "github.com/huly-stream/internal/pkg/config" - "github.com/huly-stream/internal/pkg/log" - "github.com/huly-stream/internal/pkg/resconv" - "github.com/huly-stream/internal/pkg/sharedpipe" - "github.com/huly-stream/internal/pkg/uploader" - "github.com/tus/tusd/v2/pkg/handler" - "go.uber.org/zap" -) - -// Scheduler represents manager for worker. It creates a new worker for clients and manages its life cycle. -type Scheduler struct { - conf *config.Config - - limiter *Limiter - - mainContext context.Context - logger *zap.Logger - workers sync.Map - cancels sync.Map -} - -// NewScheduler creates a new scheduler for transcode operations. -func NewScheduler(ctx context.Context, c *config.Config) *Scheduler { - return &Scheduler{ - conf: c, - limiter: NewLimiter(c.MaxCapacity), - mainContext: ctx, - logger: log.FromContext(ctx).With(zap.String("Scheduler", c.OutputDir)), - } -} - -// NewUpload creates a new worker with passed parameters -func (s *Scheduler) NewUpload(ctx context.Context, info handler.FileInfo) (handler.Upload, error) { - if info.ID == "" { - info.ID = uuid.NewString() - } - s.logger.Sugar().Debugf("upload: %v", info) - s.logger.Debug("NewUpload", zap.String("ID", info.ID)) - - var worker = &Worker{ - writer: sharedpipe.NewWriter(), - info: info, - logger: log.FromContext(s.mainContext).With(zap.String("worker", info.ID)), - done: make(chan struct{}), - } - - var scaling = resconv.SubLevels(info.MetaData["resolution"]) - var level = resconv.Level(info.MetaData["resolution"]) - var cost int64 - - for _, scale := range scaling { - cost += int64(resconv.Pixels(resconv.Resolution(scale))) - } - - if !s.limiter.TryConsume(cost) { - s.logger.Debug("run out of resources for scaling") - scaling = nil - } - - var commandOptions = Options{ - OuputDir: s.conf.OutputDir, - Threads: s.conf.MaxThreads, - UploadID: info.ID, - Level: level, - ScalingLevels: scaling, - } - - if s.conf.EndpointURL != nil { - s.logger.Sugar().Debugf("initializing uploader for %v", info) - var contentUploader, err = uploader.New(s.mainContext, s.conf.OutputDir, s.conf.EndpointURL, info) - if err != nil { - s.logger.Error("can not create uploader", zap.Error(err)) - return nil, err - } - - worker.contentUploader = contentUploader - go func() { - var serverErr = worker.contentUploader.Serve() - worker.logger.Debug("content uploader has finished", zap.Error(serverErr)) - }() - } - s.workers.Store(worker.info.ID, worker) - if err := worker.start(s.mainContext, &commandOptions); err != nil { - return nil, err - } - - go func() { - worker.wg.Wait() - s.limiter.ReturnCapacity(cost) - s.logger.Debug("returned capacity", zap.Int64("capacity", cost)) - close(worker.done) - }() - - s.logger.Debug("NewUpload", zap.String("done", info.ID)) - return worker, nil -} - -// GetUpload returns current a worker based on upload id -func (s *Scheduler) GetUpload(ctx context.Context, id string) (upload handler.Upload, err error) { - if v, ok := s.workers.Load(id); ok { - s.logger.Debug("GetUpload: found worker by id", zap.String("id", id)) - var w = v.(*Worker) - var cancelCtx, cancel = context.WithCancel(context.Background()) - if v, ok := s.cancels.Load(id); ok { - v.(context.CancelFunc)() - } - s.cancels.Store(id, cancel) - go func() { - select { - case <-w.done: - w.logger.Debug("upload timeout just canceled") - s.cancels.Delete(id) - return - case <-cancelCtx.Done(): - w.logger.Debug("upload refreshed") - return - case <-time.After(s.conf.Timeout): - w.logger.Debug("upload timeout") - s.cancels.Delete(id) - var terminateCtx, terminateCancel = context.WithTimeout(context.Background(), s.conf.Timeout) - defer terminateCancel() - _ = w.Terminate(terminateCtx) - } - }() - return w, nil - } - s.logger.Debug("GetUpload: worker not found", zap.String("id", id)) - return nil, errors.New("bad id") -} - -// AsTerminatableUpload returns tusd handler.TerminatableUpload -func (s *Scheduler) AsTerminatableUpload(upload handler.Upload) handler.TerminatableUpload { - var worker = upload.(*Worker) - s.logger.Debug("AsTerminatableUpload") - return worker -} - -// AsLengthDeclarableUpload returns tusd handler.LengthDeclarableUpload -func (s *Scheduler) AsLengthDeclarableUpload(upload handler.Upload) handler.LengthDeclarableUpload { - s.logger.Debug("AsLengthDeclarableUpload") - return upload.(*Worker) -} diff --git a/internal/pkg/uploader/datalake.go b/internal/pkg/uploader/datalake.go deleted file mode 100644 index 097bf0a6b5..0000000000 --- a/internal/pkg/uploader/datalake.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright © 2025 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. - -package uploader - -import ( - "bytes" - "context" - "io" - "mime/multipart" - "os" - "path/filepath" - - "github.com/huly-stream/internal/pkg/log" - "github.com/pkg/errors" - "github.com/valyala/fasthttp" - "go.uber.org/zap" -) - -// DatalakeStorage represents datalake storage -type DatalakeStorage struct { - baseURL string - workspace string - token string -} - -// NewDatalakeStorage creates a new datalake client -func NewDatalakeStorage(baseURL, workspace, token string) Storage { - return &DatalakeStorage{ - baseURL: baseURL, - token: token, - workspace: workspace, - } -} - -// UploadFile uploads file to the datalake -func (d *DatalakeStorage) UploadFile(ctx context.Context, fileName string) error { - // #nosec - file, err := os.Open(fileName) - if err != nil { - return err - } - defer func() { - _ = file.Close() - }() - - var objectKey = getObjectKey(fileName) - var logger = log.FromContext(ctx).With(zap.String("datalake upload", d.workspace), zap.String("fileName", fileName)) - - logger.Debug("start uploading") - - body := &bytes.Buffer{} - writer := multipart.NewWriter(body) - - part, err := writer.CreateFormFile("file", objectKey) - if err != nil { - return errors.Wrapf(err, "failed to create form file") - } - - _, err = io.Copy(part, file) - if err != nil { - return errors.Wrapf(err, "failed to copy file data") - } - - _ = writer.Close() - - req := fasthttp.AcquireRequest() - defer fasthttp.ReleaseRequest(req) - - res := fasthttp.AcquireResponse() - defer fasthttp.ReleaseResponse(res) - - req.SetRequestURI(d.baseURL + "/upload/form-data/" + d.workspace) - req.Header.SetMethod(fasthttp.MethodPost) - req.Header.Add("Authorization", "Bearer "+d.token) - req.Header.SetContentType(writer.FormDataContentType()) - req.SetBody(body.Bytes()) - - client := fasthttp.Client{} - if err := client.Do(req, res); err != nil { - return errors.Wrapf(err, "upload failed") - } - - logger.Debug("file uploaded") - - return nil -} - -// DeleteFile deletes file from the datalake -func (d *DatalakeStorage) DeleteFile(ctx context.Context, fileName string) error { - var logger = log.FromContext(ctx).With(zap.String("datalake delete", d.workspace), zap.String("fileName", fileName)) - logger.Debug("start deleting") - - var objectKey = getObjectKey(fileName) - - req := fasthttp.AcquireRequest() - defer fasthttp.ReleaseRequest(req) - - res := fasthttp.AcquireResponse() - defer fasthttp.ReleaseResponse(res) - - req.SetRequestURI(d.baseURL + "/blob/" + d.workspace + "/" + objectKey) - req.Header.SetMethod(fasthttp.MethodDelete) - req.Header.Add("Authorization", "Bearer "+d.token) - - client := fasthttp.Client{} - if err := client.Do(req, res); err != nil { - logger.Error("failed to del", zap.Error(err)) - return errors.Wrapf(err, "delete failed") - } - - logger.Debug("file deleted") - - return nil -} - -func getObjectKey(s string) string { - var _, objectKey = filepath.Split(s) - return objectKey -} diff --git a/internal/pkg/uploader/options.go b/internal/pkg/uploader/options.go index 076e2c70af..97ba5cc508 100644 --- a/internal/pkg/uploader/options.go +++ b/internal/pkg/uploader/options.go @@ -13,7 +13,15 @@ package uploader -type options struct{} +import "time" -// Option provides option for storages -type Option func(*options) +// Options reprents options for uploading files +type Options struct { + Dir string + WorkerCount uint32 + BufferSize int + RetryCount int + RetryDelay time.Duration + Timeout time.Duration + SourceFile string +} diff --git a/internal/pkg/uploader/postpone.go b/internal/pkg/uploader/postpone.go deleted file mode 100644 index 29b308dc5e..0000000000 --- a/internal/pkg/uploader/postpone.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright © 2025 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. - -package uploader - -import ( - "context" - "time" - - "github.com/huly-stream/internal/pkg/log" - "go.uber.org/zap" -) - -func (u *uploader) postpone(fileName string, action func(ctx context.Context)) { - u.waitJobs.Add(1) - var ctx, cancel = context.WithCancel(context.Background()) - ctx = log.WithLoggerFields(ctx, zap.String("pospone", "action")) - var startCh = time.After(u.postponeDuration) - - if v, ok := u.contexts.Load(fileName); ok { - (*v.(*context.CancelFunc))() - } - u.contexts.Store(fileName, &cancel) - - go func() { - u.waitJobs.Done() - defer cancel() - select { - case <-ctx.Done(): - return - case <-startCh: - action(ctx) - if ctx.Err() == nil { - u.contexts.CompareAndDelete(fileName, &cancel) - } - } - }() -} diff --git a/internal/pkg/uploader/uploader.go b/internal/pkg/uploader/uploader.go index 8d04618f57..d251d4b4da 100644 --- a/internal/pkg/uploader/uploader.go +++ b/internal/pkg/uploader/uploader.go @@ -11,204 +11,289 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package uploader provides objects and functions to work with uploading and monitoring files +// Package uploader provides a mechanism for uploading files to a remote storage. package uploader import ( "context" - "net/url" + "hash/fnv" "os" "path/filepath" "strings" "sync" "time" + "unsafe" - "github.com/pkg/errors" - "github.com/tus/tusd/v2/pkg/handler" - - "github.com/fsnotify/fsnotify" - "github.com/huly-stream/internal/pkg/config" - "github.com/huly-stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/storage" "go.uber.org/zap" + "k8s.io/utils/inotify" ) -type uploader struct { - done chan struct{} - waitJobs sync.WaitGroup - ctx context.Context - cancel context.CancelFunc - baseDir string - uploadID string - postponeDuration time.Duration - sentFiles sync.Map - storage Storage - contexts sync.Map - retryCount int - eventBufferCount uint +// See at https://man7.org/linux/man-pages/man7/inotify.7.html +const inotifyCloseWrite uint32 = 0x8 // IN_CLOSE_WRITE +const inotifyMovedTo uint32 = 0x80 // IN_MOVED_TO + +// Uploader represents file uploader +type Uploader interface { + Start() + Stop() + Cancel() } -func (u *uploader) retry(action func() error) { - var delay = time.Millisecond * 50 - for range u.retryCount { - if err := action(); err == nil { - return - } - time.Sleep(delay) - delay *= 2 +type uploaderImpl struct { + logger *zap.Logger + options *Options + storage storage.Storage + + filesCh chan string + + sentFiles sync.Map // key: string, value: struct{} + + shouldDeleteOnStop func(string) bool + + workersCh []chan func() + + watcherStopCh chan struct{} + watcherDoneCh chan struct{} + + uploadCtx context.Context + uploadCancel context.CancelFunc + + workerWaitGroup sync.WaitGroup +} + +// New creates a new instance of uploader +func New(ctx context.Context, s storage.Storage, opts Options) Uploader { + if s == nil { + panic("storage should not be nil") } -} -// Rollback deletes all delivered files and also deletes all local content by uploadID -func (u *uploader) Rollback() { - logger := log.FromContext(u.ctx).With(zap.String("uploader", "Rollback")) - logger.Debug("starting") - defer logger.Debug("done") + var res = &uploaderImpl{ + options: &opts, + storage: s, + logger: log.FromContext(ctx).With(zap.String("uploader", opts.Dir)), + shouldDeleteOnStop: func(s string) bool { + return strings.HasSuffix(s, "m3u8") + }, + filesCh: make(chan string, opts.BufferSize), + watcherStopCh: make(chan struct{}), + watcherDoneCh: make(chan struct{}), + } - u.postpone("", func(ctx context.Context) { - u.sentFiles.Range(func(key, value any) bool { - logger.Debug("deleting remote file", zap.String("key", key.(string))) - u.retry(func() error { return u.storage.DeleteFile(ctx, key.(string)) }) - return true - }) - }) + res.workersCh = make([]chan func(), opts.WorkerCount) - u.Terminate() -} + for i := range opts.WorkerCount { + res.workersCh[i] = make(chan func(), opts.BufferSize) + } -// Terminate deletes -func (u *uploader) Terminate() { - logger := log.FromContext(u.ctx).With(zap.String("uploader", "Terminate")) - logger.Debug("starting") + res.logger.Sugar().Debugf("uploader config is %v", opts) + + res.uploadCtx, res.uploadCancel = context.WithCancel(context.Background()) + + _ = os.MkdirAll(opts.Dir, os.ModePerm) + res.workerWaitGroup.Add(1) go func() { - defer logger.Debug("done") - u.waitJobs.Wait() - u.cancel() + defer res.workerWaitGroup.Done() + initFiles, _ := os.ReadDir(opts.Dir) + for _, f := range initFiles { + var filePath = filepath.Join(opts.Dir, f.Name()) + if filePath == opts.SourceFile { + continue + } + res.filesCh <- filePath + } }() + + return res } -func (u *uploader) uploadAndDelte(fileName string) { - u.postpone(fileName+"-del", func(context.Context) {}) - u.postpone(fileName, func(ctx context.Context) { - u.retry(func() error { return u.storage.UploadFile(ctx, fileName) }) - u.postpone(fileName+"-del", func(context.Context) { - _ = os.Remove(fileName) +func (u *uploaderImpl) Stop() { + u.stop(false) +} + +func (u *uploaderImpl) Cancel() { + u.stop(true) +} + +func (u *uploaderImpl) stop(rollback bool) { + close(u.watcherStopCh) + <-u.watcherDoneCh + u.logger.Debug("file watch stopped") + + if rollback { + u.logger.Debug("starting rollback...") + var i uint32 + u.sentFiles.Range(func(key, _ any) bool { + i++ + var filename = key.(string) + u.workersCh[i%u.options.WorkerCount] <- func() { + u.deleteRemoteFile(filename) + } + return true }) - }) + u.logger.Debug("rollback done") + } + close(u.filesCh) + u.workerWaitGroup.Wait() + u.logger.Debug("workers done") - u.sentFiles.Store(fileName, struct{}{}) + u.uploadCancel() + _ = os.RemoveAll(u.options.Dir) + u.sentFiles.Clear() + + u.logger.Debug("finish done", zap.Bool("cancel", rollback)) } -func (u *uploader) Serve() error { - var logger = log.FromContext(u.ctx).With(zap.String("uploader", u.uploadID), zap.String("dir", u.baseDir)) - var watcher, err = fsnotify.NewBufferedWatcher(u.eventBufferCount) - defer close(u.done) +func (u *uploaderImpl) Start() { + u.startWorkers() + u.startWatch() +} + +func (u *uploaderImpl) startWorkers() { + go func() { + var logger = u.logger.With(zap.String("func", "startWorkers")) + logger.Debug("fanout goroutine started") + defer logger.Debug("fanout goroutine stopped") + + h := fnv.New32a() + for f := range u.filesCh { + // #nosec + bytes := unsafe.Slice(unsafe.StringData(f), len(f)) + _, _ = h.Write(bytes) + id := h.Sum32() % u.options.WorkerCount + u.workersCh[id] <- func() { + u.uploadAndDelete(f) + } + h.Reset() + } + for i := range u.options.WorkerCount { + close(u.workersCh[i]) + } + }() + + for i := range u.options.WorkerCount { + var logger = u.logger.With(zap.Uint32("worker", i)) + + u.workerWaitGroup.Add(1) + go func(index uint32) { + logger.Debug("start") + defer logger.Debug("finished") + + defer u.workerWaitGroup.Done() + for work := range u.workersCh[index] { + work() + } + }(i) + } +} + +func (u *uploaderImpl) deleteRemoteFile(f string) { + var logger = u.logger.With(zap.String("delete remote", f)) + logger.Debug("deleting remote file") + + for range u.options.RetryCount { + var ctx, cancel = context.WithTimeout(u.uploadCtx, u.options.Timeout) + var err = u.storage.DeleteFile(ctx, f) + cancel() + + if err != nil { + logger.Error("attempt failed", zap.Error(err)) + } else { + logger.Debug("file deleted in remote storage") + break + } + + time.Sleep(u.options.RetryDelay) + } + + u.logger.Error("can not delete remote file") +} + +func (u *uploaderImpl) uploadAndDelete(f string) { + var logger = u.logger.With(zap.String("upload and delete", f)) + logger.Debug("uploading file") + + var _, ok = u.sentFiles.Load(f) + + if ok && !u.shouldDeleteOnStop(f) { + logger.Debug("file already uploaded") + return + } + + for range u.options.RetryCount { + var ctx, cancel = context.WithTimeout(u.uploadCtx, u.options.Timeout) + var err = u.storage.PutFile(ctx, f) + cancel() + + if err != nil { + logger.Error("attempt failed", zap.Error(err)) + } else { + if !u.shouldDeleteOnStop(f) { + _ = os.Remove(f) + logger.Debug("removed file locally") + } + u.sentFiles.Store(f, struct{}{}) + logger.Debug("file uploaded") + break + } + + time.Sleep(u.options.RetryDelay) + } +} + +func (u *uploaderImpl) startWatch() { + var logger = u.logger.With(zap.String("func", "startWatch")) + var watcher, err = inotify.NewWatcher() if err != nil { - logger.Error("can not start watcher") - return err + logger.Error("can not start file watcher", zap.Error(err)) + return } - _ = os.MkdirAll(u.baseDir, os.ModePerm) - initFiles, _ := os.ReadDir(u.baseDir) - for _, f := range initFiles { - var filePath = filepath.Join(u.baseDir, f.Name()) - u.uploadAndDelte(filePath) + if err := watcher.AddWatch(u.options.Dir, inotifyCloseWrite); err != nil { + logger.Error("can not start watching for close write", zap.Error(err)) + return } - - if err := watcher.Add(u.baseDir); err != nil { - return err + if err := watcher.AddWatch(u.options.Dir, inotifyMovedTo); err != nil { + logger.Error("can not start watching for moved to", zap.Error(err)) + return } - defer func() { _ = watcher.Close() + close(u.watcherDoneCh) }() - logger.Debug("uploader has initialized and started watching") + logger.Debug("watching for file updates") defer logger.Debug("done") for { select { - case <-u.ctx.Done(): - return u.ctx.Err() - case event, ok := <-watcher.Events: + case <-u.watcherStopCh: + return + case event, ok := <-watcher.Event: if !ok { - return u.ctx.Err() + logger.Error("file channel was closed") + return + } + if !strings.Contains(event.Name, u.options.Dir) { + continue + } + if event.Name == u.options.SourceFile { + continue } if strings.HasSuffix(event.Name, "tmp") { continue } - if !strings.Contains(event.Name, u.uploadID) { - continue - } - u.uploadAndDelte(event.Name) - case err, ok := <-watcher.Errors: + logger.Debug("received an event", zap.String("event", event.Name), zap.Uint32("mask", event.Mask)) + + u.filesCh <- event.Name + case err, ok := <-watcher.Error: if !ok { - return u.ctx.Err() + logger.Error("error channel was closed") } - logger.Error("get an error", zap.Error(err)) + logger.Error("received an error", zap.Error(err)) } } } - -// Uploader manages content delivering -type Uploader interface { - Terminate() - Rollback() - Serve() error -} - -// Storage represents file-based storage -type Storage interface { - UploadFile(ctx context.Context, fileName string) error - DeleteFile(ctx context.Context, fileName string) error -} - -// New creates a new instance of Uplaoder -func New(ctx context.Context, baseDir string, endpointURL *url.URL, uploadInfo handler.FileInfo) (Uploader, error) { - var uploaderCtx, uploadCancel = context.WithCancel(context.Background()) - uploaderCtx = log.WithLoggerFields(uploaderCtx) - - var storage Storage - var err error - - storage, err = NewStorageByURL(uploaderCtx, endpointURL, uploadInfo.MetaData) - if err != nil { - uploadCancel() - return nil, err - } - - return &uploader{ - ctx: uploaderCtx, - cancel: uploadCancel, - done: make(chan struct{}), - uploadID: uploadInfo.ID, - postponeDuration: time.Second * 2, - storage: storage, - retryCount: 10, - baseDir: filepath.Join(baseDir, uploadInfo.ID), - eventBufferCount: 100, - }, nil -} - -// NewStorageByURL creates a new storage basd on the type from the url scheme, for example "datalake://my-datalake-endpoint" -func NewStorageByURL(ctx context.Context, u *url.URL, headers map[string]string) (Storage, error) { - var workspace = headers["workspace"] - if workspace == "" { - return nil, errors.New("missed workspace in the client's metadata") - } - c, _ := config.FromEnv() - switch u.Scheme { - case "tus": - return nil, errors.New("not imlemented yet") - case "datalake": - if headers["token"] == "" { - return nil, errors.New("missed auth token in the client's metadata") - } - return NewDatalakeStorage(c.Endpoint().String(), workspace, headers["token"]), nil - case "s3": - return NewS3(ctx, c.Endpoint().String(), workspace), nil - default: - return nil, errors.New("unknown scheme") - } -} From 9c82eed873e14bb1625bb2ecf137fca2cb7f15db Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Sun, 16 Mar 2025 01:09:22 +0300 Subject: [PATCH 10/52] update ci for go 1.24.1 Signed-off-by: denis-tingaikin --- .github/workflows/main.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 635804cf49..35e5f78798 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -36,7 +36,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: 1.23.5 + go-version: 1.24.1 - name: Build run: go build -race ./... - name: Test @@ -47,7 +47,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: 1.23.5 + go-version: 1.24.1 - run: go mod tidy - name: Check for changes in go.mod or go.sum run: | @@ -63,9 +63,9 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: 1.23.5 + go-version: 1.24.1 - name: golangci-lint uses: golangci/golangci-lint-action@v4 with: - version: v1.60.3 + version: v1.64.7 args: --timeout 3m --verbose From c6a7bae060ca93dde948bb65a6f57d389d905366 Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Sun, 16 Mar 2025 01:26:40 +0300 Subject: [PATCH 11/52] apply self code review Signed-off-by: denis-tingaikin --- Dockerfile | 4 ++-- internal/pkg/transcoder/{queue.go => scheduler.go} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename internal/pkg/transcoder/{queue.go => scheduler.go} (100%) diff --git a/Dockerfile b/Dockerfile index 4304349055..9fd38ffc7e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM --platform=linux/amd64 golang:1.24.1 AS builder +FROM --platform=$BUILDPLATFORM golang:1.24.1 AS builder ENV GO111MODULE=on ENV CGO_ENABLED=0 ENV GOBIN=/bin @@ -21,7 +21,7 @@ COPY . ./ RUN set -xe && GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /go/bin/stream ./cmd/stream -FROM --platform=linux/amd64 alpine +FROM --platform=$BUILDPLATFORM alpine RUN set -xe && apk add --no-cache ffmpeg RUN apk add --no-cache ca-certificates jq bash \ diff --git a/internal/pkg/transcoder/queue.go b/internal/pkg/transcoder/scheduler.go similarity index 100% rename from internal/pkg/transcoder/queue.go rename to internal/pkg/transcoder/scheduler.go From 528306f4b0fe711b745e542694617782c4587f55 Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Tue, 18 Mar 2025 18:43:45 +0300 Subject: [PATCH 12/52] UBERF-9641: Add support for thumbnail Signed-off-by: denis-tingaikin --- Dockerfile | 4 +- internal/pkg/api/v1/recording/handler.go | 12 +-- internal/pkg/api/v1/transcoding/handler.go | 26 ++++-- .../{transcoder => mediaconvert}/command.go | 13 ++- .../command_test.go | 12 +-- .../coordinator.go | 10 ++- .../{transcoder => mediaconvert}/scheduler.go | 87 ++++++++++--------- .../{transcoder => mediaconvert}/stream.go | 4 +- 8 files changed, 99 insertions(+), 69 deletions(-) rename internal/pkg/{transcoder => mediaconvert}/command.go (89%) rename internal/pkg/{transcoder => mediaconvert}/command_test.go (65%) rename internal/pkg/{transcoder => mediaconvert}/coordinator.go (95%) rename internal/pkg/{transcoder => mediaconvert}/scheduler.go (76%) rename internal/pkg/{transcoder => mediaconvert}/stream.go (98%) diff --git a/Dockerfile b/Dockerfile index 9fd38ffc7e..121a1b5efb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM --platform=$BUILDPLATFORM golang:1.24.1 AS builder +FROM golang:1.24.1 AS builder ENV GO111MODULE=on ENV CGO_ENABLED=0 ENV GOBIN=/bin @@ -21,7 +21,7 @@ COPY . ./ RUN set -xe && GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /go/bin/stream ./cmd/stream -FROM --platform=$BUILDPLATFORM alpine +FROM alpine RUN set -xe && apk add --no-cache ffmpeg RUN apk add --no-cache ca-certificates jq bash \ diff --git a/internal/pkg/api/v1/recording/handler.go b/internal/pkg/api/v1/recording/handler.go index 4282261f76..e65dc53578 100644 --- a/internal/pkg/api/v1/recording/handler.go +++ b/internal/pkg/api/v1/recording/handler.go @@ -24,7 +24,7 @@ import ( "github.com/hcengineering/stream/internal/pkg/config" "github.com/hcengineering/stream/internal/pkg/log" - "github.com/hcengineering/stream/internal/pkg/transcoder" + "github.com/hcengineering/stream/internal/pkg/mediaconvert" "go.uber.org/zap" tusd "github.com/tus/tusd/v2/pkg/handler" @@ -56,13 +56,13 @@ func NewHandler(ctx context.Context, cfg *config.Config) http.Handler { } func (h *recordingHandler) initialize() { - scheduler := transcoder.NewStreamCoordinator(h.ctx, h.cfg) + coordinator := mediaconvert.NewStreamCoordinator(h.ctx, h.cfg) tusComposer := tusd.NewStoreComposer() - tusComposer.UseCore(scheduler) - tusComposer.UseTerminater(scheduler) - tusComposer.UseConcater(scheduler) - tusComposer.UseLengthDeferrer(scheduler) + tusComposer.UseCore(coordinator) + tusComposer.UseTerminater(coordinator) + tusComposer.UseConcater(coordinator) + tusComposer.UseLengthDeferrer(coordinator) var tusHandler, err = tusd.NewHandler(tusd.Config{ BasePath: "/recording", diff --git a/internal/pkg/api/v1/transcoding/handler.go b/internal/pkg/api/v1/transcoding/handler.go index 0391ea74da..67c4ca1fd0 100644 --- a/internal/pkg/api/v1/transcoding/handler.go +++ b/internal/pkg/api/v1/transcoding/handler.go @@ -19,15 +19,16 @@ import ( "encoding/json" "fmt" "net/http" + "strings" "github.com/hcengineering/stream/internal/pkg/config" "github.com/hcengineering/stream/internal/pkg/log" - "github.com/hcengineering/stream/internal/pkg/transcoder" + "github.com/hcengineering/stream/internal/pkg/mediaconvert" "go.uber.org/zap" ) type trascodeHandler struct { - taskQueue *transcoder.Scheduler + scheduler *mediaconvert.Scheduler logger *zap.Logger } @@ -45,7 +46,7 @@ func (t *trascodeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } var decoder = json.NewDecoder(r.Body) - var task transcoder.Task + var task mediaconvert.Task if err := decoder.Decode(&task); err != nil { w.WriteHeader(http.StatusBadRequest) @@ -53,14 +54,29 @@ func (t *trascodeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - t.taskQueue.Schedule(&task) + if !isSupportedFormat(task.Format) { + w.WriteHeader(http.StatusUnsupportedMediaType) + _, _ = fmt.Fprint(w, "output media format is not supported") + return + } + + t.scheduler.Schedule(&task) w.WriteHeader(http.StatusOK) } // NewHandler creates a new trnascoding http handler, requires context and config. func NewHandler(ctx context.Context, cfg *config.Config) http.Handler { return &trascodeHandler{ - taskQueue: transcoder.NewScheduler(ctx, cfg), + scheduler: mediaconvert.NewScheduler(ctx, cfg), logger: log.FromContext(ctx).With(zap.String("handler", "transcoding")), } } + +func isSupportedFormat(s string) bool { + switch strings.ToLower(s) { + case "hls": + return true + default: + return false + } +} diff --git a/internal/pkg/transcoder/command.go b/internal/pkg/mediaconvert/command.go similarity index 89% rename from internal/pkg/transcoder/command.go rename to internal/pkg/mediaconvert/command.go index 9cc65d9637..2b68893404 100644 --- a/internal/pkg/transcoder/command.go +++ b/internal/pkg/mediaconvert/command.go @@ -13,7 +13,7 @@ // limitations under the License. // -package transcoder +package mediaconvert import ( "context" @@ -79,11 +79,21 @@ func BuildRawVideoCommand(opts *Options) []string { "-c:v", "copy", "-hls_time", "5", + "-hls_flags", "split_by_time", "-hls_list_size", "0", "-hls_segment_filename", filepath.Join(opts.OuputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", opts.Level)), filepath.Join(opts.OuputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, opts.Level))) } +// BuildThumbnailCommand creates a command that creates a thumbnail for the input video +func BuildThumbnailCommand(opts *Options) []string { + return append([]string{}, + "-i", opts.Input, + "-vframes", "1", + filepath.Join(opts.OuputDir, opts.UploadID, opts.UploadID+".jpg"), + ) +} + // BuildScalingVideoCommand returns flags for ffmpeg for video scaling func BuildScalingVideoCommand(opts *Options) []string { var result = buildCommonComamnd(opts) @@ -97,6 +107,7 @@ func BuildScalingVideoCommand(opts *Options) []string { "-crf", "23", "-g", "60", "-hls_time", "5", + "-hls_flags", "split_by_time", "-hls_list_size", "0", "-hls_segment_filename", filepath.Join(opts.OuputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", level)), filepath.Join(opts.OuputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, level))) diff --git a/internal/pkg/transcoder/command_test.go b/internal/pkg/mediaconvert/command_test.go similarity index 65% rename from internal/pkg/transcoder/command_test.go rename to internal/pkg/mediaconvert/command_test.go index db382a555e..42ddc4ea4d 100644 --- a/internal/pkg/transcoder/command_test.go +++ b/internal/pkg/mediaconvert/command_test.go @@ -11,19 +11,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package transcoder_test +package mediaconvert_test import ( "strings" "testing" + "github.com/hcengineering/stream/internal/pkg/mediaconvert" "github.com/hcengineering/stream/internal/pkg/resconv" - "github.com/hcengineering/stream/internal/pkg/transcoder" "github.com/stretchr/testify/require" ) func Test_BuildVideoCommand_Scaling(t *testing.T) { - var scaleCommand = transcoder.BuildScalingVideoCommand(&transcoder.Options{ + var scaleCommand = mediaconvert.BuildScalingVideoCommand(&mediaconvert.Options{ OuputDir: "test", Input: "pipe:0", UploadID: "1", @@ -31,13 +31,13 @@ func Test_BuildVideoCommand_Scaling(t *testing.T) { ScalingLevels: []string{"720p", "480p"}, }) - const expected = `-threads 4 -i pipe:0 -vf scale=-2:720 -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -vf scale=-2:480 -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `-threads 4 -i pipe:0 -vf scale=-2:720 -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -vf scale=-2:480 -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(scaleCommand, " ")) } func Test_BuildVideoCommand_Raw(t *testing.T) { - var rawCommand = transcoder.BuildRawVideoCommand(&transcoder.Options{ + var rawCommand = mediaconvert.BuildRawVideoCommand(&mediaconvert.Options{ OuputDir: "test", Input: "pipe:0", UploadID: "1", @@ -45,7 +45,7 @@ func Test_BuildVideoCommand_Raw(t *testing.T) { Level: resconv.Level("651:490"), }) - const expected = `"-threads 4 -i pipe:0 -c:v copy -hls_time 5 -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `"-threads 4 -i pipe:0 -c:v copy -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(rawCommand, " ")) } diff --git a/internal/pkg/transcoder/coordinator.go b/internal/pkg/mediaconvert/coordinator.go similarity index 95% rename from internal/pkg/transcoder/coordinator.go rename to internal/pkg/mediaconvert/coordinator.go index d0705cb0ce..843716ce79 100644 --- a/internal/pkg/transcoder/coordinator.go +++ b/internal/pkg/mediaconvert/coordinator.go @@ -13,7 +13,7 @@ // limitations under the License. // -package transcoder +package mediaconvert import ( "context" @@ -77,7 +77,7 @@ func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo var stream = &Stream{ writer: sharedpipe.NewWriter(), info: info, - logger: log.FromContext(s.mainContext).With(zap.String("worker", info.ID)), + logger: log.FromContext(s.mainContext).With(zap.String("stream", info.ID)), done: make(chan struct{}), } @@ -142,12 +142,12 @@ func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo // GetUpload returns current a worker based on upload id func (s *StreamCoordinator) GetUpload(ctx context.Context, id string) (upload handler.Upload, err error) { if v, ok := s.streams.Load(id); ok { - s.logger.Debug("GetUpload: found worker by id", zap.String("id", id)) + s.logger.Debug("GetUpload: found stream by id", zap.String("id", id)) var w = v.(*Stream) s.manageTimeout(w) return w, nil } - s.logger.Debug("GetUpload: worker not found", zap.String("id", id)) + s.logger.Debug("GetUpload: stream not found", zap.String("id", id)) return nil, errors.New("bad id") } @@ -175,6 +175,7 @@ func (s *StreamCoordinator) manageTimeout(w *Stream) { case <-w.done: w.logger.Debug("stream has finished") s.cancels.Delete(w.info.ID) + s.streams.Delete(w.info.ID) return case <-cancelCtx.Done(): w.logger.Debug("stream timeout has refreshed") @@ -185,6 +186,7 @@ func (s *StreamCoordinator) manageTimeout(w *Stream) { var terminateCtx, terminateCancel = context.WithTimeout(context.Background(), s.conf.Timeout) defer terminateCancel() _ = w.Terminate(terminateCtx) + s.streams.Delete(w.info.ID) } }() } diff --git a/internal/pkg/transcoder/scheduler.go b/internal/pkg/mediaconvert/scheduler.go similarity index 76% rename from internal/pkg/transcoder/scheduler.go rename to internal/pkg/mediaconvert/scheduler.go index 4a82ae107b..83894425db 100644 --- a/internal/pkg/transcoder/scheduler.go +++ b/internal/pkg/mediaconvert/scheduler.go @@ -13,7 +13,7 @@ // limitations under the License. // -package transcoder +package mediaconvert import ( "context" @@ -37,7 +37,10 @@ import ( // HLS represents metadata for transcoding result type HLS struct { - Source string `json:"source"` + Source string `json:"source"` + Thumbnail string `json:"thumbnail"` + Width int `json:"width"` + Height int `json:"height"` } // Task represents transcoding task @@ -77,7 +80,7 @@ func NewScheduler(ctx context.Context, cfg *config.Config) *Scheduler { taskCh: make(chan *Task, 128), cfg: cfg, ctx: ctx, - logger: log.FromContext(ctx).With(zap.String("transcoding", "planner")), + logger: log.FromContext(ctx).With(zap.String("transcoding", "scheduler")), } go p.start() @@ -100,6 +103,7 @@ func (p *Scheduler) start() { } } +// TODO: add a factory pattern to process tasks by different media type func (p *Scheduler) processTask(ctx context.Context, task *Task) { var logger = p.logger.With(zap.String("task-id", task.ID)) @@ -166,60 +170,57 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { SourceFile: sourceFilePath, }) - go uploader.Start() - - logger.Debug("phase 6: start async transcode process") - var rawCommand, scaleCommand *exec.Cmd - - rawCommand, err = newFfmpegCommand(ctx, nil, BuildRawVideoCommand(&opts)) - if err != nil { - logger.Error("can not create ffmpeg command", zap.Error(err)) - go uploader.Cancel() - return - } - - scaleCommand, err = newFfmpegCommand(ctx, nil, BuildScalingVideoCommand(&opts)) - if err != nil { - logger.Error("can not create ffmpeg command", zap.Error(err)) - go uploader.Cancel() - return - } - _ = manifest.GenerateHLSPlaylist(opts.ScalingLevels, p.cfg.OutputDir, opts.UploadID) - if err = rawCommand.Start(); err != nil { - logger.Error("can not run raw ffmpeg command", zap.Error(err)) - go uploader.Cancel() - return - } + go uploader.Start() - if err = scaleCommand.Start(); err != nil { - logger.Error("can not run scale ffmpeg command", zap.Error(err)) - go uploader.Cancel() - return + logger.Debug("phase 6: start async transcode processes") + + var argsSlice = [][]string{ + BuildThumbnailCommand(&opts), + BuildRawVideoCommand(&opts), + BuildScalingVideoCommand(&opts), + } + var cmds []*exec.Cmd + + for _, args := range argsSlice { + cmd, cmdErr := newFfmpegCommand(ctx, nil, args) + if cmdErr != nil { + logger.Error("can not create a new command", zap.Error(cmdErr), zap.Strings("args", args)) + go uploader.Cancel() + return + } + cmds = append(cmds, cmd) + if err = cmd.Start(); err != nil { + logger.Error("can not start a command", zap.Error(err), zap.Strings("args", args)) + go uploader.Cancel() + return + } } logger.Debug("phase 7: wait for the result") - if err = scaleCommand.Wait(); err != nil { - logger.Error("can not scale ", zap.Error(err)) - go uploader.Cancel() - return - } - if err = rawCommand.Wait(); err != nil { - logger.Error("can not process raw", zap.Error(err)) - go uploader.Cancel() - return + for _, cmd := range cmds { + if err = cmd.Wait(); err != nil { + logger.Error("can not wait for command end ", zap.Error(err)) + go uploader.Cancel() + return + } } logger.Debug("phase 8: schedule cleanup") go uploader.Stop() logger.Debug("phase 9: try to set metadata") - var resultURL = p.cfg.Endpoint().JoinPath("blob", task.Workspace, task.ID+"_master.m3u8") if metaProvider, ok := remoteStorage.(storage.MetaProvider); ok { - var hls = &HLS{Source: resultURL.String()} - logger.Debug("applying metadata", zap.Stringer("url", resultURL), zap.String("source", task.Source)) + var hls = HLS{ + Width: probe.FirstVideoStream().Width, + Height: probe.FirstVideoStream().Height, + Source: p.cfg.Endpoint().JoinPath("blob", task.Workspace, task.ID+"_master.m3u8").String(), + Thumbnail: p.cfg.Endpoint().JoinPath("blob", task.Workspace, task.ID+".jpg").String(), + } + + logger.Debug("applying metadata", zap.String("url", hls.Source), zap.String("thumbnail", hls.Thumbnail), zap.String("source", task.Source)) err = metaProvider.PatchMeta( ctx, task.Source, diff --git a/internal/pkg/transcoder/stream.go b/internal/pkg/mediaconvert/stream.go similarity index 98% rename from internal/pkg/transcoder/stream.go rename to internal/pkg/mediaconvert/stream.go index d59c12c09f..ab47c8e771 100644 --- a/internal/pkg/transcoder/stream.go +++ b/internal/pkg/mediaconvert/stream.go @@ -11,8 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package transcoder provides types and functions for video trnascoding -package transcoder +// Package mediaconvert provides types and functions for video trnascoding +package mediaconvert import ( "context" From 04cf12cc449aaba4e7da1ee5e3fc38f2a615c794 Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Tue, 18 Mar 2025 18:52:29 +0300 Subject: [PATCH 13/52] store blobid in meta Signed-off-by: denis-tingaikin --- internal/pkg/mediaconvert/scheduler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/pkg/mediaconvert/scheduler.go b/internal/pkg/mediaconvert/scheduler.go index 83894425db..aac40c1466 100644 --- a/internal/pkg/mediaconvert/scheduler.go +++ b/internal/pkg/mediaconvert/scheduler.go @@ -216,8 +216,8 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { var hls = HLS{ Width: probe.FirstVideoStream().Width, Height: probe.FirstVideoStream().Height, - Source: p.cfg.Endpoint().JoinPath("blob", task.Workspace, task.ID+"_master.m3u8").String(), - Thumbnail: p.cfg.Endpoint().JoinPath("blob", task.Workspace, task.ID+".jpg").String(), + Source: task.ID + "_master.m3u8", + Thumbnail: task.ID + ".jpg", } logger.Debug("applying metadata", zap.String("url", hls.Source), zap.String("thumbnail", hls.Thumbnail), zap.String("source", task.Source)) From 42cd51f9664be4a3e86cd1d436bb4b2533e3cc71 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 7 May 2025 14:52:37 +0700 Subject: [PATCH 14/52] fix: writer proper content type to datalake Signed-off-by: Alexander Onnikov --- internal/pkg/storage/datalake.go | 32 +++++++++++++++++++++++++------- internal/pkg/storage/s3.go | 11 ----------- internal/pkg/storage/storage.go | 11 +++++++++++ 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index abe2f69459..e366c98d70 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -20,8 +20,10 @@ import ( "fmt" "io" "mime/multipart" + "net/textproto" "os" "path/filepath" + "strings" "github.com/hcengineering/stream/internal/pkg/log" "github.com/pkg/errors" @@ -48,6 +50,24 @@ func NewDatalakeStorage(ctx context.Context, baseURL, workspace, token string) S } } +var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") + +func escapeQuotes(s string) string { + return quoteEscaper.Replace(s) +} + +func createFormFile(writer *multipart.Writer, fieldname, filename, contentType string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, escapeQuotes(fieldname), escapeQuotes(filename))) + h.Set("Content-Type", contentType) + return writer.CreatePart(h) +} + +func getObjectKey(s string) string { + var _, objectKey = filepath.Split(s) + return objectKey +} + // PutFile uploads file to the datalake func (d *DatalakeStorage) PutFile(ctx context.Context, fileName string) error { // #nosec @@ -67,7 +87,7 @@ func (d *DatalakeStorage) PutFile(ctx context.Context, fileName string) error { body := &bytes.Buffer{} writer := multipart.NewWriter(body) - part, err := writer.CreateFormFile("file", objectKey) + part, err := createFormFile(writer, "file", objectKey, getContentType(objectKey)) if err != nil { return errors.Wrapf(err, "failed to create form file") } @@ -77,7 +97,10 @@ func (d *DatalakeStorage) PutFile(ctx context.Context, fileName string) error { return errors.Wrapf(err, "failed to copy file data") } - _ = writer.Close() + err = writer.Close() + if err != nil { + return errors.Wrapf(err, "failed to close multipart writer") + } req := fasthttp.AcquireRequest() defer fasthttp.ReleaseRequest(req) @@ -128,11 +151,6 @@ func (d *DatalakeStorage) DeleteFile(ctx context.Context, fileName string) error return nil } -func getObjectKey(s string) string { - var _, objectKey = filepath.Split(s) - return objectKey -} - // PatchMeta patches metadata for the object func (d *DatalakeStorage) PatchMeta(ctx context.Context, filename string, md *Metadata) error { var logger = d.logger.With(zap.String("patch meta", d.workspace), zap.String("fileName", filename)) diff --git a/internal/pkg/storage/s3.go b/internal/pkg/storage/s3.go index 41da2eaaf1..3864b4fa18 100644 --- a/internal/pkg/storage/s3.go +++ b/internal/pkg/storage/s3.go @@ -20,7 +20,6 @@ import ( "os" "path/filepath" - "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -63,16 +62,6 @@ func NewS3(ctx context.Context, endpoint, bucketName string) Storage { } } -func getContentType(objectKey string) string { - if strings.HasSuffix(objectKey, ".ts") { - return "video/mp2t" - } - if strings.HasSuffix(objectKey, ".m3u8") { - return "video/x-mpegurl" - } - return "application/octet-stream" -} - // DeleteFile deletes file from the s3 storage func (u *S3Storage) DeleteFile(ctx context.Context, fileName string) error { var _, objectKey = filepath.Split(fileName) diff --git a/internal/pkg/storage/storage.go b/internal/pkg/storage/storage.go index e45250284d..1914b26ac7 100644 --- a/internal/pkg/storage/storage.go +++ b/internal/pkg/storage/storage.go @@ -16,6 +16,7 @@ package storage import ( "context" "net/url" + "strings" "github.com/pkg/errors" ) @@ -53,3 +54,13 @@ func NewStorageByURL(ctx context.Context, u *url.URL, storageType, token, worksa return nil, errors.New("unknown scheme") } } + +func getContentType(objectKey string) string { + if strings.HasSuffix(objectKey, ".ts") { + return "video/mp2t" + } + if strings.HasSuffix(objectKey, ".m3u8") { + return "video/x-mpegurl" + } + return "application/octet-stream" +} From 78e9ec18eda063b0ecba21c35858881485c5d2af Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 7 May 2025 15:19:08 +0700 Subject: [PATCH 15/52] Add sentry do stream service Signed-off-by: Alexander Onnikov --- README.md | 4 +++- cmd/stream/main.go | 35 +++++++++++++++++++++++++++++++---- go.mod | 3 +++ go.sum | 19 ++++++++++++++++++- internal/pkg/config/config.go | 1 + 5 files changed, 56 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 529b764313..efedffe08c 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,8 @@ docker build . -t hcengineering/sream:latest The following environment variables can be used: ``` -KEY TYPE DEFAULT STREAM_LOG_LEVEL String debug sets log level for the application +KEY TYPE DEFAULT DESCRIPTION +STREAM_LOG_LEVEL String debug sets log level for the application STREAM_SERVER_SECRET String server secret required to generate and verify tokens STREAM_PPROF_ENABLED True or False true starts profile server on localhost:6060 if true STREAM_INSECURE True or False false ignores authorization check if true @@ -81,6 +82,7 @@ STREAM_ENDPOINT_URL URL s3://127.0.0.1:9000 STREAM_MAX_PARALLEL_SCALING_COUNT Integer 2 how much parallel scaling can be processed STREAM_MAX_THREAD_COUNT Integer 4 max number of threads for transcoder STREAM_OUTPUT_DIR String /tmp/transcoing/ path to the directory with tra +STREAM_SENTRY_DSN String "" sentry dsn for error tracking ``` ### Metadata diff --git a/cmd/stream/main.go b/cmd/stream/main.go index 9b34801c02..ea63605dac 100644 --- a/cmd/stream/main.go +++ b/cmd/stream/main.go @@ -21,7 +21,10 @@ import ( "os" "os/signal" "syscall" + "time" + "github.com/getsentry/sentry-go" + sentryhttp "github.com/getsentry/sentry-go/http" "go.uber.org/zap" "github.com/hcengineering/stream/internal/pkg/api/v1/recording" @@ -48,19 +51,43 @@ func main() { } logger.Sugar().Debugf("parsed config is %v", cfg) + if cfg.SentryDsn != "" { + if err := sentry.Init(sentry.ClientOptions{ + Dsn: cfg.SentryDsn, + Tags: map[string]string{"application": "stream"}, + }); err != nil { + logger.Sugar().Fatalf("sentry.Init: %s", err) + } + // ensure buffered events are sent before exit + defer sentry.Flush(2 * time.Second) + } + var recordingHandler = recording.NewHandler(ctx, cfg) var transcodingHandler = transcoding.NewHandler(ctx, cfg) - http.Handle("/recording/", http.StripPrefix("/recording/", recordingHandler)) - http.Handle("/recording", http.StripPrefix("/recording", recordingHandler)) - http.Handle("/transcoding", http.StripPrefix("/transcoding", transcodingHandler)) + // setup HTTP routes + mux := http.NewServeMux() + mux.Handle("/recording/", http.StripPrefix("/recording/", recordingHandler)) + mux.Handle("/recording", http.StripPrefix("/recording", recordingHandler)) + mux.Handle("/transcoding", http.StripPrefix("/transcoding", transcodingHandler)) + + // wrap with Sentry HTTP handler if enabled + var handler http.Handler = mux + if cfg.SentryDsn != "" { + sentryHandler := sentryhttp.New(sentryhttp.Options{ + Repanic: true, + WaitForDelivery: true, + Timeout: 2 * time.Second, + }) + handler = sentryHandler.Handle(mux) + } go func() { logger.Info("server started serving", zap.String("ServeURL", cfg.ServeURL)) defer logger.Info("server finished") // #nosec - var err = http.ListenAndServe(cfg.ServeURL, nil) + var err = http.ListenAndServe(cfg.ServeURL, handler) if err != nil { cancel() logger.Debug("unable to listen", zap.Error(err)) diff --git a/go.mod b/go.mod index 0671d43070..30d0f9ea7c 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.29.6 github.com/aws/aws-sdk-go-v2/credentials v1.17.59 github.com/aws/aws-sdk-go-v2/service/s3 v1.77.0 + github.com/getsentry/sentry-go v0.31.1 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.6.0 github.com/kelseyhightower/envconfig v1.4.0 @@ -42,5 +43,7 @@ require ( github.com/valyala/bytebufferpool v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.37.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 6ef7c10d82..ca1613a2fa 100644 --- a/go.sum +++ b/go.sum @@ -40,20 +40,34 @@ github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/getsentry/sentry-go v0.31.1 h1:ELVc0h7gwyhnXHDouXkhqTFSO5oslsRDk0++eyE0KJ4= +github.com/getsentry/sentry-go v0.31.1/go.mod h1:CYNcMMz73YigoHljQRG+qPF+eMq8gG72XcGN/p71BAY= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tus/tusd/v2 v2.6.0 h1:Je243QDKnFTvm/WkLH2bd1oQ+7trolrflRWyuI0PdWI= @@ -74,10 +88,13 @@ golang.org/x/exp v0.0.0-20250215185904-eff6e970281f h1:oFMYAjX0867ZD2jcNiLBrI9Bd golang.org/x/exp v0.0.0-20250215185904-eff6e970281f/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk= golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/vansante/go-ffprobe.v2 v2.2.1 h1:sFV08OT1eZ1yroLCZVClIVd9YySgCh9eGjBWO0oRayI= gopkg.in/vansante/go-ffprobe.v2 v2.2.1/go.mod h1:qF0AlAjk7Nqzqf3y333Ly+KxN3cKF2JqA3JT5ZheUGE= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 23fee3afc7..0b0badfcf0 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -25,6 +25,7 @@ import ( // Config represents configuration for the huly-stream application. type Config struct { + SentryDsn string `split_words:"true" default:"" desc:"sentry dsn value"` LogLevel string `split_words:"true" default:"debug" desc:"sets log level for the application"` ServerSecret string `split_words:"true" default:"" desc:"server secret required to generate and verify tokens"` PprofEnabled bool `split_words:"true" default:"true" desc:"starts profile server on localhost:6060 if true"` From ba149a18c2b99c1afbb40bde69419a1c685d549d Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 7 May 2025 15:19:28 +0700 Subject: [PATCH 16/52] fix: null pointer dereference in scheduler Signed-off-by: Alexander Onnikov --- internal/pkg/mediaconvert/scheduler.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/pkg/mediaconvert/scheduler.go b/internal/pkg/mediaconvert/scheduler.go index aac40c1466..2471749906 100644 --- a/internal/pkg/mediaconvert/scheduler.go +++ b/internal/pkg/mediaconvert/scheduler.go @@ -148,7 +148,14 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { return } - var res = fmt.Sprintf("%v:%v", probe.FirstVideoStream().Width, probe.FirstVideoStream().Height) + videoStream := probe.FirstVideoStream() + if videoStream == nil { + logger.Error("no video stream found in the file", zap.String("filepath", sourceFilePath)) + _ = os.RemoveAll(destinationFolder) + return + } + + var res = fmt.Sprintf("%v:%v", videoStream.Width, videoStream.Height) var level = resconv.Level(res) var opts = Options{ Input: sourceFilePath, @@ -214,8 +221,8 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { if metaProvider, ok := remoteStorage.(storage.MetaProvider); ok { var hls = HLS{ - Width: probe.FirstVideoStream().Width, - Height: probe.FirstVideoStream().Height, + Width: videoStream.Width, + Height: videoStream.Height, Source: task.ID + "_master.m3u8", Thumbnail: task.ID + ".jpg", } From 3d2a250df57e0da8fecc3af07258a3606b43e103 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 7 May 2025 16:37:41 +0700 Subject: [PATCH 17/52] fix: typos in README Signed-off-by: Alexander Onnikov --- README.md | 26 +++++++++++++------------- internal/pkg/storage/storage.go | 8 ++++---- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index efedffe08c..a1e12967c0 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,9 @@ ## About -The Stream is high-performance HTTP-based transcoding service. *Stream* supports **TUS protocol**, enabling reliable, -resumable transcodings. Designed for seamless and consistent media processing,it supports advanced transcoding features -with robust integration options. +The Stream is a high-performance HTTP-based transcoding service. *Stream* supports the **TUS protocol**, enabling +reliable, resumable transcoding. Designed for seamless and consistent media processing, it supports advanced transcoding +features with robust integration options. --- @@ -36,8 +36,8 @@ with robust integration options. #### Key Functionalities -- **Live transcoing with minimal upload time**: Transcoding results are going to be avaible after stream completion. -- **Transcoding Cancelation**: Cancel or pause ongoing transcoding in real-time. +- **Live transcoding with minimal upload time**: Transcoding results are available after stream completion. +- **Transcoding Cancellation**: Cancel or pause ongoing transcoding in real-time. - **Transcoding Resumption**: Resume incomplete transcoding tasks efficiently. ### Transcoding scheduling @@ -60,14 +60,14 @@ go mod tidy 2. Build the service: ```bash -docker build . -t hcengineering/sream:latest +docker build . -t hcengineering/stream:latest ``` --- -## Configuraiton +## Configuration -### App env configuraiton +### App Env Configuration The following environment variables can be used: @@ -77,11 +77,11 @@ STREAM_LOG_LEVEL String debug STREAM_SERVER_SECRET String server secret required to generate and verify tokens STREAM_PPROF_ENABLED True or False true starts profile server on localhost:6060 if true STREAM_INSECURE True or False false ignores authorization check if true -STREAM_SERVE_URL String 0.0.0.0:1080 listen on url +STREAM_SERVE_URL String 0.0.0.0:1080 listens on URL STREAM_ENDPOINT_URL URL s3://127.0.0.1:9000 S3 or Datalake endpoint, example: s3://my-ip-address, datalake://my-ip-address STREAM_MAX_PARALLEL_SCALING_COUNT Integer 2 how much parallel scaling can be processed STREAM_MAX_THREAD_COUNT Integer 4 max number of threads for transcoder -STREAM_OUTPUT_DIR String /tmp/transcoing/ path to the directory with tra +STREAM_OUTPUT_DIR String /tmp/transcoding/ path to the directory where transcoded files are stored STREAM_SENTRY_DSN String "" sentry dsn for error tracking ``` @@ -105,7 +105,7 @@ The service exposes an HTTP API. Below are some examples of how to interact with it. -### Trnascode via TUS +### Transcode via TUS ```bash curl -X POST http://localhost:1080/recording \ @@ -114,8 +114,8 @@ curl -X POST http://localhost:1080/recording \ --data-binary @path/to/your/file.mp4 ``` -Note: tus client is required, to play with a service locally you can use tus-js-client example -with [video](https://github.com/tus/tus-js-client/blob/main/demos/browser/video.html) +Note: A TUS client is required to play with the service locally. +You can use the tus-js-client example with [video](https://github.com/tus/tus-js-client/blob/main/demos/browser/video.html). ### Schedule a transcoding diff --git a/internal/pkg/storage/storage.go b/internal/pkg/storage/storage.go index 1914b26ac7..79b65ec9c2 100644 --- a/internal/pkg/storage/storage.go +++ b/internal/pkg/storage/storage.go @@ -38,8 +38,8 @@ type Storage interface { } // NewStorageByURL creates a new storage based on the type from the url scheme, for example "datalake://my-datalake-endpoint" -func NewStorageByURL(ctx context.Context, u *url.URL, storageType, token, worksapce string) (Storage, error) { - if worksapce == "" { +func NewStorageByURL(ctx context.Context, u *url.URL, storageType, token, workspace string) (Storage, error) { + if workspace == "" { return nil, errors.New("workspace is missed") } switch storageType { @@ -47,9 +47,9 @@ func NewStorageByURL(ctx context.Context, u *url.URL, storageType, token, worksa if token == "" { return nil, errors.New("token is missed") } - return NewDatalakeStorage(ctx, u.String(), worksapce, token), nil + return NewDatalakeStorage(ctx, u.String(), workspace, token), nil case "s3": - return NewS3(ctx, u.String(), worksapce), nil + return NewS3(ctx, u.String(), workspace), nil default: return nil, errors.New("unknown scheme") } From 0b70965641822555a098cc2646e109064ed009df Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 7 May 2025 16:41:46 +0700 Subject: [PATCH 18/52] add comment about CreateFormFile function Signed-off-by: Alexander Onnikov --- internal/pkg/storage/datalake.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index e366c98d70..068c22e8cd 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -56,6 +56,9 @@ func escapeQuotes(s string) string { return quoteEscaper.Replace(s) } +// multipart writer CreateFormFile function does not support custom content type +// here we have to have a modified copy that uses actual type instead of application/octet-stream +// see https://github.com/golang/go/issues/49329 func createFormFile(writer *multipart.Writer, fieldname, filename, contentType string) (io.Writer, error) { h := make(textproto.MIMEHeader) h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, escapeQuotes(fieldname), escapeQuotes(filename))) From f6d3e11fecaec6df080f91f10a44d211ddd16b28 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Tue, 13 May 2025 00:35:27 +0700 Subject: [PATCH 19/52] fix: enable conversion for mp4 files only Signed-off-by: Alexander Onnikov --- internal/pkg/mediaconvert/scheduler.go | 42 +++++++++++++++++++++++--- internal/pkg/storage/datalake.go | 32 ++++++++++++++++++++ internal/pkg/storage/s3.go | 21 +++++++++++++ internal/pkg/storage/storage.go | 7 +++++ 4 files changed, 98 insertions(+), 4 deletions(-) diff --git a/internal/pkg/mediaconvert/scheduler.go b/internal/pkg/mediaconvert/scheduler.go index 2471749906..24fc394425 100644 --- a/internal/pkg/mediaconvert/scheduler.go +++ b/internal/pkg/mediaconvert/scheduler.go @@ -121,20 +121,37 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { var destinationFolder = filepath.Join(p.cfg.OutputDir, task.ID) var _, filename = filepath.Split(task.Source) var sourceFilePath = filepath.Join(destinationFolder, filename) - _ = os.MkdirAll(destinationFolder, os.ModePerm) + err = os.MkdirAll(destinationFolder, os.ModePerm) + if err != nil { + logger.Error("can not create temporary folder", zap.Error(err)) + return + } logger.Debug("phase 3: get the remote file") remoteStorage, err := storage.NewStorageByURL(ctx, p.cfg.Endpoint(), p.cfg.EndpointURL.Scheme, tokenString, task.Workspace) if err != nil { - logger.Error("can not create storage by url", zap.Error(err)) + logger.Error("can not create storage by url", zap.Error(err), zap.String("url", p.cfg.EndpointURL.String())) + _ = os.RemoveAll(destinationFolder) + return + } + + stat, err := remoteStorage.StatFile(ctx, task.Source) + if err != nil { + logger.Error("can not stat a file", zap.Error(err), zap.String("filepath", task.Source)) + _ = os.RemoveAll(destinationFolder) + return + } + + if !IsSupportedMediaType(stat.Type) { + logger.Info("unsupported media type", zap.String("type", stat.Type)) _ = os.RemoveAll(destinationFolder) return } if err = remoteStorage.GetFile(ctx, task.Source, sourceFilePath); err != nil { - logger.Error("can not download a file", zap.Error(err)) + logger.Error("can not download a file", zap.Error(err), zap.String("filepath", task.Source)) _ = os.RemoveAll(destinationFolder) // TODO: reschedule return @@ -177,7 +194,12 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { SourceFile: sourceFilePath, }) - _ = manifest.GenerateHLSPlaylist(opts.ScalingLevels, p.cfg.OutputDir, opts.UploadID) + err = manifest.GenerateHLSPlaylist(opts.ScalingLevels, p.cfg.OutputDir, opts.UploadID) + if err != nil { + logger.Error("can not generate hls playlist", zap.String("out", p.cfg.OutputDir), zap.String("uploadID", opts.UploadID)) + _ = os.RemoveAll(destinationFolder) + return + } go uploader.Start() @@ -240,3 +262,15 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { } } } + +func IsSupportedMediaType(mediaType string) bool { + // Explicitly disable conversion for video/mp2t and video/x-mpegurl + switch mediaType { + case "video/mp2t", "video/x-mpegurl": + return false + case "video/mp4": + return true + default: + return false + } +} diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index 068c22e8cd..25d07268fc 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -271,5 +271,37 @@ func (d *DatalakeStorage) GetFile(ctx context.Context, filename, destination str return nil } +func (d *DatalakeStorage) StatFile(ctx context.Context, filename string) (*BlobInfo, error) { + var logger = d.logger.With(zap.String("head", d.workspace), zap.String("fileName", filename)) + logger.Debug("start") + + var objectKey = getObjectKey(filename) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(d.baseURL + "/blob/" + d.workspace + "/" + objectKey) + req.Header.SetMethod(fasthttp.MethodHead) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + return nil, err + } + + // Check the response status code + if resp.StatusCode() != fasthttp.StatusOK { + var err = fmt.Errorf("unexpected status code: %d", resp.StatusCode()) + logger.Debug("bad status code", zap.Error(err)) + return nil, err + } + + var info BlobInfo + info.Size = int64(resp.Header.ContentLength()) + info.Type = string(resp.Header.ContentType()) + info.ETag = string(resp.Header.Peek("ETag")) + return &info, nil +} + var _ Storage = (*DatalakeStorage)(nil) var _ MetaProvider = (*DatalakeStorage)(nil) diff --git a/internal/pkg/storage/s3.go b/internal/pkg/storage/s3.go index 3864b4fa18..3a4baca952 100644 --- a/internal/pkg/storage/s3.go +++ b/internal/pkg/storage/s3.go @@ -164,3 +164,24 @@ func (u *S3Storage) GetFile(ctx context.Context, filename, dest string) error { return nil } + +func (u *S3Storage) StatFile(ctx context.Context, filename string) (*BlobInfo, error) { + var logger = u.logger.With(zap.String("head", u.bucketName), zap.String("fileName", filename)) + + var head, err = u.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &u.bucketName, + Key: &filename, + }) + + if err != nil { + logger.Error("failed to head object", zap.Error(err)) + return nil, err + } + + var info BlobInfo + info.Size = *head.ContentLength + info.Type = *head.ContentType + info.ETag = *head.ETag + + return &info, nil +} diff --git a/internal/pkg/storage/storage.go b/internal/pkg/storage/storage.go index 79b65ec9c2..8eb726a337 100644 --- a/internal/pkg/storage/storage.go +++ b/internal/pkg/storage/storage.go @@ -30,11 +30,18 @@ type MetaProvider interface { PatchMeta(ctx context.Context, filename string, value *Metadata) error } +type BlobInfo struct { + Size int64 + Type string + ETag string +} + // Storage represents file-based storage type Storage interface { PutFile(ctx context.Context, fileName string) error DeleteFile(ctx context.Context, fileName string) error GetFile(ctx context.Context, fileName, destination string) error + StatFile(ctx context.Context, fileName string) (*BlobInfo, error) } // NewStorageByURL creates a new storage based on the type from the url scheme, for example "datalake://my-datalake-endpoint" From 3b5cfed3205e0a3ca2f916ec46a2a0dbbac950c1 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Tue, 13 May 2025 01:05:01 +0700 Subject: [PATCH 20/52] fix: increase min complexity Signed-off-by: Alexander Onnikov --- .golangci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yaml b/.golangci.yaml index a5335cf208..57e1b81c28 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -57,7 +57,7 @@ linters-settings: goimports: local-prefixes: github.com/networkservicemesh/sdk gocyclo: - min-complexity: 15 + min-complexity: 20 dupl: threshold: 150 funlen: From d82387f8473528bc744d0ea0500a681946a19e27 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Tue, 13 May 2025 01:08:52 +0700 Subject: [PATCH 21/52] fix: asjust linter settings Signed-off-by: Alexander Onnikov --- .golangci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yaml b/.golangci.yaml index 57e1b81c28..3c796cc8c1 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -62,7 +62,7 @@ linters-settings: threshold: 150 funlen: lines: 140 - statements: 80 + statements: 100 goconst: min-len: 2 min-occurrences: 2 From ce265f9518f1ed495acca530c36db45f1e4dd909 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Tue, 13 May 2025 01:13:53 +0700 Subject: [PATCH 22/52] fix: more fixes Signed-off-by: Alexander Onnikov --- .golangci.yaml | 2 +- internal/pkg/mediaconvert/scheduler.go | 1 + internal/pkg/storage/datalake.go | 1 + internal/pkg/storage/s3.go | 1 + internal/pkg/storage/storage.go | 1 + 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.golangci.yaml b/.golangci.yaml index 3c796cc8c1..16f9ae9d7b 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -61,7 +61,7 @@ linters-settings: dupl: threshold: 150 funlen: - lines: 140 + lines: 160 statements: 100 goconst: min-len: 2 diff --git a/internal/pkg/mediaconvert/scheduler.go b/internal/pkg/mediaconvert/scheduler.go index 24fc394425..5c686629ee 100644 --- a/internal/pkg/mediaconvert/scheduler.go +++ b/internal/pkg/mediaconvert/scheduler.go @@ -263,6 +263,7 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { } } +// IsSupportedMediaType checks whether transcoding is supported for given media type func IsSupportedMediaType(mediaType string) bool { // Explicitly disable conversion for video/mp2t and video/x-mpegurl switch mediaType { diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index 25d07268fc..b2f1f64bb8 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -271,6 +271,7 @@ func (d *DatalakeStorage) GetFile(ctx context.Context, filename, destination str return nil } +// StatFile gets file stat from the storage func (d *DatalakeStorage) StatFile(ctx context.Context, filename string) (*BlobInfo, error) { var logger = d.logger.With(zap.String("head", d.workspace), zap.String("fileName", filename)) logger.Debug("start") diff --git a/internal/pkg/storage/s3.go b/internal/pkg/storage/s3.go index 3a4baca952..96c2aa9bf1 100644 --- a/internal/pkg/storage/s3.go +++ b/internal/pkg/storage/s3.go @@ -165,6 +165,7 @@ func (u *S3Storage) GetFile(ctx context.Context, filename, dest string) error { return nil } +// StatFile gets file stat from the storage func (u *S3Storage) StatFile(ctx context.Context, filename string) (*BlobInfo, error) { var logger = u.logger.With(zap.String("head", u.bucketName), zap.String("fileName", filename)) diff --git a/internal/pkg/storage/storage.go b/internal/pkg/storage/storage.go index 8eb726a337..db619e8fdc 100644 --- a/internal/pkg/storage/storage.go +++ b/internal/pkg/storage/storage.go @@ -30,6 +30,7 @@ type MetaProvider interface { PatchMeta(ctx context.Context, filename string, value *Metadata) error } +// BlobInfo contains blob stat information type BlobInfo struct { Size int64 Type string From 1c6192a5762f6af27dc624413f12861ef29ad05d Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 14 May 2025 19:22:14 +0700 Subject: [PATCH 23/52] UBERF-10528 Set blob parent when uploading video segments Signed-off-by: Alexander Onnikov --- internal/pkg/storage/datalake.go | 43 +++++++++++++++++++++++++++++++ internal/pkg/storage/s3.go | 8 ++++++ internal/pkg/storage/storage.go | 1 + internal/pkg/uploader/uploader.go | 8 ++++++ 4 files changed, 60 insertions(+) diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index b2f1f64bb8..e5de6e818b 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -301,8 +301,51 @@ func (d *DatalakeStorage) StatFile(ctx context.Context, filename string) (*BlobI info.Size = int64(resp.Header.ContentLength()) info.Type = string(resp.Header.ContentType()) info.ETag = string(resp.Header.Peek("ETag")) + + logger.Debug("finished") return &info, nil } +// SetParent updates blob parent reference +func (d *DatalakeStorage) SetParent(ctx context.Context, filename string, parent string) error { + var logger = d.logger.With(zap.String("parent", d.workspace), zap.String("fileName", filename), zap.String("parent", parent)) + + logger.Debug("start") + + var objectKey = getObjectKey(filename) + var parentKey = getObjectKey(parent) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(d.baseURL + "/blob/" + d.workspace + "/" + objectKey + "/parent") + req.Header.SetMethod(fasthttp.MethodPatch) + req.Header.Add("Authorization", "Bearer "+d.token) + req.Header.Add("Content-Type", "application/json") + + body := map[string]any{ + "parent": parentKey, + } + json.NewEncoder(req.BodyWriter()).Encode(body) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + return err + } + + // Check the response status code + var statusOK = resp.StatusCode() >= 200 && resp.StatusCode() < 300 + if !statusOK { + var err = fmt.Errorf("unexpected status code: %d", resp.StatusCode()) + logger.Debug("bad status code", zap.Error(err)) + return err + } + + logger.Debug("finished") + + return nil +} + var _ Storage = (*DatalakeStorage)(nil) var _ MetaProvider = (*DatalakeStorage)(nil) diff --git a/internal/pkg/storage/s3.go b/internal/pkg/storage/s3.go index 96c2aa9bf1..3d76f0e6f9 100644 --- a/internal/pkg/storage/s3.go +++ b/internal/pkg/storage/s3.go @@ -186,3 +186,11 @@ func (u *S3Storage) StatFile(ctx context.Context, filename string) (*BlobInfo, e return &info, nil } + +// SetParent updates blob parent reference +func (u *S3Storage) SetParent(ctx context.Context, filename string, parent string) error { + // Not implemented for S3 Storage + return nil +} + +var _ Storage = (*S3Storage)(nil) diff --git a/internal/pkg/storage/storage.go b/internal/pkg/storage/storage.go index db619e8fdc..0feb587825 100644 --- a/internal/pkg/storage/storage.go +++ b/internal/pkg/storage/storage.go @@ -43,6 +43,7 @@ type Storage interface { DeleteFile(ctx context.Context, fileName string) error GetFile(ctx context.Context, fileName, destination string) error StatFile(ctx context.Context, fileName string) (*BlobInfo, error) + SetParent(ctx context.Context, fileName string, parentName string) error } // NewStorageByURL creates a new storage based on the type from the url scheme, for example "datalake://my-datalake-endpoint" diff --git a/internal/pkg/uploader/uploader.go b/internal/pkg/uploader/uploader.go index d251d4b4da..77b7cfccda 100644 --- a/internal/pkg/uploader/uploader.go +++ b/internal/pkg/uploader/uploader.go @@ -236,6 +236,14 @@ func (u *uploaderImpl) uploadAndDelete(f string) { } u.sentFiles.Store(f, struct{}{}) logger.Debug("file uploaded") + + if u.options.SourceFile != "" { + err = u.storage.SetParent(ctx, f, u.options.SourceFile) + if err != nil { + logger.Error("can not set blob parent", zap.Error(err), zap.String("filename", f), zap.String("source", u.options.SourceFile)) + } + } + break } From 917530d93ec9a33631462f71afef3e27a02c5254 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 14 May 2025 20:12:04 +0700 Subject: [PATCH 24/52] fix lint issues Signed-off-by: Alexander Onnikov --- internal/pkg/storage/datalake.go | 8 ++++++-- internal/pkg/storage/s3.go | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index e5de6e818b..f3b0132487 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -307,7 +307,7 @@ func (d *DatalakeStorage) StatFile(ctx context.Context, filename string) (*BlobI } // SetParent updates blob parent reference -func (d *DatalakeStorage) SetParent(ctx context.Context, filename string, parent string) error { +func (d *DatalakeStorage) SetParent(ctx context.Context, filename, parent string) error { var logger = d.logger.With(zap.String("parent", d.workspace), zap.String("fileName", filename), zap.String("parent", parent)) logger.Debug("start") @@ -325,7 +325,11 @@ func (d *DatalakeStorage) SetParent(ctx context.Context, filename string, parent body := map[string]any{ "parent": parentKey, } - json.NewEncoder(req.BodyWriter()).Encode(body) + + if err := json.NewEncoder(req.BodyWriter()).Encode(body); err != nil { + logger.Debug("can not encode body", zap.Error(err)) + return err + } resp := fasthttp.AcquireResponse() defer fasthttp.ReleaseResponse(resp) diff --git a/internal/pkg/storage/s3.go b/internal/pkg/storage/s3.go index 3d76f0e6f9..e4f3050d09 100644 --- a/internal/pkg/storage/s3.go +++ b/internal/pkg/storage/s3.go @@ -188,7 +188,7 @@ func (u *S3Storage) StatFile(ctx context.Context, filename string) (*BlobInfo, e } // SetParent updates blob parent reference -func (u *S3Storage) SetParent(ctx context.Context, filename string, parent string) error { +func (u *S3Storage) SetParent(ctx context.Context, filename, parent string) error { // Not implemented for S3 Storage return nil } From 69cdad3154ed6bbaea7bfa2bda7931158f913c93 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Tue, 27 May 2025 11:40:53 +0700 Subject: [PATCH 25/52] fix: copy audio stream Signed-off-by: Alexander Onnikov --- internal/pkg/mediaconvert/command.go | 8 ++++---- internal/pkg/mediaconvert/command_test.go | 4 ++-- internal/pkg/resconv/resconv_test.go | 5 +++++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/internal/pkg/mediaconvert/command.go b/internal/pkg/mediaconvert/command.go index 2b68893404..ceb95c85de 100644 --- a/internal/pkg/mediaconvert/command.go +++ b/internal/pkg/mediaconvert/command.go @@ -76,8 +76,8 @@ func BuildAudioCommand(opts *Options) []string { // BuildRawVideoCommand returns an extremely lightweight ffmpeg command for converting raw video without extra cost. func BuildRawVideoCommand(opts *Options) []string { return append(buildCommonComamnd(opts), - "-c:v", - "copy", + "-c:a", "copy", // Copy audio stream + "-c:v", "copy", // Copy video stream "-hls_time", "5", "-hls_flags", "split_by_time", "-hls_list_size", "0", @@ -101,8 +101,8 @@ func BuildScalingVideoCommand(opts *Options) []string { for _, level := range opts.ScalingLevels { result = append(result, "-vf", "scale=-2:"+level[:len(level)-1], - "-c:v", - "libx264", + "-c:a", "copy", // Copy audio stream + "-c:v", "libx264", "-preset", "veryfast", "-crf", "23", "-g", "60", diff --git a/internal/pkg/mediaconvert/command_test.go b/internal/pkg/mediaconvert/command_test.go index 42ddc4ea4d..b67fe8060b 100644 --- a/internal/pkg/mediaconvert/command_test.go +++ b/internal/pkg/mediaconvert/command_test.go @@ -31,7 +31,7 @@ func Test_BuildVideoCommand_Scaling(t *testing.T) { ScalingLevels: []string{"720p", "480p"}, }) - const expected = `-threads 4 -i pipe:0 -vf scale=-2:720 -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -vf scale=-2:480 -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `-threads 4 -i pipe:0 -vf scale=-2:720 -c:a copy -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -vf scale=-2:480 -c:a copy -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(scaleCommand, " ")) } @@ -45,7 +45,7 @@ func Test_BuildVideoCommand_Raw(t *testing.T) { Level: resconv.Level("651:490"), }) - const expected = `"-threads 4 -i pipe:0 -c:v copy -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `"-threads 4 -i pipe:0 -c:a copy -c:v copy -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(rawCommand, " ")) } diff --git a/internal/pkg/resconv/resconv_test.go b/internal/pkg/resconv/resconv_test.go index 48fd291d57..553beb6d05 100644 --- a/internal/pkg/resconv/resconv_test.go +++ b/internal/pkg/resconv/resconv_test.go @@ -91,6 +91,11 @@ func Test_Resconv_ShouldReturnCorrectPrefixes(t *testing.T) { res: "2560:1440", expected: []string{"1080p", "720p"}, }, + { + name: "pixels between 1k and 2k", + res: "2880:1800", + expected: []string{"1080p", "720p"}, + }, { name: "pixels equal to 2k", res: "3840:2160", From 9becda38ac76c26901252b728d18304449633c02 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 28 May 2025 12:49:06 +0700 Subject: [PATCH 26/52] fix: change artifact name Signed-off-by: Alexander Onnikov --- .github/workflows/docker-push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-push.yaml b/.github/workflows/docker-push.yaml index 47baea039f..eb2621cfd4 100644 --- a/.github/workflows/docker-push.yaml +++ b/.github/workflows/docker-push.yaml @@ -41,4 +41,4 @@ jobs: context: . platforms: linux/amd64,linux/arm64 push: true - tags: hardcoreeng/stream:${{ inputs.version }} + tags: hardcoreeng/service_stream:${{ inputs.version }} From b35382b21679d563a808acd78c0c07fdc44c8c4f Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Fri, 30 May 2025 17:32:03 +0700 Subject: [PATCH 27/52] fix: properly initialize uploader Signed-off-by: Alexander Onnikov --- internal/pkg/uploader/uploader.go | 97 ++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 33 deletions(-) diff --git a/internal/pkg/uploader/uploader.go b/internal/pkg/uploader/uploader.go index 77b7cfccda..247f5ec84a 100644 --- a/internal/pkg/uploader/uploader.go +++ b/internal/pkg/uploader/uploader.go @@ -91,20 +91,10 @@ func New(ctx context.Context, s storage.Storage, opts Options) Uploader { res.uploadCtx, res.uploadCancel = context.WithCancel(context.Background()) - _ = os.MkdirAll(opts.Dir, os.ModePerm) - res.workerWaitGroup.Add(1) - - go func() { - defer res.workerWaitGroup.Done() - initFiles, _ := os.ReadDir(opts.Dir) - for _, f := range initFiles { - var filePath = filepath.Join(opts.Dir, f.Name()) - if filePath == opts.SourceFile { - continue - } - res.filesCh <- filePath - } - }() + err := os.MkdirAll(opts.Dir, os.ModePerm) + if err != nil { + res.logger.Error("can not create upload directory", zap.Error(err), zap.String("dir", opts.Dir)) + } return res } @@ -117,6 +107,35 @@ func (u *uploaderImpl) Cancel() { u.stop(true) } +func (u *uploaderImpl) scanInitialFiles() { + u.workerWaitGroup.Add(1) + + go func() { + defer u.workerWaitGroup.Done() + + initFiles, err := os.ReadDir(u.options.Dir) + if err != nil { + u.logger.Error("failed to read initial files", zap.Error(err), zap.String("dir", u.options.Dir)) + return + } + + for _, f := range initFiles { + if f.IsDir() { + continue + } + + var filePath = filepath.Join(u.options.Dir, f.Name()) + if filePath == u.options.SourceFile { + continue + } + u.filesCh <- filePath + } + + u.logger.Info("initial file scan complete", zap.String("dir", u.options.Dir), zap.Int("count", len(initFiles))) + }() + +} + func (u *uploaderImpl) stop(rollback bool) { close(u.watcherStopCh) <-u.watcherDoneCh @@ -147,8 +166,14 @@ func (u *uploaderImpl) stop(rollback bool) { } func (u *uploaderImpl) Start() { + watcherReady := make(chan struct{}) + u.startWorkers() - u.startWatch() + go u.startWatch(watcherReady) + + <-watcherReady + + u.scanInitialFiles() } func (u *uploaderImpl) startWorkers() { @@ -230,13 +255,11 @@ func (u *uploaderImpl) uploadAndDelete(f string) { if err != nil { logger.Error("attempt failed", zap.Error(err)) } else { - if !u.shouldDeleteOnStop(f) { - _ = os.Remove(f) - logger.Debug("removed file locally") - } + // Mark the file as uploaded u.sentFiles.Store(f, struct{}{}) logger.Debug("file uploaded") + // Update the file's parent if SourceFile is set if u.options.SourceFile != "" { err = u.storage.SetParent(ctx, f, u.options.SourceFile) if err != nil { @@ -244,6 +267,16 @@ func (u *uploaderImpl) uploadAndDelete(f string) { } } + // Delete the file locally if it should be deleted + if !u.shouldDeleteOnStop(f) { + if err := os.Remove(f); err != nil { + logger.Error("failed to remove file locally", zap.Error(err), zap.String("file", f)) + } else { + logger.Debug("removed file locally") + } + } + + break } @@ -251,28 +284,26 @@ func (u *uploaderImpl) uploadAndDelete(f string) { } } -func (u *uploaderImpl) startWatch() { +func (u *uploaderImpl) startWatch(ready chan<- struct{}) { + defer close(u.watcherDoneCh) + var logger = u.logger.With(zap.String("func", "startWatch")) var watcher, err = inotify.NewWatcher() if err != nil { logger.Error("can not start file watcher", zap.Error(err)) + close(ready) + return + } + defer watcher.Close() + + if err := watcher.AddWatch(u.options.Dir, inotifyCloseWrite | inotifyMovedTo); err != nil { + logger.Error("can not start watching", zap.Error(err)) + close(ready) return } - if err := watcher.AddWatch(u.options.Dir, inotifyCloseWrite); err != nil { - logger.Error("can not start watching for close write", zap.Error(err)) - return - } - if err := watcher.AddWatch(u.options.Dir, inotifyMovedTo); err != nil { - logger.Error("can not start watching for moved to", zap.Error(err)) - return - } - defer func() { - _ = watcher.Close() - close(u.watcherDoneCh) - }() - + close(ready) logger.Debug("watching for file updates") defer logger.Debug("done") From d2f20a5acc7ec45ba1b27801e1de8f10c68876f2 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Fri, 30 May 2025 18:31:07 +0700 Subject: [PATCH 28/52] fix: typo fixes Signed-off-by: Alexander Onnikov --- internal/pkg/api/v1/transcoding/handler.go | 6 +++++- internal/pkg/mediaconvert/command.go | 22 +++++++++++----------- internal/pkg/mediaconvert/command_test.go | 4 ++-- internal/pkg/mediaconvert/coordinator.go | 2 +- internal/pkg/mediaconvert/scheduler.go | 14 ++++++++++---- internal/pkg/mediaconvert/stream.go | 2 +- 6 files changed, 30 insertions(+), 20 deletions(-) diff --git a/internal/pkg/api/v1/transcoding/handler.go b/internal/pkg/api/v1/transcoding/handler.go index 67c4ca1fd0..c3a3ecfc92 100644 --- a/internal/pkg/api/v1/transcoding/handler.go +++ b/internal/pkg/api/v1/transcoding/handler.go @@ -60,7 +60,11 @@ func (t *trascodeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - t.scheduler.Schedule(&task) + if err := t.scheduler.Schedule(&task); err != nil { + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) } diff --git a/internal/pkg/mediaconvert/command.go b/internal/pkg/mediaconvert/command.go index ceb95c85de..29245eccb2 100644 --- a/internal/pkg/mediaconvert/command.go +++ b/internal/pkg/mediaconvert/command.go @@ -32,7 +32,7 @@ import ( // Options represents configuration for the ffmpeg command type Options struct { Input string - OuputDir string + OutputDir string ScalingLevels []string Level string Threads int @@ -56,7 +56,7 @@ func newFfmpegCommand(ctx context.Context, in io.Reader, args []string) (*exec.C return result, nil } -func buildCommonComamnd(opts *Options) []string { +func buildCommonCommand(opts *Options) []string { return []string{ "-threads", fmt.Sprint(opts.Threads), "-i", opts.Input, @@ -65,24 +65,24 @@ func buildCommonComamnd(opts *Options) []string { // BuildAudioCommand returns flags for getting the audio from the input func BuildAudioCommand(opts *Options) []string { - var commonPart = buildCommonComamnd(opts) + var commonPart = buildCommonCommand(opts) return append(commonPart, "-vn", "-acodec", - "copy", filepath.Join(opts.OuputDir, opts.UploadID), + "copy", filepath.Join(opts.OutputDir, opts.UploadID), ) } // BuildRawVideoCommand returns an extremely lightweight ffmpeg command for converting raw video without extra cost. func BuildRawVideoCommand(opts *Options) []string { - return append(buildCommonComamnd(opts), + return append(buildCommonCommand(opts), "-c:a", "copy", // Copy audio stream "-c:v", "copy", // Copy video stream "-hls_time", "5", "-hls_flags", "split_by_time", "-hls_list_size", "0", - "-hls_segment_filename", filepath.Join(opts.OuputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", opts.Level)), - filepath.Join(opts.OuputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, opts.Level))) + "-hls_segment_filename", filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", opts.Level)), + filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, opts.Level))) } // BuildThumbnailCommand creates a command that creates a thumbnail for the input video @@ -90,13 +90,13 @@ func BuildThumbnailCommand(opts *Options) []string { return append([]string{}, "-i", opts.Input, "-vframes", "1", - filepath.Join(opts.OuputDir, opts.UploadID, opts.UploadID+".jpg"), + filepath.Join(opts.OutputDir, opts.UploadID, opts.UploadID+".jpg"), ) } // BuildScalingVideoCommand returns flags for ffmpeg for video scaling func BuildScalingVideoCommand(opts *Options) []string { - var result = buildCommonComamnd(opts) + var result = buildCommonCommand(opts) for _, level := range opts.ScalingLevels { result = append(result, @@ -109,8 +109,8 @@ func BuildScalingVideoCommand(opts *Options) []string { "-hls_time", "5", "-hls_flags", "split_by_time", "-hls_list_size", "0", - "-hls_segment_filename", filepath.Join(opts.OuputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", level)), - filepath.Join(opts.OuputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, level))) + "-hls_segment_filename", filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", level)), + filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, level))) } return result diff --git a/internal/pkg/mediaconvert/command_test.go b/internal/pkg/mediaconvert/command_test.go index b67fe8060b..09ef87cd8a 100644 --- a/internal/pkg/mediaconvert/command_test.go +++ b/internal/pkg/mediaconvert/command_test.go @@ -24,7 +24,7 @@ import ( func Test_BuildVideoCommand_Scaling(t *testing.T) { var scaleCommand = mediaconvert.BuildScalingVideoCommand(&mediaconvert.Options{ - OuputDir: "test", + OutputDir: "test", Input: "pipe:0", UploadID: "1", Threads: 4, @@ -38,7 +38,7 @@ func Test_BuildVideoCommand_Scaling(t *testing.T) { func Test_BuildVideoCommand_Raw(t *testing.T) { var rawCommand = mediaconvert.BuildRawVideoCommand(&mediaconvert.Options{ - OuputDir: "test", + OutputDir: "test", Input: "pipe:0", UploadID: "1", Threads: 4, diff --git a/internal/pkg/mediaconvert/coordinator.go b/internal/pkg/mediaconvert/coordinator.go index 843716ce79..80c4b3b21a 100644 --- a/internal/pkg/mediaconvert/coordinator.go +++ b/internal/pkg/mediaconvert/coordinator.go @@ -97,7 +97,7 @@ func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo var commandOptions = Options{ Input: "pipe:0", - OuputDir: s.conf.OutputDir, + OutputDir: s.conf.OutputDir, Threads: s.conf.MaxThreadCount, UploadID: info.ID, Level: level, diff --git a/internal/pkg/mediaconvert/scheduler.go b/internal/pkg/mediaconvert/scheduler.go index 5c686629ee..4c20d52ce6 100644 --- a/internal/pkg/mediaconvert/scheduler.go +++ b/internal/pkg/mediaconvert/scheduler.go @@ -62,15 +62,15 @@ type Scheduler struct { } // Schedule schedules a task to transcode -func (p *Scheduler) Schedule(t *Task) { +func (p *Scheduler) Schedule(t *Task) error { t.ID = uuid.NewString() t.Status = "planned" select { case p.taskCh <- t: - p.logger.Sugar().Debugf("task %v is scheduled", t) + return nil default: - p.logger.Error("task channel is full") + return fmt.Errorf("task queue is full") } } @@ -127,6 +127,12 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { return } + defer func() { + if err := os.RemoveAll(destinationFolder); err != nil { + logger.Error("failed to cleanup temporary folder", zap.Error(err)) + } + }() + logger.Debug("phase 3: get the remote file") remoteStorage, err := storage.NewStorageByURL(ctx, p.cfg.Endpoint(), p.cfg.EndpointURL.Scheme, tokenString, task.Workspace) @@ -176,7 +182,7 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { var level = resconv.Level(res) var opts = Options{ Input: sourceFilePath, - OuputDir: p.cfg.OutputDir, + OutputDir: p.cfg.OutputDir, Level: level, ScalingLevels: append(resconv.SubLevels(res), level), UploadID: task.ID, diff --git a/internal/pkg/mediaconvert/stream.go b/internal/pkg/mediaconvert/stream.go index ab47c8e771..db207e9f61 100644 --- a/internal/pkg/mediaconvert/stream.go +++ b/internal/pkg/mediaconvert/stream.go @@ -115,7 +115,7 @@ func (s *StreamCoordinator) AsConcatableUpload(upload handler.Upload) handler.Co func (w *Stream) start(ctx context.Context, options *Options) error { defer w.logger.Debug("start done") w.reader = w.writer.Transpile() - if err := manifest.GenerateHLSPlaylist(append(options.ScalingLevels, options.Level), options.OuputDir, options.UploadID); err != nil { + if err := manifest.GenerateHLSPlaylist(append(options.ScalingLevels, options.Level), options.OutputDir, options.UploadID); err != nil { return err } w.commandGroup.Add(1) From faeca99748169d1f7663ca20901c1de5c42775a2 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Fri, 30 May 2025 20:39:28 +0700 Subject: [PATCH 29/52] ci fixes Signed-off-by: Alexander Onnikov --- .golangci.yaml | 2 +- internal/pkg/mediaconvert/command_test.go | 10 +++++----- internal/pkg/mediaconvert/coordinator.go | 2 +- internal/pkg/mediaconvert/scheduler.go | 4 ++-- internal/pkg/uploader/uploader.go | 10 ++++++---- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 16f9ae9d7b..5fd9dcf2b3 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -61,7 +61,7 @@ linters-settings: dupl: threshold: 150 funlen: - lines: 160 + lines: 180 statements: 100 goconst: min-len: 2 diff --git a/internal/pkg/mediaconvert/command_test.go b/internal/pkg/mediaconvert/command_test.go index 09ef87cd8a..b9cc151a77 100644 --- a/internal/pkg/mediaconvert/command_test.go +++ b/internal/pkg/mediaconvert/command_test.go @@ -24,7 +24,7 @@ import ( func Test_BuildVideoCommand_Scaling(t *testing.T) { var scaleCommand = mediaconvert.BuildScalingVideoCommand(&mediaconvert.Options{ - OutputDir: "test", + OutputDir: "test", Input: "pipe:0", UploadID: "1", Threads: 4, @@ -39,10 +39,10 @@ func Test_BuildVideoCommand_Scaling(t *testing.T) { func Test_BuildVideoCommand_Raw(t *testing.T) { var rawCommand = mediaconvert.BuildRawVideoCommand(&mediaconvert.Options{ OutputDir: "test", - Input: "pipe:0", - UploadID: "1", - Threads: 4, - Level: resconv.Level("651:490"), + Input: "pipe:0", + UploadID: "1", + Threads: 4, + Level: resconv.Level("651:490"), }) const expected = `"-threads 4 -i pipe:0 -c:a copy -c:v copy -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` diff --git a/internal/pkg/mediaconvert/coordinator.go b/internal/pkg/mediaconvert/coordinator.go index 80c4b3b21a..ede68b3b65 100644 --- a/internal/pkg/mediaconvert/coordinator.go +++ b/internal/pkg/mediaconvert/coordinator.go @@ -97,7 +97,7 @@ func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo var commandOptions = Options{ Input: "pipe:0", - OutputDir: s.conf.OutputDir, + OutputDir: s.conf.OutputDir, Threads: s.conf.MaxThreadCount, UploadID: info.ID, Level: level, diff --git a/internal/pkg/mediaconvert/scheduler.go b/internal/pkg/mediaconvert/scheduler.go index 4c20d52ce6..31507f1747 100644 --- a/internal/pkg/mediaconvert/scheduler.go +++ b/internal/pkg/mediaconvert/scheduler.go @@ -128,7 +128,7 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { } defer func() { - if err := os.RemoveAll(destinationFolder); err != nil { + if err = os.RemoveAll(destinationFolder); err != nil { logger.Error("failed to cleanup temporary folder", zap.Error(err)) } }() @@ -182,7 +182,7 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { var level = resconv.Level(res) var opts = Options{ Input: sourceFilePath, - OutputDir: p.cfg.OutputDir, + OutputDir: p.cfg.OutputDir, Level: level, ScalingLevels: append(resconv.SubLevels(res), level), UploadID: task.ID, diff --git a/internal/pkg/uploader/uploader.go b/internal/pkg/uploader/uploader.go index 247f5ec84a..ad1798fa00 100644 --- a/internal/pkg/uploader/uploader.go +++ b/internal/pkg/uploader/uploader.go @@ -133,7 +133,6 @@ func (u *uploaderImpl) scanInitialFiles() { u.logger.Info("initial file scan complete", zap.String("dir", u.options.Dir), zap.Int("count", len(initFiles))) }() - } func (u *uploaderImpl) stop(rollback bool) { @@ -276,7 +275,6 @@ func (u *uploaderImpl) uploadAndDelete(f string) { } } - break } @@ -295,9 +293,13 @@ func (u *uploaderImpl) startWatch(ready chan<- struct{}) { close(ready) return } - defer watcher.Close() + defer func() { + if err := watcher.Close(); err != nil { + logger.Error("can not close watcher", zap.Error(err)) + } + }() - if err := watcher.AddWatch(u.options.Dir, inotifyCloseWrite | inotifyMovedTo); err != nil { + if err := watcher.AddWatch(u.options.Dir, inotifyCloseWrite|inotifyMovedTo); err != nil { logger.Error("can not start watching", zap.Error(err)) close(ready) return From d55eccd628665688824020d95c09a7ebe6cb4421 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 11 Jun 2025 12:40:45 +0700 Subject: [PATCH 30/52] feat: use queue for transcoding tasks input Signed-off-by: Alexander Onnikov --- cmd/stream/main.go | 60 +++++- go.mod | 10 +- go.sum | 74 ++++++- internal/pkg/config/config.go | 3 + internal/pkg/mediaconvert/command.go | 64 +++++- internal/pkg/mediaconvert/command_test.go | 39 +++- internal/pkg/mediaconvert/coordinator.go | 4 +- internal/pkg/mediaconvert/scheduler.go | 24 ++- internal/pkg/mediaconvert/stream.go | 1 + internal/pkg/mediaconvert/transcoder.go | 234 ++++++++++++++++++++++ internal/pkg/queue/config.go | 43 ++++ internal/pkg/queue/config_test.go | 90 +++++++++ internal/pkg/queue/queue.go | 162 +++++++++++++++ internal/pkg/queue/worker.go | 138 +++++++++++++ internal/pkg/resconv/resconv.go | 36 ++-- internal/pkg/resconv/resconv_test.go | 51 ++++- internal/pkg/storage/datalake.go | 72 +++++-- internal/pkg/storage/s3.go | 5 + internal/pkg/storage/storage.go | 1 + internal/pkg/uploader/options.go | 1 + internal/pkg/uploader/uploader.go | 43 +++- 21 files changed, 1081 insertions(+), 74 deletions(-) create mode 100644 internal/pkg/mediaconvert/transcoder.go create mode 100644 internal/pkg/queue/config.go create mode 100644 internal/pkg/queue/config_test.go create mode 100644 internal/pkg/queue/queue.go create mode 100644 internal/pkg/queue/worker.go diff --git a/cmd/stream/main.go b/cmd/stream/main.go index ea63605dac..0edc84ab52 100644 --- a/cmd/stream/main.go +++ b/cmd/stream/main.go @@ -17,6 +17,7 @@ package main import ( "context" "net/http" + "sync" "os" "os/signal" @@ -31,6 +32,7 @@ import ( "github.com/hcengineering/stream/internal/pkg/api/v1/transcoding" "github.com/hcengineering/stream/internal/pkg/config" "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/queue" ) func main() { @@ -49,7 +51,7 @@ func main() { if err != nil { panic(err.Error()) } - logger.Sugar().Debugf("parsed config is %v", cfg) + logger.Sugar().Debug("using config", zap.Any("config", cfg)) if cfg.SentryDsn != "" { if err := sentry.Init(sentry.ClientOptions{ @@ -82,17 +84,63 @@ func main() { handler = sentryHandler.Handle(mux) } + server := &http.Server{ + Addr: cfg.ServeURL, + Handler: handler, + } + + var wg sync.WaitGroup + + wg.Add(1) go func() { - logger.Info("server started serving", zap.String("ServeURL", cfg.ServeURL)) - defer logger.Info("server finished") + defer wg.Done() + + logger.Info("http server started", zap.String("ServeURL", cfg.ServeURL)) + defer logger.Info("http server finished") // #nosec - var err = http.ListenAndServe(cfg.ServeURL, handler) - if err != nil { + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + logger.Error("http server error", zap.Error(err)) + cancel() + } + }() + + config := queue.ParseConfig(cfg.QueueConfig, "stream", cfg.Region) + logger.Info("using queue config", zap.Any("config", config)) + consumer := queue.NewConsumer(ctx, queue.TopicTranscodeRequest, "stream", config) + // producer := queue.NewProducer(queue.TopicTranscodeResult, config) + + wg.Add(1) + go func() { + defer wg.Done() + + logger.Info("queue worker started") + defer logger.Info("queue worker finished") + + worker := queue.NewWorker(ctx, consumer, cfg) + if err := worker.Start(ctx); err != nil { + logger.Error("queue worker error", zap.Error(err)) cancel() - logger.Debug("unable to listen", zap.Error(err)) } }() <-ctx.Done() + + // Stop HTTP server to prevent accepting new connections + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer shutdownCancel() + + if err := server.Shutdown(shutdownCtx); err != nil { + logger.Error("http server shutdown error", zap.Error(err)) + } + + // Wait for all workers to complete + wg.Wait() + + // Close the consumer + if err := consumer.Close(); err != nil { + logger.Error("failed to close queue consumer", zap.Error(err)) + } + + logger.Info("shutdown complete") } diff --git a/go.mod b/go.mod index 30d0f9ea7c..905e524afa 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/google/uuid v1.6.0 github.com/kelseyhightower/envconfig v1.4.0 github.com/pkg/errors v0.9.1 + github.com/segmentio/kafka-go v0.4.48 github.com/stretchr/testify v1.10.0 github.com/tus/tusd/v2 v2.6.0 github.com/valyala/fasthttp v1.59.0 @@ -38,12 +39,13 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect github.com/aws/smithy-go v1.22.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.37.0 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/net v0.40.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.25.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index ca1613a2fa..c531b49b39 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,7 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 h1:TzeR06UCMUq+KA3bDkujxK1GVGy github.com/aws/aws-sdk-go-v2/service/sts v1.33.14/go.mod h1:dspXf/oYWGWo6DEvj98wpaTeqt5+DMidZD0A9BYTizc= github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/getsentry/sentry-go v0.31.1 h1:ELVc0h7gwyhnXHDouXkhqTFSO5oslsRDk0++eyE0KJ4= @@ -54,12 +55,16 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -68,6 +73,12 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/segmentio/kafka-go v0.4.48 h1:9jyu9CWK4W5W+SroCe8EffbrRZVqAOkuaLd/ApID4Vs= +github.com/segmentio/kafka-go v0.4.48/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tus/tusd/v2 v2.6.0 h1:Je243QDKnFTvm/WkLH2bd1oQ+7trolrflRWyuI0PdWI= @@ -76,27 +87,74 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.59.0 h1:Qu0qYHfXvPk1mSLNqcFtEk6DpxgA26hy6bmydotDpRI= github.com/valyala/fasthttp v1.59.0/go.mod h1:GTxNb9Bc6r2a9D0TWNSPwDz78UxnTGBViY3xZNEqyYU= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20250215185904-eff6e970281f h1:oFMYAjX0867ZD2jcNiLBrI9BdpmEkvPyi5YrBGXbamg= golang.org/x/exp v0.0.0-20250215185904-eff6e970281f/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk= -golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= -golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/vansante/go-ffprobe.v2 v2.2.1 h1:sFV08OT1eZ1yroLCZVClIVd9YySgCh9eGjBWO0oRayI= gopkg.in/vansante/go-ffprobe.v2 v2.2.1/go.mod h1:qF0AlAjk7Nqzqf3y333Ly+KxN3cKF2JqA3JT5ZheUGE= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0= diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 0b0badfcf0..f182a61075 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -35,6 +35,9 @@ type Config struct { MaxParallelScalingCount int `split_words:"true" default:"2" desc:"how much parallel scaling can be processed"` MaxThreadCount int `split_words:"true" default:"4" desc:"max number of threads for transcoder"` + QueueConfig string `split_words:"true" default:"" desc:"queue config"` + Region string `split_words:"true" default:"" desc:"service region"` + OutputDir string `split_words:"true" default:"/tmp/transcoing/" desc:"path to the directory with transcoding result."` Timeout time.Duration `default:"5m" desc:"timeout for the upload"` } diff --git a/internal/pkg/mediaconvert/command.go b/internal/pkg/mediaconvert/command.go index 29245eccb2..872f9f26d6 100644 --- a/internal/pkg/mediaconvert/command.go +++ b/internal/pkg/mediaconvert/command.go @@ -22,6 +22,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "github.com/pkg/errors" @@ -35,8 +36,10 @@ type Options struct { OutputDir string ScalingLevels []string Level string + Transcode bool Threads int UploadID string + WithAudio bool } func newFfmpegCommand(ctx context.Context, in io.Reader, args []string) (*exec.Cmd, error) { @@ -57,10 +60,20 @@ func newFfmpegCommand(ctx context.Context, in io.Reader, args []string) (*exec.C } func buildCommonCommand(opts *Options) []string { - return []string{ + var result = []string{ "-threads", fmt.Sprint(opts.Threads), "-i", opts.Input, } + + if strings.HasPrefix(opts.Input, "http://") || strings.HasPrefix(opts.Input, "https://") { + result = append(result, + "-reconnect", "1", + "-reconnect_streamed", "1", + "-reconnect_delay_max", "5", + ) + } + + return result } // BuildAudioCommand returns flags for getting the audio from the input @@ -75,14 +88,30 @@ func BuildAudioCommand(opts *Options) []string { // BuildRawVideoCommand returns an extremely lightweight ffmpeg command for converting raw video without extra cost. func BuildRawVideoCommand(opts *Options) []string { - return append(buildCommonCommand(opts), - "-c:a", "copy", // Copy audio stream - "-c:v", "copy", // Copy video stream - "-hls_time", "5", - "-hls_flags", "split_by_time", - "-hls_list_size", "0", - "-hls_segment_filename", filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", opts.Level)), - filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, opts.Level))) + if opts.Transcode { + return append(buildCommonCommand(opts), + "-c:a", "aac", + "-c:v", "libx264", + "-preset", "veryfast", + "-crf", "23", + "-g", "60", + "-f", "hls", + "-hls_time", "5", + "-hls_flags", "split_by_time", + "-hls_list_size", "0", + "-hls_segment_filename", filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", opts.Level)), + filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, opts.Level))) + } else { + return append(buildCommonCommand(opts), + "-c:a", "copy", // Copy audio stream + "-c:v", "copy", // Copy video stream + "-f", "hls", + "-hls_time", "5", + "-hls_flags", "split_by_time", + "-hls_list_size", "0", + "-hls_segment_filename", filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", opts.Level)), + filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, opts.Level))) + } } // BuildThumbnailCommand creates a command that creates a thumbnail for the input video @@ -90,6 +119,7 @@ func BuildThumbnailCommand(opts *Options) []string { return append([]string{}, "-i", opts.Input, "-vframes", "1", + "-update", "1", filepath.Join(opts.OutputDir, opts.UploadID, opts.UploadID+".jpg"), ) } @@ -99,13 +129,27 @@ func BuildScalingVideoCommand(opts *Options) []string { var result = buildCommonCommand(opts) for _, level := range opts.ScalingLevels { + if level == opts.Level { + continue + } + + // result = append(result, "-map", "0:v") + if opts.WithAudio { + // result = append(result, + // "-map", "0:a", + // "-c:a", "aac", + // ) + } + result = append(result, + "-map", "0:v", "-vf", "scale=-2:"+level[:len(level)-1], - "-c:a", "copy", // Copy audio stream + "-c:a", "aac", "-c:v", "libx264", "-preset", "veryfast", "-crf", "23", "-g", "60", + "-f", "hls", "-hls_time", "5", "-hls_flags", "split_by_time", "-hls_list_size", "0", diff --git a/internal/pkg/mediaconvert/command_test.go b/internal/pkg/mediaconvert/command_test.go index b9cc151a77..531c3ed588 100644 --- a/internal/pkg/mediaconvert/command_test.go +++ b/internal/pkg/mediaconvert/command_test.go @@ -31,21 +31,54 @@ func Test_BuildVideoCommand_Scaling(t *testing.T) { ScalingLevels: []string{"720p", "480p"}, }) - const expected = `-threads 4 -i pipe:0 -vf scale=-2:720 -c:a copy -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -vf scale=-2:480 -c:a copy -c:v libx264 -preset veryfast -crf 23 -g 60 -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `-threads 4 -i pipe:0 -map 0:v -vf scale=-2:720 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -map 0:v -vf scale=-2:480 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(scaleCommand, " ")) } -func Test_BuildVideoCommand_Raw(t *testing.T) { +func Test_BuildVideoCommand_Scaling_NoRaw(t *testing.T) { + var scaleCommand = mediaconvert.BuildScalingVideoCommand(&mediaconvert.Options{ + OutputDir: "test", + Input: "pipe:0", + UploadID: "1", + Threads: 4, + Level: "720p", + ScalingLevels: []string{"720p", "480p"}, + }) + + const expected = `-threads 4 -i pipe:0 -map 0:v -vf scale=-2:480 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + + require.Contains(t, expected, strings.Join(scaleCommand, " ")) +} + +func Test_BuildVideoCommand_Raw_NoTranscode(t *testing.T) { var rawCommand = mediaconvert.BuildRawVideoCommand(&mediaconvert.Options{ OutputDir: "test", Input: "pipe:0", UploadID: "1", Threads: 4, Level: resconv.Level("651:490"), + Transcode: false, + WithAudio: true, }) - const expected = `"-threads 4 -i pipe:0 -c:a copy -c:v copy -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `"-threads 4 -i pipe:0 -c:a copy -c:v copy -f hls -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + + require.Contains(t, expected, strings.Join(rawCommand, " ")) +} + +func Test_BuildVideoCommand_Raw_Transcode(t *testing.T) { + var rawCommand = mediaconvert.BuildRawVideoCommand(&mediaconvert.Options{ + OutputDir: "test", + Input: "pipe:0", + UploadID: "1", + Threads: 4, + Level: resconv.Level("651:490"), + Transcode: true, + WithAudio: true, + }) + + const expected = `-threads 4 -i pipe:0 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(rawCommand, " ")) } diff --git a/internal/pkg/mediaconvert/coordinator.go b/internal/pkg/mediaconvert/coordinator.go index ede68b3b65..50c8846a76 100644 --- a/internal/pkg/mediaconvert/coordinator.go +++ b/internal/pkg/mediaconvert/coordinator.go @@ -57,7 +57,7 @@ func NewStreamCoordinator(ctx context.Context, c *config.Config) *StreamCoordina RetryDelay: time.Millisecond * 100, Timeout: c.Timeout, WorkerCount: uint32(c.MaxThreadCount), - RetryCount: 5, + RetryCount: 10, BufferSize: 128, Dir: c.OutputDir, }, @@ -100,6 +100,8 @@ func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo OutputDir: s.conf.OutputDir, Threads: s.conf.MaxThreadCount, UploadID: info.ID, + Transcode: true, + WithAudio: true, Level: level, ScalingLevels: scaling, } diff --git a/internal/pkg/mediaconvert/scheduler.go b/internal/pkg/mediaconvert/scheduler.go index 31507f1747..9506e54169 100644 --- a/internal/pkg/mediaconvert/scheduler.go +++ b/internal/pkg/mediaconvert/scheduler.go @@ -171,19 +171,29 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { return } + audioStream := probe.FirstAudioStream() + if audioStream == nil { + logger.Info("no audio stream found in the file", zap.String("filepath", sourceFilePath)) + } + videoStream := probe.FirstVideoStream() if videoStream == nil { logger.Error("no video stream found in the file", zap.String("filepath", sourceFilePath)) _ = os.RemoveAll(destinationFolder) return + } else { + logger.Debug("video stream found", zap.String("codec", videoStream.CodecName), zap.Int("width", videoStream.Width), zap.Int("height", videoStream.Height)) } var res = fmt.Sprintf("%v:%v", videoStream.Width, videoStream.Height) + var codec = videoStream.CodecName var level = resconv.Level(res) var opts = Options{ Input: sourceFilePath, OutputDir: p.cfg.OutputDir, Level: level, + Transcode: !IsHLSSupportedVideoCodec(codec), + WithAudio: audioStream != nil, ScalingLevels: append(resconv.SubLevels(res), level), UploadID: task.ID, Threads: p.cfg.MaxThreadCount, @@ -194,9 +204,10 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { Dir: destinationFolder, WorkerCount: uint32(opts.Threads), BufferSize: 128, - RetryCount: 5, + RetryCount: 10, RetryDelay: time.Millisecond * 100, Timeout: p.cfg.Timeout, + Source: task.Source, SourceFile: sourceFilePath, }) @@ -269,6 +280,15 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { } } +func IsHLSSupportedVideoCodec(codec string) bool { + switch codec { + case "h264", "h265": + return true + default: + return false + } +} + // IsSupportedMediaType checks whether transcoding is supported for given media type func IsSupportedMediaType(mediaType string) bool { // Explicitly disable conversion for video/mp2t and video/x-mpegurl @@ -277,6 +297,8 @@ func IsSupportedMediaType(mediaType string) bool { return false case "video/mp4": return true + case "video/webm": + return true default: return false } diff --git a/internal/pkg/mediaconvert/stream.go b/internal/pkg/mediaconvert/stream.go index db207e9f61..e41042d2e0 100644 --- a/internal/pkg/mediaconvert/stream.go +++ b/internal/pkg/mediaconvert/stream.go @@ -118,6 +118,7 @@ func (w *Stream) start(ctx context.Context, options *Options) error { if err := manifest.GenerateHLSPlaylist(append(options.ScalingLevels, options.Level), options.OutputDir, options.UploadID); err != nil { return err } + w.commandGroup.Add(1) go func() { defer w.commandGroup.Done() diff --git a/internal/pkg/mediaconvert/transcoder.go b/internal/pkg/mediaconvert/transcoder.go new file mode 100644 index 0000000000..99b9e62244 --- /dev/null +++ b/internal/pkg/mediaconvert/transcoder.go @@ -0,0 +1,234 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package mediaconvert + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/hcengineering/stream/internal/pkg/config" + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/manifest" + "github.com/hcengineering/stream/internal/pkg/resconv" + "github.com/hcengineering/stream/internal/pkg/storage" + "github.com/hcengineering/stream/internal/pkg/token" + "github.com/hcengineering/stream/internal/pkg/uploader" + "go.uber.org/zap" + "gopkg.in/vansante/go-ffprobe.v2" +) + +// Transcoder process one transcoding task +type Transcoder struct { + ctx context.Context + cfg *config.Config + logger *zap.Logger +} + +// NewTranscoder creates a new instance of task transcoder +func NewTranscoder(ctx context.Context, cfg *config.Config) *Transcoder { + var p = &Transcoder{ + cfg: cfg, + ctx: ctx, + logger: log.FromContext(ctx).With(zap.String("transcoding", "transcoder")), + } + + return p +} + +// TODO: add a factory pattern to process tasks by different media type +func (p *Transcoder) Transcode(ctx context.Context, task *Task) { + var logger = p.logger.With(zap.String("task-id", task.ID)) + + logger.Debug("start") + defer logger.Debug("finished") + + logger.Debug("phase 1: get a token") + var tokenString, err = token.NewToken(p.cfg.ServerSecret, task.Workspace, "stream", "datalake") + if err != nil { + logger.Error("can not create token", zap.Error(err)) + return + } + + logger.Debug("phase 2: preparing fs") + var destinationFolder = filepath.Join(p.cfg.OutputDir, task.ID) + var _, filename = filepath.Split(task.Source) + err = os.MkdirAll(destinationFolder, os.ModePerm) + if err != nil { + logger.Error("can not create temporary folder", zap.Error(err)) + return + } + + defer func() { + if err = os.RemoveAll(destinationFolder); err != nil { + logger.Error("failed to cleanup temporary folder", zap.Error(err)) + } + }() + + logger.Debug("phase 3: get the remote file") + + remoteStorage, err := storage.NewStorageByURL(ctx, p.cfg.Endpoint(), p.cfg.EndpointURL.Scheme, tokenString, task.Workspace) + + if err != nil { + logger.Error("can not create storage by url", zap.Error(err), zap.String("url", p.cfg.EndpointURL.String())) + _ = os.RemoveAll(destinationFolder) + return + } + + stat, err := remoteStorage.StatFile(ctx, task.Source) + if err != nil { + logger.Error("can not stat a file", zap.Error(err), zap.String("filepath", task.Source)) + _ = os.RemoveAll(destinationFolder) + return + } + + if !IsSupportedMediaType(stat.Type) { + logger.Info("unsupported media type", zap.String("type", stat.Type)) + _ = os.RemoveAll(destinationFolder) + return + } + + // sourceFilePath := remoteStorage.GetFileURL(ctx, task.Source) + // if sourceFilePath == "" { + sourceFilePath := filepath.Join(destinationFolder, filename) + if err = remoteStorage.GetFile(ctx, task.Source, sourceFilePath); err != nil { + logger.Error("can not download a file", zap.Error(err), zap.String("filepath", task.Source)) + _ = os.RemoveAll(destinationFolder) + // TODO: reschedule + return + } + // } + + logger.Debug("phase 4: prepare to transcode") + probe, err := ffprobe.ProbeURL(ctx, sourceFilePath) + if err != nil { + logger.Error("can not get probe for a file", zap.Error(err), zap.String("filepath", sourceFilePath)) + _ = os.RemoveAll(destinationFolder) + return + } + + videoStream := probe.FirstVideoStream() + if videoStream == nil { + logger.Error("no video stream found in the file", zap.String("filepath", sourceFilePath)) + _ = os.RemoveAll(destinationFolder) + return + } else { + logger.Debug("video stream found", zap.String("codec", videoStream.CodecName), zap.Int("width", videoStream.Width), zap.Int("height", videoStream.Height)) + } + + audioStream := probe.FirstAudioStream() + if audioStream == nil { + logger.Info("no audio stream found in the file", zap.String("filepath", sourceFilePath)) + } + + var res = fmt.Sprintf("%v:%v", videoStream.Width, videoStream.Height) + var codec = videoStream.CodecName + var level = resconv.Level(res) + var sublevels = resconv.SubLevels(res) + var opts = Options{ + Input: sourceFilePath, + OutputDir: p.cfg.OutputDir, + Level: level, + Transcode: !IsHLSSupportedVideoCodec(codec), + WithAudio: audioStream != nil, + ScalingLevels: append(sublevels, level), + UploadID: task.ID, + Threads: p.cfg.MaxThreadCount, + } + + logger.Debug("phase 5: start async upload process") + var uploader = uploader.New(ctx, remoteStorage, uploader.Options{ + Dir: destinationFolder, + WorkerCount: uint32(opts.Threads), + BufferSize: 128, + RetryCount: 10, + RetryDelay: time.Millisecond * 100, + Timeout: p.cfg.Timeout, + Source: task.Source, + SourceFile: sourceFilePath, + }) + + err = manifest.GenerateHLSPlaylist(opts.ScalingLevels, p.cfg.OutputDir, opts.UploadID) + if err != nil { + logger.Error("can not generate hls playlist", zap.String("out", p.cfg.OutputDir), zap.String("uploadID", opts.UploadID)) + _ = os.RemoveAll(destinationFolder) + return + } + + go uploader.Start() + + logger.Debug("phase 6: start async transcode processes") + + var argsSlice = [][]string{ + BuildThumbnailCommand(&opts), + BuildRawVideoCommand(&opts), + BuildScalingVideoCommand(&opts), + } + var cmds []*exec.Cmd + + for _, args := range argsSlice { + cmd, cmdErr := newFfmpegCommand(ctx, nil, args) + if cmdErr != nil { + logger.Error("can not create a new command", zap.Error(cmdErr), zap.Strings("args", args)) + go uploader.Cancel() + return + } + cmds = append(cmds, cmd) + if err = cmd.Start(); err != nil { + logger.Error("can not start a command", zap.Error(err), zap.Strings("args", args)) + go uploader.Cancel() + return + } + } + + logger.Debug("phase 7: wait for the result") + for _, cmd := range cmds { + if err = cmd.Wait(); err != nil { + logger.Error("can not wait for command end ", zap.Error(err)) + go uploader.Cancel() + return + } + } + + logger.Debug("phase 8: schedule cleanup") + go uploader.Stop() + + logger.Debug("phase 9: try to set metadata") + + if metaProvider, ok := remoteStorage.(storage.MetaProvider); ok { + var hls = HLS{ + Width: videoStream.Width, + Height: videoStream.Height, + Source: task.ID + "_master.m3u8", + Thumbnail: task.ID + ".jpg", + } + + logger.Debug("applying metadata", zap.String("url", hls.Source), zap.String("thumbnail", hls.Thumbnail), zap.String("source", task.Source)) + err = metaProvider.PatchMeta( + ctx, + task.Source, + &storage.Metadata{ + "hls": hls, + }, + ) + if err != nil { + logger.Error("can not patch the source file", zap.Error(err)) + } + } +} diff --git a/internal/pkg/queue/config.go b/internal/pkg/queue/config.go new file mode 100644 index 0000000000..df2b2c9a7c --- /dev/null +++ b/internal/pkg/queue/config.go @@ -0,0 +1,43 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package queue + +import "strings" + +// Config reprents queue config +type Config struct { + Postfix string + Brokers []string + ClientID string + Region string +} + +func ParseConfig(config, clientID, region string) Config { + var parts = strings.Split(config, ";") + var brokers = strings.Split(parts[0], ",") + var postfix = "" + + if len(parts) > 1 { + postfix = parts[1] + } + + var cfg = Config{ + Postfix: postfix, + Brokers: brokers, + ClientID: clientID, + Region: region, + } + + return cfg +} diff --git a/internal/pkg/queue/config_test.go b/internal/pkg/queue/config_test.go new file mode 100644 index 0000000000..4040db1b52 --- /dev/null +++ b/internal/pkg/queue/config_test.go @@ -0,0 +1,90 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package queue_test + +import ( + "reflect" + "testing" + + "github.com/hcengineering/stream/internal/pkg/queue" +) + +func Test_ParseConfig(t *testing.T) { + tests := []struct { + name string + config string + clientId string + region string + want queue.Config + }{ + { + name: "Basic broker configuration", + config: "broker1:9092,broker2:9092", + clientId: "test-client", + region: "us-west", + want: queue.Config{ + Postfix: "", + Brokers: []string{"broker1:9092", "broker2:9092"}, + ClientID: "test-client", + Region: "us-west", + }, + }, + { + name: "Broker configuration with postfix", + config: "broker1:9092,broker2:9092;custom-postfix", + clientId: "test-client", + region: "eu-central", + want: queue.Config{ + Postfix: "custom-postfix", + Brokers: []string{"broker1:9092", "broker2:9092"}, + ClientID: "test-client", + Region: "eu-central", + }, + }, + { + name: "Single broker configuration", + config: "broker1:9092", + clientId: "single-broker-client", + region: "asia-east", + want: queue.Config{ + Postfix: "", + Brokers: []string{"broker1:9092"}, + ClientID: "single-broker-client", + Region: "asia-east", + }, + }, + { + name: "Multiple semicolons in config", + config: "broker1:9092,broker2:9092;postfix;extra", + clientId: "test-client", + region: "us-east", + want: queue.Config{ + Postfix: "postfix", + Brokers: []string{"broker1:9092", "broker2:9092"}, + ClientID: "test-client", + Region: "us-east", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := queue.ParseConfig(tt.config, tt.clientId, tt.region) + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseConfig() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/pkg/queue/queue.go b/internal/pkg/queue/queue.go new file mode 100644 index 0000000000..7eae483927 --- /dev/null +++ b/internal/pkg/queue/queue.go @@ -0,0 +1,162 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package queue provides access to kafka queue +package queue + +import ( + "context" + "encoding/json" + "time" + + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/segmentio/kafka-go" + "go.uber.org/zap" +) + +const TopicTranscodeRequest = "stream.transcode.request" +const TopicTranscodeResult = "stream.transcode.result" + +type TranscodeRequest struct { + BlobID string + WorkspaceUUID string + ContentType string +} + +type TranscodeResult struct { + BlobID string + WorkspaceUUID string +} + +type Consumer interface { + // Heartbeat(ctx context.Context) error + Read(ctx context.Context) (kafka.Message, error) + Fetch(ctx context.Context) (kafka.Message, error) + Commit(ctx context.Context, msg kafka.Message) error + Close() error +} + +type Producer interface { + Send(ctx context.Context, workspaceID string, data any) error + Close() error +} + +type TConsumer struct { + topic string + reader *kafka.Reader +} + +type TProducer struct { + topic string + writer *kafka.Writer +} + +type Logger struct { + logger *zap.Logger +} + +var _ Producer = (*TProducer)(nil) +var _ Consumer = (*TConsumer)(nil) +var _ kafka.Logger = (*Logger)(nil) + +// NewLogger creates new kafka.Logger +func NewLogger(ctx context.Context) kafka.Logger { + return &Logger{ + logger: log.FromContext(ctx), + } +} + +func (l *Logger) Printf(msg string, args ...any) { + l.logger.Sugar().Debugf(msg, args...) +} + +// NewConsumer creates a new transcoding request consumer +func NewConsumer(ctx context.Context, topic, group string, config Config) Consumer { + platformTopic := makeTopicID(topic, config) + groupID := makeGroupID(group, topic, config) + + reader := kafka.NewReader(kafka.ReaderConfig{ + Brokers: config.Brokers, + GroupID: groupID, + Topic: platformTopic, + Logger: NewLogger(ctx), + HeartbeatInterval: 1 * time.Second, + SessionTimeout: 10 * time.Second, + }) + + return &TConsumer{ + topic: platformTopic, + reader: reader, + } +} + +func (c *TConsumer) Read(ctx context.Context) (kafka.Message, error) { + return c.reader.ReadMessage(ctx) +} + +func (c *TConsumer) Fetch(ctx context.Context) (kafka.Message, error) { + return c.reader.FetchMessage(ctx) +} + +func (c *TConsumer) Commit(ctx context.Context, msg kafka.Message) error { + return c.reader.CommitMessages(ctx, msg) +} + +func (c *TConsumer) Close() error { + return c.reader.Close() +} + +// NewProducer creates a new transcoding result producer +func NewProducer(ctx context.Context, topic string, config Config) Producer { + platformTopic := makeTopicID(topic, config) + writer := &kafka.Writer{ + Addr: kafka.TCP(config.Brokers...), + Topic: platformTopic, + AllowAutoTopicCreation: true, + Logger: NewLogger(ctx), + } + return &TProducer{ + topic: platformTopic, + writer: writer, + } +} + +func (p *TProducer) Send(ctx context.Context, workspaceID string, data any) error { + value, err := json.Marshal(data) + if err != nil { + return err + } + + return p.writer.WriteMessages( + ctx, + kafka.Message{ + Key: []byte(workspaceID), + Value: value, + }, + ) +} + +func (p *TProducer) Close() error { + return p.writer.Close() +} + +func makeGroupID(groupID, topic string, config Config) string { + return makeTopicID(topic, config) + "-" + groupID +} + +func makeTopicID(topic string, config Config) string { + if config.Region != "" { + return config.Region + "." + topic + config.Postfix + } + return topic + config.Postfix +} diff --git a/internal/pkg/queue/worker.go b/internal/pkg/queue/worker.go new file mode 100644 index 0000000000..8925d024da --- /dev/null +++ b/internal/pkg/queue/worker.go @@ -0,0 +1,138 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package queue + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/hcengineering/stream/internal/pkg/config" + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/mediaconvert" + "github.com/pkg/errors" + "github.com/segmentio/kafka-go" + "go.uber.org/zap" +) + +type Worker struct { + logger *zap.Logger + cfg *config.Config + consumer Consumer +} + +func NewWorker(ctx context.Context, consumer Consumer, cfg *config.Config) *Worker { + logger := log.FromContext(ctx).With(zap.String("queue", "worker")) + + return &Worker{ + logger: logger, + cfg: cfg, + consumer: consumer, + } +} + +func (w *Worker) Start(ctx context.Context) error { + w.logger.Info("starting worker") + defer w.logger.Info("worker stopped") + + for { + select { + case <-ctx.Done(): + w.logger.Info("worker shutdown requested") + return ctx.Err() + default: + if err := w.fetchAndProcessMessage(ctx); err != nil { + if errors.Is(err, context.Canceled) { + return err + } + w.logger.Error("failed to process message", zap.Error(err)) + } + } + } +} + +func (w *Worker) fetchAndProcessMessage(ctx context.Context) error { + // TODO We should use Fetch and Commit here but as long processing + // time is longer than the max poll interval, we have to use Read. + // msg, err := w.consumer.Fetch(ctx) + commit := false + msg, err := w.consumer.Read(ctx) + if err != nil { + return fmt.Errorf("read message: %w", err) + } + + logger := w.logger.With( + zap.String("message-key", string(msg.Key)), + zap.Int64("message-offset", msg.Offset), + zap.Int("partition", msg.Partition), + ) + logger.Debug("message", zap.ByteString("message", msg.Value)) + + _, err = w.processMessage(ctx, msg, logger) + if err != nil { + w.logger.Error("failed to process message", zap.Error(err)) + } + + if commit { + if err := w.commitMessage(ctx, msg, logger); err != nil { + return errors.Wrapf(err, "failed to commit message") + } + } + + return err +} + +func (w *Worker) commitMessage(ctx context.Context, msg kafka.Message, logger *zap.Logger) error { + maxRetries := 3 + for i := range maxRetries { + err := w.consumer.Commit(ctx, msg) + if err == nil { + return nil + } + + if i < maxRetries-1 { + logger.Error("failed to commit message", zap.Error(err), zap.Int("attempt", i+1)) + time.Sleep(time.Duration(100*(i+1)) * time.Millisecond) + } else { + return err + } + } + + return nil +} + +func (w *Worker) processMessage(ctx context.Context, msg kafka.Message, logger *zap.Logger) (bool, error) { + var req TranscodeRequest + if err := json.Unmarshal(msg.Value, &req); err != nil { + return true, fmt.Errorf("parse failed: %w", err) + } + + if mediaconvert.IsSupportedMediaType(req.ContentType) == false { + logger.Debug("unsupported content type", zap.String("contentType", req.ContentType)) + return true, nil + } + + task := mediaconvert.Task{ + ID: req.BlobID, + Workspace: req.WorkspaceUUID, + Source: req.BlobID, + } + + transcoder := mediaconvert.NewTranscoder(ctx, w.cfg) + transcoder.Transcode(ctx, &task) + + return true, nil +} diff --git a/internal/pkg/resconv/resconv.go b/internal/pkg/resconv/resconv.go index 3ed4a0b3c2..97d716f3f0 100644 --- a/internal/pkg/resconv/resconv.go +++ b/internal/pkg/resconv/resconv.go @@ -26,14 +26,16 @@ const defaultLevel = "360p" var prefixes = []struct { pixels int + width int + height int label string }{ - {pixels: 640 * 360, label: "360p"}, - {pixels: 1280 * 720, label: "480p"}, - {pixels: 1920 * 1080, label: "720p"}, - {pixels: 2560 * 1440, label: "1080p"}, - {pixels: 3840 * 2160, label: "1440p"}, - {pixels: 7680 * 4320, label: "2160p"}, + {pixels: 640 * 360, width: 640, height: 360, label: "360p"}, + {pixels: 1280 * 720, width: 720, height: 720, label: "480p"}, + {pixels: 1920 * 1080, width: 1920, height: 1080, label: "720p"}, + {pixels: 2560 * 1440, width: 2560, height: 1440, label: "1080p"}, + {pixels: 3840 * 2160, width: 3840, height: 2160, label: "1440p"}, + {pixels: 7680 * 4320, width: 7680, height: 4320, label: "2160p"}, } var bandwidthMap = map[string]int{ @@ -41,7 +43,7 @@ var bandwidthMap = map[string]int{ "480p": 2000000, "720p": 5000000, "1080p": 8000000, - "1440p": 16000000, + "1440p": 12000000, "2160p": 25000000, "4320p": 50000000, } @@ -58,9 +60,9 @@ var resolutions = map[string]string{ // SubLevels returns sublevels for the resolution func SubLevels(resolution string) (res []string) { - var pixels = Pixels(resolution) + var height = Height(resolution) var idx = sort.Search(len(prefixes), func(i int) bool { - return pixels < prefixes[i].pixels + return height < prefixes[i].height }) if idx < 2 { return res @@ -90,9 +92,9 @@ func Resolution(level string) string { // Level converts the resolution to short prefix func Level(resolution string) string { - var pixels = Pixels(resolution) + var height = Height(resolution) idx := sort.Search(len(prefixes), func(i int) bool { - return pixels < prefixes[i].pixels + return height < prefixes[i].height }) if idx == len(prefixes) { return "4320p" @@ -101,10 +103,20 @@ func Level(resolution string) string { return prefixes[idx].label } +func Height(resolution string) int { + var parts = strings.Split(resolution, ":") + var h = 240 + if len(parts) > 1 { + var _h, _ = strconv.Atoi(parts[1]) + h = max(h, _h) + } + return h +} + // Pixels returns amount of pixels for the resolution func Pixels(resolution string) int { var parts = strings.Split(resolution, ":") - var w, h = 420, 240 + var w, h = 320, 240 if len(parts) > 1 { var _w, _ = strconv.Atoi(parts[0]) diff --git a/internal/pkg/resconv/resconv_test.go b/internal/pkg/resconv/resconv_test.go index 553beb6d05..e2b187ef01 100644 --- a/internal/pkg/resconv/resconv_test.go +++ b/internal/pkg/resconv/resconv_test.go @@ -20,21 +20,26 @@ import ( "github.com/stretchr/testify/require" ) -func Test_Resconv_ShouldReturnCorrectPrefix(t *testing.T) { +func Test_Resconv_Level(t *testing.T) { tests := []struct { res string expected string }{ {res: "320:240", expected: "360p"}, - {res: "320:360", expected: "360p"}, + {res: "320:360", expected: "480p"}, {res: "640:480", expected: "480p"}, {res: "1280:720", expected: "720p"}, {res: "1920:1080", expected: "1080p"}, + {res: "1600:1080", expected: "1080p"}, + {res: "2160:1080", expected: "1080p"}, {res: "2880:1800", expected: "1440p"}, {res: "2560:1440", expected: "1440p"}, {res: "3840:2160", expected: "2160p"}, {res: "5120:2880", expected: "2160p"}, - {res: "9000:4000", expected: "4320p"}, + {res: "480:720", expected: "720p"}, + {res: "720:1080", expected: "1080p"}, + {res: "1440:2560", expected: "2160p"}, + {res: "9000:4000", expected: "2160p"}, } for _, tt := range tests { @@ -45,7 +50,7 @@ func Test_Resconv_ShouldReturnCorrectPrefix(t *testing.T) { } } -func Test_Resconv_ShouldReturnCorrectPrefixes(t *testing.T) { +func Test_Resconv_SubLevels(t *testing.T) { tests := []struct { name string res string @@ -116,6 +121,11 @@ func Test_Resconv_ShouldReturnCorrectPrefixes(t *testing.T) { res: "7681:4320", expected: []string{"1080p", "720p"}, }, + { + name: "vertical video", + res: "1440:2560", + expected: []string{"1080p", "720p"}, + }, } for _, tt := range tests { @@ -125,3 +135,36 @@ func Test_Resconv_ShouldReturnCorrectPrefixes(t *testing.T) { }) } } + +func Test_Resconv_Height(t *testing.T) { + tests := []struct { + res string + expected int + }{ + // 3x2 + {res: "320:240", expected: 240}, + {res: "640:480", expected: 480}, + {res: "1280:720", expected: 720}, + // 16x9 + {res: "1920:1080", expected: 1080}, + {res: "2560:1440", expected: 1440}, + {res: "3840:2160", expected: 2160}, + // Vertical + {res: "480:720", expected: 720}, + {res: "720:1080", expected: 1080}, + {res: "1440:2560", expected: 2560}, + // Custom + {res: "1600:1080", expected: 1080}, + {res: "1930:1090", expected: 1090}, + {res: "2160:1080", expected: 1080}, + {res: "5120:2880", expected: 2880}, + {res: "9000:4000", expected: 4000}, + } + + for _, tt := range tests { + t.Run(tt.res, func(t *testing.T) { + height := resconv.Height(tt.res) + require.Equal(t, tt.expected, height, "Height(%s)", tt.res) + }) + } +} diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index f3b0132487..d470bb6bf8 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -24,6 +24,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/hcengineering/stream/internal/pkg/log" "github.com/pkg/errors" @@ -31,6 +32,11 @@ import ( "go.uber.org/zap" ) +type uploadResult struct { + key string `json:"key"` + error string `json:"error"` +} + // DatalakeStorage represents datalake storage type DatalakeStorage struct { baseURL string @@ -47,6 +53,10 @@ func NewDatalakeStorage(ctx context.Context, baseURL, workspace, token string) S token: token, workspace: workspace, logger: log.FromContext(ctx).With(zap.String("storage", "datalake")), + client: fasthttp.Client{ + MaxIdleConnDuration: 5 * time.Second, + MaxConnsPerHost: 100, + }, } } @@ -66,7 +76,7 @@ func createFormFile(writer *multipart.Writer, fieldname, filename, contentType s return writer.CreatePart(h) } -func getObjectKey(s string) string { +func getObjectKeyFromPath(s string) string { var _, objectKey = filepath.Split(s) return objectKey } @@ -82,7 +92,7 @@ func (d *DatalakeStorage) PutFile(ctx context.Context, fileName string) error { _ = file.Close() }() - var objectKey = getObjectKey(fileName) + var objectKey = getObjectKeyFromPath(fileName) var logger = d.logger.With(zap.String("upload", d.workspace), zap.String("fileName", fileName)) logger.Debug("start") @@ -118,10 +128,21 @@ func (d *DatalakeStorage) PutFile(ctx context.Context, fileName string) error { req.SetBody(body.Bytes()) if err := d.client.Do(req, res); err != nil { - logger.Error("upload failed", zap.Error(err)) + logRequestError(logger, err, "upload failed", res) return errors.Wrapf(err, "upload failed") } + var result []uploadResult + if err := json.Unmarshal(res.Body(), &result); err != nil { + return errors.Wrapf(err, "parse error") + } + + for _, res := range result { + if res.error != "" { + return fmt.Errorf("upload error: %v", res.error) + } + } + logger.Debug("uploaded") return nil @@ -132,7 +153,7 @@ func (d *DatalakeStorage) DeleteFile(ctx context.Context, fileName string) error var logger = d.logger.With(zap.String("delete", d.workspace), zap.String("fileName", fileName)) logger.Debug("start") - var objectKey = getObjectKey(fileName) + var objectKey = getObjectKeyFromPath(fileName) req := fasthttp.AcquireRequest() defer fasthttp.ReleaseRequest(req) @@ -145,7 +166,7 @@ func (d *DatalakeStorage) DeleteFile(ctx context.Context, fileName string) error req.Header.Add("Authorization", "Bearer "+d.token) if err := d.client.Do(req, res); err != nil { - logger.Error("delete failed", zap.Error(err)) + logRequestError(logger, err, "delete failed", res) return errors.Wrapf(err, "delete failed") } @@ -160,7 +181,7 @@ func (d *DatalakeStorage) PatchMeta(ctx context.Context, filename string, md *Me logger.Debug("start") defer logger.Debug("finished") - var objectKey = getObjectKey(filename) + var objectKey = getObjectKeyFromPath(filename) req := fasthttp.AcquireRequest() defer fasthttp.ReleaseRequest(req) @@ -180,6 +201,7 @@ func (d *DatalakeStorage) PatchMeta(ctx context.Context, filename string, md *Me defer fasthttp.ReleaseResponse(resp) if err := d.client.Do(req, resp); err != nil { + logRequestError(logger, err, "request failed", resp) return err } @@ -194,12 +216,18 @@ func (d *DatalakeStorage) PatchMeta(ctx context.Context, filename string, md *Me return nil } +// GetFileURL returns blob URL +func (d *DatalakeStorage) GetFileURL(ctx context.Context, filename string) string { + objectKey := getObjectKeyFromPath(filename) + return d.baseURL + "/blob/" + d.workspace + "/" + objectKey +} + // GetMeta gets metadata related to the object func (d *DatalakeStorage) GetMeta(ctx context.Context, filename string) (*Metadata, error) { var logger = d.logger.With(zap.String("get meta", d.workspace), zap.String("fileName", filename)) logger.Debug("start") - var objectKey = getObjectKey(filename) + var objectKey = getObjectKeyFromPath(filename) req := fasthttp.AcquireRequest() defer fasthttp.ReleaseRequest(req) @@ -211,6 +239,7 @@ func (d *DatalakeStorage) GetMeta(ctx context.Context, filename string) (*Metada defer fasthttp.ReleaseResponse(resp) if err := d.client.Do(req, resp); err != nil { + logRequestError(logger, err, "request failed", resp) return nil, err } @@ -232,17 +261,19 @@ func (d *DatalakeStorage) GetFile(ctx context.Context, filename, destination str var logger = d.logger.With(zap.String("get", d.workspace), zap.String("fileName", filename), zap.String("destination", destination)) logger.Debug("start") - var objectKey = getObjectKey(filename) + var objectKey = getObjectKeyFromPath(filename) req := fasthttp.AcquireRequest() defer fasthttp.ReleaseRequest(req) req.SetRequestURI(d.baseURL + "/blob/" + d.workspace + "/" + objectKey) req.Header.SetMethod(fasthttp.MethodGet) + req.Header.Add("Authorization", "Bearer "+d.token) resp := fasthttp.AcquireResponse() defer fasthttp.ReleaseResponse(resp) if err := d.client.Do(req, resp); err != nil { + logRequestError(logger, err, "request failed", resp) return err } @@ -276,24 +307,26 @@ func (d *DatalakeStorage) StatFile(ctx context.Context, filename string) (*BlobI var logger = d.logger.With(zap.String("head", d.workspace), zap.String("fileName", filename)) logger.Debug("start") - var objectKey = getObjectKey(filename) + var objectKey = getObjectKeyFromPath(filename) req := fasthttp.AcquireRequest() defer fasthttp.ReleaseRequest(req) req.SetRequestURI(d.baseURL + "/blob/" + d.workspace + "/" + objectKey) req.Header.SetMethod(fasthttp.MethodHead) + req.Header.Add("Authorization", "Bearer "+d.token) resp := fasthttp.AcquireResponse() defer fasthttp.ReleaseResponse(resp) if err := d.client.Do(req, resp); err != nil { + logRequestError(logger, err, "request failed", resp) return nil, err } // Check the response status code if resp.StatusCode() != fasthttp.StatusOK { var err = fmt.Errorf("unexpected status code: %d", resp.StatusCode()) - logger.Debug("bad status code", zap.Error(err)) + logRequestError(logger, err, "bad status code", resp) return nil, err } @@ -308,12 +341,12 @@ func (d *DatalakeStorage) StatFile(ctx context.Context, filename string) (*BlobI // SetParent updates blob parent reference func (d *DatalakeStorage) SetParent(ctx context.Context, filename, parent string) error { - var logger = d.logger.With(zap.String("parent", d.workspace), zap.String("fileName", filename), zap.String("parent", parent)) + var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("fileName", filename), zap.String("parent", parent)) logger.Debug("start") - var objectKey = getObjectKey(filename) - var parentKey = getObjectKey(parent) + var objectKey = getObjectKeyFromPath(filename) + var parentKey = getObjectKeyFromPath(parent) req := fasthttp.AcquireRequest() defer fasthttp.ReleaseRequest(req) @@ -335,6 +368,7 @@ func (d *DatalakeStorage) SetParent(ctx context.Context, filename, parent string defer fasthttp.ReleaseResponse(resp) if err := d.client.Do(req, resp); err != nil { + logRequestError(logger, err, "request failed", resp) return err } @@ -342,7 +376,7 @@ func (d *DatalakeStorage) SetParent(ctx context.Context, filename, parent string var statusOK = resp.StatusCode() >= 200 && resp.StatusCode() < 300 if !statusOK { var err = fmt.Errorf("unexpected status code: %d", resp.StatusCode()) - logger.Debug("bad status code", zap.Error(err)) + logger.Debug("bad status code", zap.Error(err), zap.Int("status", resp.StatusCode()), zap.String("response", resp.String())) return err } @@ -351,5 +385,15 @@ func (d *DatalakeStorage) SetParent(ctx context.Context, filename, parent string return nil } +func logRequestError(logger *zap.Logger, err error, msg string, res *fasthttp.Response) { + logger.Error( + msg, + zap.Error(err), + zap.Int("status", res.StatusCode()), + zap.String("headers", res.Header.String()), + zap.String("response", res.String()), + ) +} + var _ Storage = (*DatalakeStorage)(nil) var _ MetaProvider = (*DatalakeStorage)(nil) diff --git a/internal/pkg/storage/s3.go b/internal/pkg/storage/s3.go index e4f3050d09..254cb9d723 100644 --- a/internal/pkg/storage/s3.go +++ b/internal/pkg/storage/s3.go @@ -165,6 +165,11 @@ func (u *S3Storage) GetFile(ctx context.Context, filename, dest string) error { return nil } +// GetFileURL returns blob URL +func (u *S3Storage) GetFileURL(ctx context.Context, filename string) string { + return "" +} + // StatFile gets file stat from the storage func (u *S3Storage) StatFile(ctx context.Context, filename string) (*BlobInfo, error) { var logger = u.logger.With(zap.String("head", u.bucketName), zap.String("fileName", filename)) diff --git a/internal/pkg/storage/storage.go b/internal/pkg/storage/storage.go index 0feb587825..8f7cfbe3f1 100644 --- a/internal/pkg/storage/storage.go +++ b/internal/pkg/storage/storage.go @@ -41,6 +41,7 @@ type BlobInfo struct { type Storage interface { PutFile(ctx context.Context, fileName string) error DeleteFile(ctx context.Context, fileName string) error + GetFileURL(ctx context.Context, fileName string) string GetFile(ctx context.Context, fileName, destination string) error StatFile(ctx context.Context, fileName string) (*BlobInfo, error) SetParent(ctx context.Context, fileName string, parentName string) error diff --git a/internal/pkg/uploader/options.go b/internal/pkg/uploader/options.go index 97ba5cc508..d3642cca6a 100644 --- a/internal/pkg/uploader/options.go +++ b/internal/pkg/uploader/options.go @@ -23,5 +23,6 @@ type Options struct { RetryCount int RetryDelay time.Duration Timeout time.Duration + Source string SourceFile string } diff --git a/internal/pkg/uploader/uploader.go b/internal/pkg/uploader/uploader.go index ad1798fa00..d605dfe0c2 100644 --- a/internal/pkg/uploader/uploader.go +++ b/internal/pkg/uploader/uploader.go @@ -16,11 +16,13 @@ package uploader import ( "context" + "errors" "hash/fnv" "os" "path/filepath" "strings" "sync" + "syscall" "time" "unsafe" @@ -113,9 +115,12 @@ func (u *uploaderImpl) scanInitialFiles() { go func() { defer u.workerWaitGroup.Done() + logger := u.logger.With(zap.String("dir", u.options.Dir)) + + logger.Info("initial file scan") initFiles, err := os.ReadDir(u.options.Dir) if err != nil { - u.logger.Error("failed to read initial files", zap.Error(err), zap.String("dir", u.options.Dir)) + logger.Error("failed to read initial files", zap.Error(err)) return } @@ -124,6 +129,7 @@ func (u *uploaderImpl) scanInitialFiles() { continue } + // Ignore source file var filePath = filepath.Join(u.options.Dir, f.Name()) if filePath == u.options.SourceFile { continue @@ -131,7 +137,7 @@ func (u *uploaderImpl) scanInitialFiles() { u.filesCh <- filePath } - u.logger.Info("initial file scan complete", zap.String("dir", u.options.Dir), zap.Int("count", len(initFiles))) + logger.Info("initial file scan complete", zap.Int("count", len(initFiles))) }() } @@ -226,27 +232,39 @@ func (u *uploaderImpl) deleteRemoteFile(f string) { logger.Error("attempt failed", zap.Error(err)) } else { logger.Debug("file deleted in remote storage") - break + return } time.Sleep(u.options.RetryDelay) } - u.logger.Error("can not delete remote file") + logger.Error("can not delete remote file") } func (u *uploaderImpl) uploadAndDelete(f string) { var logger = u.logger.With(zap.String("upload and delete", f)) logger.Debug("uploading file") - var _, ok = u.sentFiles.Load(f) + if f == u.options.Dir { + return + } + // Check if the file exists + _, err := os.Stat(f) + if err != nil { + logger.Debug("file does not exist") + return + } + + // Check if the file has already been uploaded + var _, ok = u.sentFiles.Load(f) if ok && !u.shouldDeleteOnStop(f) { logger.Debug("file already uploaded") return } - for range u.options.RetryCount { + for attempt := range u.options.RetryCount { + logger = logger.With(zap.Int("attempt", attempt)) var ctx, cancel = context.WithTimeout(u.uploadCtx, u.options.Timeout) var err = u.storage.PutFile(ctx, f) cancel() @@ -259,16 +277,16 @@ func (u *uploaderImpl) uploadAndDelete(f string) { logger.Debug("file uploaded") // Update the file's parent if SourceFile is set - if u.options.SourceFile != "" { - err = u.storage.SetParent(ctx, f, u.options.SourceFile) + if u.options.Source != "" { + err = u.storage.SetParent(ctx, f, u.options.Source) if err != nil { - logger.Error("can not set blob parent", zap.Error(err), zap.String("filename", f), zap.String("source", u.options.SourceFile)) + logger.Error("can not set blob parent", zap.Error(err), zap.String("filename", f), zap.String("source", u.options.Source)) } } // Delete the file locally if it should be deleted if !u.shouldDeleteOnStop(f) { - if err := os.Remove(f); err != nil { + if err := os.Remove(f); err != nil && !errors.Is(err, syscall.ENOENT) { logger.Error("failed to remove file locally", zap.Error(err), zap.String("file", f)) } else { logger.Debug("removed file locally") @@ -321,10 +339,13 @@ func (u *uploaderImpl) startWatch(ready chan<- struct{}) { if !strings.Contains(event.Name, u.options.Dir) { continue } + if event.Name == u.options.Dir { + continue + } if event.Name == u.options.SourceFile { continue } - if strings.HasSuffix(event.Name, "tmp") { + if strings.HasSuffix(event.Name, ".tmp") { continue } logger.Debug("received an event", zap.String("event", event.Name), zap.Uint32("mask", event.Mask)) From cf07e88e400148eb1d1959e634b9df4ced90626f Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 11 Jun 2025 13:34:05 +0700 Subject: [PATCH 31/52] fix: add .gitignore Signed-off-by: Alexander Onnikov --- .gitignore | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..643a5c9e89 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +# IntelliJ +.idea + +# VSCode +.vscode + +# Binary +stream From aec074f91c5a3971b3a38e58c39e6ab695116800 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 11 Jun 2025 13:37:23 +0700 Subject: [PATCH 32/52] fix build Signed-off-by: Alexander Onnikov --- cmd/stream/main.go | 4 +- internal/pkg/mediaconvert/command.go | 29 +++++--------- internal/pkg/mediaconvert/command_test.go | 2 - internal/pkg/mediaconvert/coordinator.go | 1 - internal/pkg/mediaconvert/scheduler.go | 6 +-- internal/pkg/mediaconvert/transcoder.go | 7 ++-- internal/pkg/queue/config.go | 1 + internal/pkg/queue/config_test.go | 12 +++--- internal/pkg/queue/queue.go | 18 ++++++++- internal/pkg/queue/worker.go | 49 ++++++----------------- internal/pkg/resconv/resconv.go | 1 + internal/pkg/storage/datalake.go | 6 +-- internal/pkg/uploader/uploader.go | 2 +- 13 files changed, 60 insertions(+), 78 deletions(-) diff --git a/cmd/stream/main.go b/cmd/stream/main.go index 0edc84ab52..17e2b6fe12 100644 --- a/cmd/stream/main.go +++ b/cmd/stream/main.go @@ -108,7 +108,7 @@ func main() { config := queue.ParseConfig(cfg.QueueConfig, "stream", cfg.Region) logger.Info("using queue config", zap.Any("config", config)) consumer := queue.NewConsumer(ctx, queue.TopicTranscodeRequest, "stream", config) - // producer := queue.NewProducer(queue.TopicTranscodeResult, config) + producer := queue.NewProducer(ctx, queue.TopicTranscodeResult, config) wg.Add(1) go func() { @@ -117,7 +117,7 @@ func main() { logger.Info("queue worker started") defer logger.Info("queue worker finished") - worker := queue.NewWorker(ctx, consumer, cfg) + worker := queue.NewWorker(ctx, consumer, producer, cfg) if err := worker.Start(ctx); err != nil { logger.Error("queue worker error", zap.Error(err)) cancel() diff --git a/internal/pkg/mediaconvert/command.go b/internal/pkg/mediaconvert/command.go index 872f9f26d6..047aa3a5a7 100644 --- a/internal/pkg/mediaconvert/command.go +++ b/internal/pkg/mediaconvert/command.go @@ -39,7 +39,6 @@ type Options struct { Transcode bool Threads int UploadID string - WithAudio bool } func newFfmpegCommand(ctx context.Context, in io.Reader, args []string) (*exec.Cmd, error) { @@ -101,17 +100,17 @@ func BuildRawVideoCommand(opts *Options) []string { "-hls_list_size", "0", "-hls_segment_filename", filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", opts.Level)), filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, opts.Level))) - } else { - return append(buildCommonCommand(opts), - "-c:a", "copy", // Copy audio stream - "-c:v", "copy", // Copy video stream - "-f", "hls", - "-hls_time", "5", - "-hls_flags", "split_by_time", - "-hls_list_size", "0", - "-hls_segment_filename", filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", opts.Level)), - filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, opts.Level))) } + + return append(buildCommonCommand(opts), + "-c:a", "copy", // Copy audio stream + "-c:v", "copy", // Copy video stream + "-f", "hls", + "-hls_time", "5", + "-hls_flags", "split_by_time", + "-hls_list_size", "0", + "-hls_segment_filename", filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", opts.Level)), + filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, opts.Level))) } // BuildThumbnailCommand creates a command that creates a thumbnail for the input video @@ -133,14 +132,6 @@ func BuildScalingVideoCommand(opts *Options) []string { continue } - // result = append(result, "-map", "0:v") - if opts.WithAudio { - // result = append(result, - // "-map", "0:a", - // "-c:a", "aac", - // ) - } - result = append(result, "-map", "0:v", "-vf", "scale=-2:"+level[:len(level)-1], diff --git a/internal/pkg/mediaconvert/command_test.go b/internal/pkg/mediaconvert/command_test.go index 531c3ed588..b6161e7ea7 100644 --- a/internal/pkg/mediaconvert/command_test.go +++ b/internal/pkg/mediaconvert/command_test.go @@ -59,7 +59,6 @@ func Test_BuildVideoCommand_Raw_NoTranscode(t *testing.T) { Threads: 4, Level: resconv.Level("651:490"), Transcode: false, - WithAudio: true, }) const expected = `"-threads 4 -i pipe:0 -c:a copy -c:v copy -f hls -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` @@ -75,7 +74,6 @@ func Test_BuildVideoCommand_Raw_Transcode(t *testing.T) { Threads: 4, Level: resconv.Level("651:490"), Transcode: true, - WithAudio: true, }) const expected = `-threads 4 -i pipe:0 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` diff --git a/internal/pkg/mediaconvert/coordinator.go b/internal/pkg/mediaconvert/coordinator.go index 50c8846a76..d2721134ee 100644 --- a/internal/pkg/mediaconvert/coordinator.go +++ b/internal/pkg/mediaconvert/coordinator.go @@ -101,7 +101,6 @@ func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo Threads: s.conf.MaxThreadCount, UploadID: info.ID, Transcode: true, - WithAudio: true, Level: level, ScalingLevels: scaling, } diff --git a/internal/pkg/mediaconvert/scheduler.go b/internal/pkg/mediaconvert/scheduler.go index 9506e54169..dd2f75ee68 100644 --- a/internal/pkg/mediaconvert/scheduler.go +++ b/internal/pkg/mediaconvert/scheduler.go @@ -181,10 +181,10 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { logger.Error("no video stream found in the file", zap.String("filepath", sourceFilePath)) _ = os.RemoveAll(destinationFolder) return - } else { - logger.Debug("video stream found", zap.String("codec", videoStream.CodecName), zap.Int("width", videoStream.Width), zap.Int("height", videoStream.Height)) } + logger.Debug("video stream found", zap.String("codec", videoStream.CodecName), zap.Int("width", videoStream.Width), zap.Int("height", videoStream.Height)) + var res = fmt.Sprintf("%v:%v", videoStream.Width, videoStream.Height) var codec = videoStream.CodecName var level = resconv.Level(res) @@ -193,7 +193,6 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { OutputDir: p.cfg.OutputDir, Level: level, Transcode: !IsHLSSupportedVideoCodec(codec), - WithAudio: audioStream != nil, ScalingLevels: append(resconv.SubLevels(res), level), UploadID: task.ID, Threads: p.cfg.MaxThreadCount, @@ -280,6 +279,7 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { } } +// IsHLSSupportedVideoCodec checks whether the codec is supported by HLS func IsHLSSupportedVideoCodec(codec string) bool { switch codec { case "h264", "h265": diff --git a/internal/pkg/mediaconvert/transcoder.go b/internal/pkg/mediaconvert/transcoder.go index 99b9e62244..4372b9b01f 100644 --- a/internal/pkg/mediaconvert/transcoder.go +++ b/internal/pkg/mediaconvert/transcoder.go @@ -52,7 +52,7 @@ func NewTranscoder(ctx context.Context, cfg *config.Config) *Transcoder { return p } -// TODO: add a factory pattern to process tasks by different media type +// Transcode handles one transcoding task func (p *Transcoder) Transcode(ctx context.Context, task *Task) { var logger = p.logger.With(zap.String("task-id", task.ID)) @@ -128,10 +128,10 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) { logger.Error("no video stream found in the file", zap.String("filepath", sourceFilePath)) _ = os.RemoveAll(destinationFolder) return - } else { - logger.Debug("video stream found", zap.String("codec", videoStream.CodecName), zap.Int("width", videoStream.Width), zap.Int("height", videoStream.Height)) } + logger.Debug("video stream found", zap.String("codec", videoStream.CodecName), zap.Int("width", videoStream.Width), zap.Int("height", videoStream.Height)) + audioStream := probe.FirstAudioStream() if audioStream == nil { logger.Info("no audio stream found in the file", zap.String("filepath", sourceFilePath)) @@ -146,7 +146,6 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) { OutputDir: p.cfg.OutputDir, Level: level, Transcode: !IsHLSSupportedVideoCodec(codec), - WithAudio: audioStream != nil, ScalingLevels: append(sublevels, level), UploadID: task.ID, Threads: p.cfg.MaxThreadCount, diff --git a/internal/pkg/queue/config.go b/internal/pkg/queue/config.go index df2b2c9a7c..193f9eb140 100644 --- a/internal/pkg/queue/config.go +++ b/internal/pkg/queue/config.go @@ -23,6 +23,7 @@ type Config struct { Region string } +// ParseConfig parses the queue config string func ParseConfig(config, clientID, region string) Config { var parts = strings.Split(config, ";") var brokers = strings.Split(parts[0], ",") diff --git a/internal/pkg/queue/config_test.go b/internal/pkg/queue/config_test.go index 4040db1b52..b4772780c5 100644 --- a/internal/pkg/queue/config_test.go +++ b/internal/pkg/queue/config_test.go @@ -24,14 +24,14 @@ func Test_ParseConfig(t *testing.T) { tests := []struct { name string config string - clientId string + clientID string region string want queue.Config }{ { name: "Basic broker configuration", config: "broker1:9092,broker2:9092", - clientId: "test-client", + clientID: "test-client", region: "us-west", want: queue.Config{ Postfix: "", @@ -43,7 +43,7 @@ func Test_ParseConfig(t *testing.T) { { name: "Broker configuration with postfix", config: "broker1:9092,broker2:9092;custom-postfix", - clientId: "test-client", + clientID: "test-client", region: "eu-central", want: queue.Config{ Postfix: "custom-postfix", @@ -55,7 +55,7 @@ func Test_ParseConfig(t *testing.T) { { name: "Single broker configuration", config: "broker1:9092", - clientId: "single-broker-client", + clientID: "single-broker-client", region: "asia-east", want: queue.Config{ Postfix: "", @@ -67,7 +67,7 @@ func Test_ParseConfig(t *testing.T) { { name: "Multiple semicolons in config", config: "broker1:9092,broker2:9092;postfix;extra", - clientId: "test-client", + clientID: "test-client", region: "us-east", want: queue.Config{ Postfix: "postfix", @@ -80,7 +80,7 @@ func Test_ParseConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := queue.ParseConfig(tt.config, tt.clientId, tt.region) + got := queue.ParseConfig(tt.config, tt.clientID, tt.region) if !reflect.DeepEqual(got, tt.want) { t.Errorf("ParseConfig() = %v, want %v", got, tt.want) diff --git a/internal/pkg/queue/queue.go b/internal/pkg/queue/queue.go index 7eae483927..c41d93b4cf 100644 --- a/internal/pkg/queue/queue.go +++ b/internal/pkg/queue/queue.go @@ -24,43 +24,52 @@ import ( "go.uber.org/zap" ) +// TopicTranscodeRequest is the name of the transcode requests topic. const TopicTranscodeRequest = "stream.transcode.request" + +// TopicTranscodeResult is the name of the transcode results topic. const TopicTranscodeResult = "stream.transcode.result" +// TranscodeRequest represents transcode request. type TranscodeRequest struct { BlobID string WorkspaceUUID string ContentType string } +// TranscodeResult represents transcode result. type TranscodeResult struct { BlobID string WorkspaceUUID string } +// Consumer provides a consumer interface to a Kafka queue type Consumer interface { - // Heartbeat(ctx context.Context) error Read(ctx context.Context) (kafka.Message, error) Fetch(ctx context.Context) (kafka.Message, error) Commit(ctx context.Context, msg kafka.Message) error Close() error } +// Producer provides a producer interface to a Kafka queue type Producer interface { Send(ctx context.Context, workspaceID string, data any) error Close() error } +// TConsumer implements Consumer interface type TConsumer struct { topic string reader *kafka.Reader } +// TProducer implements Producer interface type TProducer struct { topic string writer *kafka.Writer } +// Logger is kafka.Logger implementation type Logger struct { logger *zap.Logger } @@ -76,6 +85,7 @@ func NewLogger(ctx context.Context) kafka.Logger { } } +// Printf logs a message func (l *Logger) Printf(msg string, args ...any) { l.logger.Sugar().Debugf(msg, args...) } @@ -100,18 +110,22 @@ func NewConsumer(ctx context.Context, topic, group string, config Config) Consum } } +// Read reads (fetch and commit) a queue message. func (c *TConsumer) Read(ctx context.Context) (kafka.Message, error) { return c.reader.ReadMessage(ctx) } +// Fetch fetches a queue message. func (c *TConsumer) Fetch(ctx context.Context) (kafka.Message, error) { return c.reader.FetchMessage(ctx) } +// Commit commits the queue message. func (c *TConsumer) Commit(ctx context.Context, msg kafka.Message) error { return c.reader.CommitMessages(ctx, msg) } +// Close closes the consumer. func (c *TConsumer) Close() error { return c.reader.Close() } @@ -131,6 +145,7 @@ func NewProducer(ctx context.Context, topic string, config Config) Producer { } } +// Send sends a message to the queue topic func (p *TProducer) Send(ctx context.Context, workspaceID string, data any) error { value, err := json.Marshal(data) if err != nil { @@ -146,6 +161,7 @@ func (p *TProducer) Send(ctx context.Context, workspaceID string, data any) erro ) } +// Close closes the producer. func (p *TProducer) Close() error { return p.writer.Close() } diff --git a/internal/pkg/queue/worker.go b/internal/pkg/queue/worker.go index 8925d024da..6a6aafe9cf 100644 --- a/internal/pkg/queue/worker.go +++ b/internal/pkg/queue/worker.go @@ -16,9 +16,7 @@ package queue import ( "context" "encoding/json" - "errors" "fmt" - "time" "github.com/hcengineering/stream/internal/pkg/config" "github.com/hcengineering/stream/internal/pkg/log" @@ -28,22 +26,27 @@ import ( "go.uber.org/zap" ) +// Worker is a queue processing worker type Worker struct { logger *zap.Logger cfg *config.Config consumer Consumer + producer Producer } -func NewWorker(ctx context.Context, consumer Consumer, cfg *config.Config) *Worker { +// NewWorker creates new worker +func NewWorker(ctx context.Context, consumer Consumer, producer Producer, cfg *config.Config) *Worker { logger := log.FromContext(ctx).With(zap.String("queue", "worker")) return &Worker{ logger: logger, cfg: cfg, consumer: consumer, + producer: producer, } } +// Start queue processing func (w *Worker) Start(ctx context.Context) error { w.logger.Info("starting worker") defer w.logger.Info("worker stopped") @@ -67,8 +70,6 @@ func (w *Worker) Start(ctx context.Context) error { func (w *Worker) fetchAndProcessMessage(ctx context.Context) error { // TODO We should use Fetch and Commit here but as long processing // time is longer than the max poll interval, we have to use Read. - // msg, err := w.consumer.Fetch(ctx) - commit := false msg, err := w.consumer.Read(ctx) if err != nil { return fmt.Errorf("read message: %w", err) @@ -81,48 +82,24 @@ func (w *Worker) fetchAndProcessMessage(ctx context.Context) error { ) logger.Debug("message", zap.ByteString("message", msg.Value)) - _, err = w.processMessage(ctx, msg, logger) + err = w.processMessage(ctx, msg, logger) if err != nil { w.logger.Error("failed to process message", zap.Error(err)) - } - - if commit { - if err := w.commitMessage(ctx, msg, logger); err != nil { - return errors.Wrapf(err, "failed to commit message") - } - } - - return err -} - -func (w *Worker) commitMessage(ctx context.Context, msg kafka.Message, logger *zap.Logger) error { - maxRetries := 3 - for i := range maxRetries { - err := w.consumer.Commit(ctx, msg) - if err == nil { - return nil - } - - if i < maxRetries-1 { - logger.Error("failed to commit message", zap.Error(err), zap.Int("attempt", i+1)) - time.Sleep(time.Duration(100*(i+1)) * time.Millisecond) - } else { - return err - } + return err } return nil } -func (w *Worker) processMessage(ctx context.Context, msg kafka.Message, logger *zap.Logger) (bool, error) { +func (w *Worker) processMessage(ctx context.Context, msg kafka.Message, logger *zap.Logger) error { var req TranscodeRequest if err := json.Unmarshal(msg.Value, &req); err != nil { - return true, fmt.Errorf("parse failed: %w", err) + return fmt.Errorf("parse failed: %w", err) } - if mediaconvert.IsSupportedMediaType(req.ContentType) == false { + if !mediaconvert.IsSupportedMediaType(req.ContentType) { logger.Debug("unsupported content type", zap.String("contentType", req.ContentType)) - return true, nil + return nil } task := mediaconvert.Task{ @@ -134,5 +111,5 @@ func (w *Worker) processMessage(ctx context.Context, msg kafka.Message, logger * transcoder := mediaconvert.NewTranscoder(ctx, w.cfg) transcoder.Transcode(ctx, &task) - return true, nil + return nil } diff --git a/internal/pkg/resconv/resconv.go b/internal/pkg/resconv/resconv.go index 97d716f3f0..da9b855538 100644 --- a/internal/pkg/resconv/resconv.go +++ b/internal/pkg/resconv/resconv.go @@ -103,6 +103,7 @@ func Level(resolution string) string { return prefixes[idx].label } +// Height returns height for the resolution func Height(resolution string) int { var parts = strings.Split(resolution, ":") var h = 240 diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index d470bb6bf8..56bf8a60e6 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -33,8 +33,8 @@ import ( ) type uploadResult struct { - key string `json:"key"` - error string `json:"error"` + key string + error string } // DatalakeStorage represents datalake storage @@ -139,7 +139,7 @@ func (d *DatalakeStorage) PutFile(ctx context.Context, fileName string) error { for _, res := range result { if res.error != "" { - return fmt.Errorf("upload error: %v", res.error) + return fmt.Errorf("upload error: %v %v", res.key, res.error) } } diff --git a/internal/pkg/uploader/uploader.go b/internal/pkg/uploader/uploader.go index d605dfe0c2..957d889f61 100644 --- a/internal/pkg/uploader/uploader.go +++ b/internal/pkg/uploader/uploader.go @@ -16,7 +16,6 @@ package uploader import ( "context" - "errors" "hash/fnv" "os" "path/filepath" @@ -28,6 +27,7 @@ import ( "github.com/hcengineering/stream/internal/pkg/log" "github.com/hcengineering/stream/internal/pkg/storage" + "github.com/pkg/errors" "go.uber.org/zap" "k8s.io/utils/inotify" ) From 94dddeb29c167b7631653c5d8d928f380e499db6 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 11 Jun 2025 18:38:59 +0700 Subject: [PATCH 33/52] add transcode result Signed-off-by: Alexander Onnikov --- internal/pkg/mediaconvert/scheduler.go | 18 +++--- internal/pkg/mediaconvert/transcoder.go | 73 ++++++++++++------------- internal/pkg/queue/queue.go | 2 + internal/pkg/queue/worker.go | 10 +++- internal/pkg/storage/datalake.go | 7 --- internal/pkg/storage/s3.go | 5 -- internal/pkg/storage/storage.go | 1 - 7 files changed, 54 insertions(+), 62 deletions(-) diff --git a/internal/pkg/mediaconvert/scheduler.go b/internal/pkg/mediaconvert/scheduler.go index dd2f75ee68..1d71d9113f 100644 --- a/internal/pkg/mediaconvert/scheduler.go +++ b/internal/pkg/mediaconvert/scheduler.go @@ -35,14 +35,6 @@ import ( "gopkg.in/vansante/go-ffprobe.v2" ) -// HLS represents metadata for transcoding result -type HLS struct { - Source string `json:"source"` - Thumbnail string `json:"thumbnail"` - Width int `json:"width"` - Height int `json:"height"` -} - // Task represents transcoding task type Task struct { ID string @@ -53,6 +45,14 @@ type Task struct { Metadata map[string]string } +// TaskResult represents transcoding task result +type TaskResult struct { + Source string `json:"source"` + Thumbnail string `json:"thumbnail"` + Width int `json:"width"` + Height int `json:"height"` +} + // Scheduler manages transcoding tasks by passed config type Scheduler struct { logger *zap.Logger @@ -258,7 +258,7 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { logger.Debug("phase 9: try to set metadata") if metaProvider, ok := remoteStorage.(storage.MetaProvider); ok { - var hls = HLS{ + var hls = TaskResult{ Width: videoStream.Width, Height: videoStream.Height, Source: task.ID + "_master.m3u8", diff --git a/internal/pkg/mediaconvert/transcoder.go b/internal/pkg/mediaconvert/transcoder.go index 4372b9b01f..8b04ca677d 100644 --- a/internal/pkg/mediaconvert/transcoder.go +++ b/internal/pkg/mediaconvert/transcoder.go @@ -30,6 +30,7 @@ import ( "github.com/hcengineering/stream/internal/pkg/storage" "github.com/hcengineering/stream/internal/pkg/token" "github.com/hcengineering/stream/internal/pkg/uploader" + "github.com/pkg/errors" "go.uber.org/zap" "gopkg.in/vansante/go-ffprobe.v2" ) @@ -53,7 +54,7 @@ func NewTranscoder(ctx context.Context, cfg *config.Config) *Transcoder { } // Transcode handles one transcoding task -func (p *Transcoder) Transcode(ctx context.Context, task *Task) { +func (p *Transcoder) Transcode(ctx context.Context, task *Task) (TaskResult, error) { var logger = p.logger.With(zap.String("task-id", task.ID)) logger.Debug("start") @@ -63,7 +64,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) { var tokenString, err = token.NewToken(p.cfg.ServerSecret, task.Workspace, "stream", "datalake") if err != nil { logger.Error("can not create token", zap.Error(err)) - return + return TaskResult{}, errors.Wrapf(err, "can not create token") } logger.Debug("phase 2: preparing fs") @@ -72,7 +73,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) { err = os.MkdirAll(destinationFolder, os.ModePerm) if err != nil { logger.Error("can not create temporary folder", zap.Error(err)) - return + return TaskResult{}, errors.Wrapf(err, "can not create temporary folder") } defer func() { @@ -84,57 +85,47 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) { logger.Debug("phase 3: get the remote file") remoteStorage, err := storage.NewStorageByURL(ctx, p.cfg.Endpoint(), p.cfg.EndpointURL.Scheme, tokenString, task.Workspace) - if err != nil { logger.Error("can not create storage by url", zap.Error(err), zap.String("url", p.cfg.EndpointURL.String())) - _ = os.RemoveAll(destinationFolder) - return + return TaskResult{}, errors.Wrapf(err, "can not create storage by url") } stat, err := remoteStorage.StatFile(ctx, task.Source) if err != nil { - logger.Error("can not stat a file", zap.Error(err), zap.String("filepath", task.Source)) - _ = os.RemoveAll(destinationFolder) - return + logger.Error("can not stat file", zap.Error(err), zap.String("filepath", task.Source)) + return TaskResult{}, errors.Wrapf(err, "can not stat file") } if !IsSupportedMediaType(stat.Type) { logger.Info("unsupported media type", zap.String("type", stat.Type)) - _ = os.RemoveAll(destinationFolder) - return + return TaskResult{}, fmt.Errorf("unsupported media type: %s", stat.Type) } - // sourceFilePath := remoteStorage.GetFileURL(ctx, task.Source) - // if sourceFilePath == "" { sourceFilePath := filepath.Join(destinationFolder, filename) if err = remoteStorage.GetFile(ctx, task.Source, sourceFilePath); err != nil { - logger.Error("can not download a file", zap.Error(err), zap.String("filepath", task.Source)) - _ = os.RemoveAll(destinationFolder) + logger.Error("can not download source file", zap.Error(err), zap.String("filepath", task.Source)) // TODO: reschedule - return + return TaskResult{}, errors.Wrapf(err, "can not download source file") } - // } logger.Debug("phase 4: prepare to transcode") probe, err := ffprobe.ProbeURL(ctx, sourceFilePath) if err != nil { - logger.Error("can not get probe for a file", zap.Error(err), zap.String("filepath", sourceFilePath)) - _ = os.RemoveAll(destinationFolder) - return + logger.Error("can not get ffprobe", zap.Error(err), zap.String("filepath", sourceFilePath)) + return TaskResult{}, errors.Wrapf(err, "can not get ffprobe") } videoStream := probe.FirstVideoStream() if videoStream == nil { - logger.Error("no video stream found in the file", zap.String("filepath", sourceFilePath)) - _ = os.RemoveAll(destinationFolder) - return + logger.Error("no video stream found", zap.String("filepath", sourceFilePath)) + return TaskResult{}, errors.Wrapf(err, "no video stream found") } logger.Debug("video stream found", zap.String("codec", videoStream.CodecName), zap.Int("width", videoStream.Width), zap.Int("height", videoStream.Height)) audioStream := probe.FirstAudioStream() if audioStream == nil { - logger.Info("no audio stream found in the file", zap.String("filepath", sourceFilePath)) + logger.Info("no audio stream found", zap.String("filepath", sourceFilePath)) } var res = fmt.Sprintf("%v:%v", videoStream.Width, videoStream.Height) @@ -166,8 +157,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) { err = manifest.GenerateHLSPlaylist(opts.ScalingLevels, p.cfg.OutputDir, opts.UploadID) if err != nil { logger.Error("can not generate hls playlist", zap.String("out", p.cfg.OutputDir), zap.String("uploadID", opts.UploadID)) - _ = os.RemoveAll(destinationFolder) - return + return TaskResult{}, errors.Wrapf(err, "can not generate hls playlist") } go uploader.Start() @@ -186,13 +176,13 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) { if cmdErr != nil { logger.Error("can not create a new command", zap.Error(cmdErr), zap.Strings("args", args)) go uploader.Cancel() - return + return TaskResult{}, errors.Wrapf(err, "can not create a new command") } cmds = append(cmds, cmd) if err = cmd.Start(); err != nil { logger.Error("can not start a command", zap.Error(err), zap.Strings("args", args)) go uploader.Cancel() - return + return TaskResult{}, errors.Wrapf(err, "can not start a command") } } @@ -201,7 +191,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) { if err = cmd.Wait(); err != nil { logger.Error("can not wait for command end ", zap.Error(err)) go uploader.Cancel() - return + return TaskResult{}, errors.Wrapf(err, "can not wait for command end") } } @@ -210,24 +200,31 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) { logger.Debug("phase 9: try to set metadata") - if metaProvider, ok := remoteStorage.(storage.MetaProvider); ok { - var hls = HLS{ - Width: videoStream.Width, - Height: videoStream.Height, - Source: task.ID + "_master.m3u8", - Thumbnail: task.ID + ".jpg", - } + var result = TaskResult{ + Width: videoStream.Width, + Height: videoStream.Height, + Source: task.ID + "_master.m3u8", + Thumbnail: task.ID + ".jpg", + } - logger.Debug("applying metadata", zap.String("url", hls.Source), zap.String("thumbnail", hls.Thumbnail), zap.String("source", task.Source)) + if metaProvider, ok := remoteStorage.(storage.MetaProvider); ok { + logger.Debug( + "applying metadata", + zap.String("url", result.Source), + zap.String("thumbnail", result.Thumbnail), + zap.String("source", task.Source), + ) err = metaProvider.PatchMeta( ctx, task.Source, &storage.Metadata{ - "hls": hls, + "hls": result, }, ) if err != nil { logger.Error("can not patch the source file", zap.Error(err)) } } + + return result, nil } diff --git a/internal/pkg/queue/queue.go b/internal/pkg/queue/queue.go index c41d93b4cf..9786581529 100644 --- a/internal/pkg/queue/queue.go +++ b/internal/pkg/queue/queue.go @@ -41,6 +41,8 @@ type TranscodeRequest struct { type TranscodeResult struct { BlobID string WorkspaceUUID string + Playlist string + Thumbnail string } // Consumer provides a consumer interface to a Kafka queue diff --git a/internal/pkg/queue/worker.go b/internal/pkg/queue/worker.go index 6a6aafe9cf..58f5506f46 100644 --- a/internal/pkg/queue/worker.go +++ b/internal/pkg/queue/worker.go @@ -109,7 +109,13 @@ func (w *Worker) processMessage(ctx context.Context, msg kafka.Message, logger * } transcoder := mediaconvert.NewTranscoder(ctx, w.cfg) - transcoder.Transcode(ctx, &task) + res, err := transcoder.Transcode(ctx, &task) - return nil + if err == nil { + if err := w.producer.Send(ctx, req.WorkspaceUUID, res); err != nil { + logger.Error("failed to send transcode result", zap.Error(err)) + } + } + + return err } diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index 56bf8a60e6..8a5284197f 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -216,12 +216,6 @@ func (d *DatalakeStorage) PatchMeta(ctx context.Context, filename string, md *Me return nil } -// GetFileURL returns blob URL -func (d *DatalakeStorage) GetFileURL(ctx context.Context, filename string) string { - objectKey := getObjectKeyFromPath(filename) - return d.baseURL + "/blob/" + d.workspace + "/" + objectKey -} - // GetMeta gets metadata related to the object func (d *DatalakeStorage) GetMeta(ctx context.Context, filename string) (*Metadata, error) { var logger = d.logger.With(zap.String("get meta", d.workspace), zap.String("fileName", filename)) @@ -250,7 +244,6 @@ func (d *DatalakeStorage) GetMeta(ctx context.Context, filename string) (*Metada } var md Metadata - fmt.Println(string(resp.Body())) var err = json.Unmarshal(resp.Body(), &md) return &md, err diff --git a/internal/pkg/storage/s3.go b/internal/pkg/storage/s3.go index 254cb9d723..e4f3050d09 100644 --- a/internal/pkg/storage/s3.go +++ b/internal/pkg/storage/s3.go @@ -165,11 +165,6 @@ func (u *S3Storage) GetFile(ctx context.Context, filename, dest string) error { return nil } -// GetFileURL returns blob URL -func (u *S3Storage) GetFileURL(ctx context.Context, filename string) string { - return "" -} - // StatFile gets file stat from the storage func (u *S3Storage) StatFile(ctx context.Context, filename string) (*BlobInfo, error) { var logger = u.logger.With(zap.String("head", u.bucketName), zap.String("fileName", filename)) diff --git a/internal/pkg/storage/storage.go b/internal/pkg/storage/storage.go index 8f7cfbe3f1..0feb587825 100644 --- a/internal/pkg/storage/storage.go +++ b/internal/pkg/storage/storage.go @@ -41,7 +41,6 @@ type BlobInfo struct { type Storage interface { PutFile(ctx context.Context, fileName string) error DeleteFile(ctx context.Context, fileName string) error - GetFileURL(ctx context.Context, fileName string) string GetFile(ctx context.Context, fileName, destination string) error StatFile(ctx context.Context, fileName string) (*BlobInfo, error) SetParent(ctx context.Context, fileName string, parentName string) error From ce18a76568e9afcb38b4544dfa733a936615e8c9 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Wed, 11 Jun 2025 23:50:49 +0700 Subject: [PATCH 34/52] extract producer and consumer options Signed-off-by: Alexander Onnikov --- cmd/stream/main.go | 22 +++++++++--- internal/pkg/queue/queue.go | 70 ++++++++++++++++++++++++------------ internal/pkg/queue/worker.go | 2 +- 3 files changed, 67 insertions(+), 27 deletions(-) diff --git a/cmd/stream/main.go b/cmd/stream/main.go index 17e2b6fe12..37174f32af 100644 --- a/cmd/stream/main.go +++ b/cmd/stream/main.go @@ -85,8 +85,9 @@ func main() { } server := &http.Server{ - Addr: cfg.ServeURL, - Handler: handler, + Addr: cfg.ServeURL, + Handler: handler, + ReadHeaderTimeout: 15 * time.Second, } var wg sync.WaitGroup @@ -107,8 +108,21 @@ func main() { config := queue.ParseConfig(cfg.QueueConfig, "stream", cfg.Region) logger.Info("using queue config", zap.Any("config", config)) - consumer := queue.NewConsumer(ctx, queue.TopicTranscodeRequest, "stream", config) - producer := queue.NewProducer(ctx, queue.TopicTranscodeResult, config) + + consumerOptions := queue.ConsumerOptions{ + Topic: queue.TopicTranscodeRequest, + Group: "stream", + Config: config, + } + consumer := queue.NewConsumer(ctx, consumerOptions) + + producerOptions := queue.ProducerOptions{ + Topic: queue.TopicTranscodeResult, + Group: "stream", + Config: config, + RetryCount: 3, + } + producer := queue.NewProducer(ctx, producerOptions) wg.Add(1) go func() { diff --git a/internal/pkg/queue/queue.go b/internal/pkg/queue/queue.go index 9786581529..64389e8335 100644 --- a/internal/pkg/queue/queue.go +++ b/internal/pkg/queue/queue.go @@ -45,6 +45,13 @@ type TranscodeResult struct { Thumbnail string } +// ConsumerOptions represents options for the consumer +type ConsumerOptions struct { + Topic string + Group string + Config Config +} + // Consumer provides a consumer interface to a Kafka queue type Consumer interface { Read(ctx context.Context) (kafka.Message, error) @@ -53,6 +60,14 @@ type Consumer interface { Close() error } +// ProducerOptions represents options for the producer +type ProducerOptions struct { + Topic string + Group string + Config Config + RetryCount int +} + // Producer provides a producer interface to a Kafka queue type Producer interface { Send(ctx context.Context, workspaceID string, data any) error @@ -61,14 +76,16 @@ type Producer interface { // TConsumer implements Consumer interface type TConsumer struct { - topic string - reader *kafka.Reader + topic string + reader *kafka.Reader + options ConsumerOptions } // TProducer implements Producer interface type TProducer struct { - topic string - writer *kafka.Writer + topic string + writer *kafka.Writer + options ProducerOptions } // Logger is kafka.Logger implementation @@ -93,12 +110,12 @@ func (l *Logger) Printf(msg string, args ...any) { } // NewConsumer creates a new transcoding request consumer -func NewConsumer(ctx context.Context, topic, group string, config Config) Consumer { - platformTopic := makeTopicID(topic, config) - groupID := makeGroupID(group, topic, config) +func NewConsumer(ctx context.Context, options ConsumerOptions) Consumer { + platformTopic := makeTopicID(options.Topic, options.Config) + groupID := makeGroupID(options.Group, options.Topic, options.Config) reader := kafka.NewReader(kafka.ReaderConfig{ - Brokers: config.Brokers, + Brokers: options.Config.Brokers, GroupID: groupID, Topic: platformTopic, Logger: NewLogger(ctx), @@ -107,8 +124,9 @@ func NewConsumer(ctx context.Context, topic, group string, config Config) Consum }) return &TConsumer{ - topic: platformTopic, - reader: reader, + options: options, + topic: platformTopic, + reader: reader, } } @@ -133,17 +151,18 @@ func (c *TConsumer) Close() error { } // NewProducer creates a new transcoding result producer -func NewProducer(ctx context.Context, topic string, config Config) Producer { - platformTopic := makeTopicID(topic, config) +func NewProducer(ctx context.Context, options ProducerOptions) Producer { + platformTopic := makeTopicID(options.Topic, options.Config) writer := &kafka.Writer{ - Addr: kafka.TCP(config.Brokers...), + Addr: kafka.TCP(options.Config.Brokers...), Topic: platformTopic, AllowAutoTopicCreation: true, Logger: NewLogger(ctx), } return &TProducer{ - topic: platformTopic, - writer: writer, + options: options, + topic: platformTopic, + writer: writer, } } @@ -154,13 +173,20 @@ func (p *TProducer) Send(ctx context.Context, workspaceID string, data any) erro return err } - return p.writer.WriteMessages( - ctx, - kafka.Message{ - Key: []byte(workspaceID), - Value: value, - }, - ) + for range p.options.RetryCount { + err = p.writer.WriteMessages( + ctx, + kafka.Message{ + Key: []byte(workspaceID), + Value: value, + }, + ) + if err == nil { + break + } + } + + return err } // Close closes the producer. diff --git a/internal/pkg/queue/worker.go b/internal/pkg/queue/worker.go index 58f5506f46..0137f48be9 100644 --- a/internal/pkg/queue/worker.go +++ b/internal/pkg/queue/worker.go @@ -112,7 +112,7 @@ func (w *Worker) processMessage(ctx context.Context, msg kafka.Message, logger * res, err := transcoder.Transcode(ctx, &task) if err == nil { - if err := w.producer.Send(ctx, req.WorkspaceUUID, res); err != nil { + if err = w.producer.Send(ctx, req.WorkspaceUUID, res); err != nil { logger.Error("failed to send transcode result", zap.Error(err)) } } From c83ff3bf6837eb2b3d4a3530a82fb88c2595ca5e Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Fri, 20 Jun 2025 16:25:10 +0700 Subject: [PATCH 35/52] fix: properly detect created hls segments Signed-off-by: Alexander Onnikov --- Dockerfile | 2 +- go.mod | 2 +- go.sum | 4 +- internal/pkg/mediaconvert/command.go | 34 +++++++++++--- internal/pkg/mediaconvert/command_test.go | 8 ++-- internal/pkg/mediaconvert/scheduler.go | 2 + internal/pkg/mediaconvert/transcoder.go | 56 ++++++++++++++++------- internal/pkg/storage/datalake.go | 56 +++++++++++++---------- internal/pkg/uploader/uploader.go | 25 ++++++++-- 9 files changed, 131 insertions(+), 58 deletions(-) diff --git a/Dockerfile b/Dockerfile index 121a1b5efb..1e7a5cd10b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.24.1 AS builder +FROM golang:1.24.4 AS builder ENV GO111MODULE=on ENV CGO_ENABLED=0 ENV GOBIN=/bin diff --git a/go.mod b/go.mod index 905e524afa..701291bda0 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.17.59 github.com/aws/aws-sdk-go-v2/service/s3 v1.77.0 github.com/getsentry/sentry-go v0.31.1 - github.com/golang-jwt/jwt/v5 v5.2.1 + github.com/golang-jwt/jwt/v5 v5.2.2 github.com/google/uuid v1.6.0 github.com/kelseyhightower/envconfig v1.4.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index c531b49b39..3ca0c0a4f2 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,8 @@ github.com/getsentry/sentry-go v0.31.1 h1:ELVc0h7gwyhnXHDouXkhqTFSO5oslsRDk0++ey github.com/getsentry/sentry-go v0.31.1/go.mod h1:CYNcMMz73YigoHljQRG+qPF+eMq8gG72XcGN/p71BAY= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= -github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= diff --git a/internal/pkg/mediaconvert/command.go b/internal/pkg/mediaconvert/command.go index 047aa3a5a7..2efbf54cf7 100644 --- a/internal/pkg/mediaconvert/command.go +++ b/internal/pkg/mediaconvert/command.go @@ -19,7 +19,6 @@ import ( "context" "fmt" "io" - "os" "os/exec" "path/filepath" "strings" @@ -30,12 +29,27 @@ import ( "go.uber.org/zap" ) +type LogLevel string + +const ( + LogLevelQuiet LogLevel = "quiet" + LogLevelPanic LogLevel = "panic" + LogLevelFatal LogLevel = "fatal" + LogLevelError LogLevel = "error" + LogLevelWarning LogLevel = "warning" + LogLevelInfo LogLevel = "info" + LogLevelVerbose LogLevel = "verbose" + LogLevelDebug LogLevel = "debug" + LogLevelTrace LogLevel = "trace" +) + // Options represents configuration for the ffmpeg command type Options struct { Input string OutputDir string ScalingLevels []string Level string + LogLevel LogLevel Transcode bool Threads int UploadID string @@ -51,8 +65,6 @@ func newFfmpegCommand(ctx context.Context, in io.Reader, args []string) (*exec.C logger.Debug("prepared command: ", zap.Strings("args", args)) var result = exec.CommandContext(ctx, "ffmpeg", args...) - result.Stderr = os.Stdout - result.Stdout = os.Stdout result.Stdin = in return result, nil @@ -60,6 +72,8 @@ func newFfmpegCommand(ctx context.Context, in io.Reader, args []string) (*exec.C func buildCommonCommand(opts *Options) []string { var result = []string{ + "-y", // Overwrite output files without asking. + "-v", string(opts.LogLevel), "-threads", fmt.Sprint(opts.Threads), "-i", opts.Input, } @@ -96,7 +110,7 @@ func BuildRawVideoCommand(opts *Options) []string { "-g", "60", "-f", "hls", "-hls_time", "5", - "-hls_flags", "split_by_time", + "-hls_flags", "split_by_time+temp_file", "-hls_list_size", "0", "-hls_segment_filename", filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", opts.Level)), filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, opts.Level))) @@ -107,7 +121,7 @@ func BuildRawVideoCommand(opts *Options) []string { "-c:v", "copy", // Copy video stream "-f", "hls", "-hls_time", "5", - "-hls_flags", "split_by_time", + "-hls_flags", "split_by_time+temp_file", "-hls_list_size", "0", "-hls_segment_filename", filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", opts.Level)), filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, opts.Level))) @@ -142,7 +156,15 @@ func BuildScalingVideoCommand(opts *Options) []string { "-g", "60", "-f", "hls", "-hls_time", "5", - "-hls_flags", "split_by_time", + // Use HLS flags + // - split_by_time + // Allow segments to start on frames other than key frames. + // This improves behavior on some players when the time between key frames is inconsistent, + // but may make things worse on others, and can cause some oddities during seeking. + // This flag should be used with the hls_time option. + // - temp_file + // Write segment data to filename.tmp and rename to filename only once the segment is complete. + "-hls_flags", "split_by_time+temp_file", "-hls_list_size", "0", "-hls_segment_filename", filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", level)), filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, level))) diff --git a/internal/pkg/mediaconvert/command_test.go b/internal/pkg/mediaconvert/command_test.go index b6161e7ea7..26beed3272 100644 --- a/internal/pkg/mediaconvert/command_test.go +++ b/internal/pkg/mediaconvert/command_test.go @@ -31,7 +31,7 @@ func Test_BuildVideoCommand_Scaling(t *testing.T) { ScalingLevels: []string{"720p", "480p"}, }) - const expected = `-threads 4 -i pipe:0 -map 0:v -vf scale=-2:720 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -map 0:v -vf scale=-2:480 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `-threads 4 -i pipe:0 -map 0:v -vf scale=-2:720 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -map 0:v -vf scale=-2:480 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(scaleCommand, " ")) } @@ -46,7 +46,7 @@ func Test_BuildVideoCommand_Scaling_NoRaw(t *testing.T) { ScalingLevels: []string{"720p", "480p"}, }) - const expected = `-threads 4 -i pipe:0 -map 0:v -vf scale=-2:480 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `-threads 4 -i pipe:0 -map 0:v -vf scale=-2:480 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(scaleCommand, " ")) } @@ -61,7 +61,7 @@ func Test_BuildVideoCommand_Raw_NoTranscode(t *testing.T) { Transcode: false, }) - const expected = `"-threads 4 -i pipe:0 -c:a copy -c:v copy -f hls -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `"-threads 4 -i pipe:0 -c:a copy -c:v copy -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(rawCommand, " ")) } @@ -76,7 +76,7 @@ func Test_BuildVideoCommand_Raw_Transcode(t *testing.T) { Transcode: true, }) - const expected = `-threads 4 -i pipe:0 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `-threads 4 -i pipe:0 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(rawCommand, " ")) } diff --git a/internal/pkg/mediaconvert/scheduler.go b/internal/pkg/mediaconvert/scheduler.go index 1d71d9113f..5f33537360 100644 --- a/internal/pkg/mediaconvert/scheduler.go +++ b/internal/pkg/mediaconvert/scheduler.go @@ -299,6 +299,8 @@ func IsSupportedMediaType(mediaType string) bool { return true case "video/webm": return true + case "video/quicktime": + return true default: return false } diff --git a/internal/pkg/mediaconvert/transcoder.go b/internal/pkg/mediaconvert/transcoder.go index 8b04ca677d..fca9d02353 100644 --- a/internal/pkg/mediaconvert/transcoder.go +++ b/internal/pkg/mediaconvert/transcoder.go @@ -16,8 +16,10 @@ package mediaconvert import ( + "bytes" "context" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -42,6 +44,12 @@ type Transcoder struct { logger *zap.Logger } +type Command struct { + cmd *exec.Cmd + stdoutBuf bytes.Buffer + stderrBuf bytes.Buffer +} + // NewTranscoder creates a new instance of task transcoder func NewTranscoder(ctx context.Context, cfg *config.Config) *Transcoder { var p = &Transcoder{ @@ -54,7 +62,7 @@ func NewTranscoder(ctx context.Context, cfg *config.Config) *Transcoder { } // Transcode handles one transcoding task -func (p *Transcoder) Transcode(ctx context.Context, task *Task) (TaskResult, error) { +func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, error) { var logger = p.logger.With(zap.String("task-id", task.ID)) logger.Debug("start") @@ -64,7 +72,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (TaskResult, err var tokenString, err = token.NewToken(p.cfg.ServerSecret, task.Workspace, "stream", "datalake") if err != nil { logger.Error("can not create token", zap.Error(err)) - return TaskResult{}, errors.Wrapf(err, "can not create token") + return nil, errors.Wrapf(err, "can not create token") } logger.Debug("phase 2: preparing fs") @@ -73,7 +81,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (TaskResult, err err = os.MkdirAll(destinationFolder, os.ModePerm) if err != nil { logger.Error("can not create temporary folder", zap.Error(err)) - return TaskResult{}, errors.Wrapf(err, "can not create temporary folder") + return nil, errors.Wrapf(err, "can not create temporary folder") } defer func() { @@ -87,38 +95,38 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (TaskResult, err remoteStorage, err := storage.NewStorageByURL(ctx, p.cfg.Endpoint(), p.cfg.EndpointURL.Scheme, tokenString, task.Workspace) if err != nil { logger.Error("can not create storage by url", zap.Error(err), zap.String("url", p.cfg.EndpointURL.String())) - return TaskResult{}, errors.Wrapf(err, "can not create storage by url") + return nil, errors.Wrapf(err, "can not create storage by url") } stat, err := remoteStorage.StatFile(ctx, task.Source) if err != nil { logger.Error("can not stat file", zap.Error(err), zap.String("filepath", task.Source)) - return TaskResult{}, errors.Wrapf(err, "can not stat file") + return nil, errors.Wrapf(err, "can not stat file") } if !IsSupportedMediaType(stat.Type) { logger.Info("unsupported media type", zap.String("type", stat.Type)) - return TaskResult{}, fmt.Errorf("unsupported media type: %s", stat.Type) + return nil, fmt.Errorf("unsupported media type: %s", stat.Type) } sourceFilePath := filepath.Join(destinationFolder, filename) if err = remoteStorage.GetFile(ctx, task.Source, sourceFilePath); err != nil { logger.Error("can not download source file", zap.Error(err), zap.String("filepath", task.Source)) // TODO: reschedule - return TaskResult{}, errors.Wrapf(err, "can not download source file") + return nil, errors.Wrapf(err, "can not download source file") } logger.Debug("phase 4: prepare to transcode") probe, err := ffprobe.ProbeURL(ctx, sourceFilePath) if err != nil { logger.Error("can not get ffprobe", zap.Error(err), zap.String("filepath", sourceFilePath)) - return TaskResult{}, errors.Wrapf(err, "can not get ffprobe") + return nil, errors.Wrapf(err, "can not get ffprobe") } videoStream := probe.FirstVideoStream() if videoStream == nil { logger.Error("no video stream found", zap.String("filepath", sourceFilePath)) - return TaskResult{}, errors.Wrapf(err, "no video stream found") + return nil, errors.Wrapf(err, "no video stream found") } logger.Debug("video stream found", zap.String("codec", videoStream.CodecName), zap.Int("width", videoStream.Width), zap.Int("height", videoStream.Height)) @@ -136,6 +144,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (TaskResult, err Input: sourceFilePath, OutputDir: p.cfg.OutputDir, Level: level, + LogLevel: LogLevel(p.cfg.LogLevel), Transcode: !IsHLSSupportedVideoCodec(codec), ScalingLevels: append(sublevels, level), UploadID: task.ID, @@ -157,7 +166,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (TaskResult, err err = manifest.GenerateHLSPlaylist(opts.ScalingLevels, p.cfg.OutputDir, opts.UploadID) if err != nil { logger.Error("can not generate hls playlist", zap.String("out", p.cfg.OutputDir), zap.String("uploadID", opts.UploadID)) - return TaskResult{}, errors.Wrapf(err, "can not generate hls playlist") + return nil, errors.Wrapf(err, "can not generate hls playlist") } go uploader.Start() @@ -169,29 +178,42 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (TaskResult, err BuildRawVideoCommand(&opts), BuildScalingVideoCommand(&opts), } - var cmds []*exec.Cmd + var cmds []Command for _, args := range argsSlice { cmd, cmdErr := newFfmpegCommand(ctx, nil, args) if cmdErr != nil { logger.Error("can not create a new command", zap.Error(cmdErr), zap.Strings("args", args)) go uploader.Cancel() - return TaskResult{}, errors.Wrapf(err, "can not create a new command") + return nil, errors.Wrapf(err, "can not create a new command") } - cmds = append(cmds, cmd) + + var command = Command{ + cmd: cmd, + stdoutBuf: bytes.Buffer{}, + stderrBuf: bytes.Buffer{}, + } + + cmd.Stdout = io.MultiWriter(os.Stdout, &command.stdoutBuf) + cmd.Stderr = io.MultiWriter(os.Stderr, &command.stderrBuf) + + cmds = append(cmds, command) if err = cmd.Start(); err != nil { logger.Error("can not start a command", zap.Error(err), zap.Strings("args", args)) go uploader.Cancel() - return TaskResult{}, errors.Wrapf(err, "can not start a command") + return nil, errors.Wrapf(err, "can not start a command") } } logger.Debug("phase 7: wait for the result") + for _, cmd := range cmds { - if err = cmd.Wait(); err != nil { + if err = cmd.cmd.Wait(); err != nil { logger.Error("can not wait for command end ", zap.Error(err)) + os.Stdout.Write(cmd.stdoutBuf.Bytes()) + os.Stderr.Write(cmd.stderrBuf.Bytes()) go uploader.Cancel() - return TaskResult{}, errors.Wrapf(err, "can not wait for command end") + return nil, errors.Wrapf(err, "can not wait for command end") } } @@ -226,5 +248,5 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (TaskResult, err } } - return result, nil + return &result, nil } diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index 8a5284197f..aaf18ac273 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -21,8 +21,10 @@ import ( "io" "mime/multipart" "net/textproto" + "net/url" "os" "path/filepath" + "strconv" "strings" "time" @@ -170,6 +172,11 @@ func (d *DatalakeStorage) DeleteFile(ctx context.Context, fileName string) error return errors.Wrapf(err, "delete failed") } + if err := okResponse(res); err != nil { + logRequestError(logger, err, "bad status code", res) + return err + } + logger.Debug("deleted") return nil @@ -205,14 +212,11 @@ func (d *DatalakeStorage) PatchMeta(ctx context.Context, filename string, md *Me return err } - if resp.StatusCode() != fasthttp.StatusOK { - var err = fmt.Errorf("unexpected status code: %d", resp.StatusCode()) - logger.Debug("bad status code", zap.Error(err)) + if err := okResponse(resp); err != nil { + logRequestError(logger, err, "bad status code", resp) return err } - fmt.Println(string(resp.Body())) - return nil } @@ -237,9 +241,8 @@ func (d *DatalakeStorage) GetMeta(ctx context.Context, filename string) (*Metada return nil, err } - if resp.StatusCode() != fasthttp.StatusOK { - var err = fmt.Errorf("unexpected status code: %d", resp.StatusCode()) - logger.Debug("bad status code", zap.Error(err)) + if err := okResponse(resp); err != nil { + logRequestError(logger, err, "bad status code", resp) return nil, err } @@ -270,10 +273,8 @@ func (d *DatalakeStorage) GetFile(ctx context.Context, filename, destination str return err } - // Check the response status code - if resp.StatusCode() != fasthttp.StatusOK { - var err = fmt.Errorf("unexpected status code: %d", resp.StatusCode()) - logger.Debug("bad status code", zap.Error(err)) + if err := okResponse(resp); err != nil { + logRequestError(logger, err, "bad status code", resp) return err } @@ -291,7 +292,13 @@ func (d *DatalakeStorage) GetFile(ctx context.Context, filename, destination str return err } - logger.Debug("file downloaded successfully") + stat, err := os.Stat(destination) + if err != nil { + logger.Error("can't stat the file", zap.Error(err)) + return err + } + + logger.Info("file downloaded successfully", zap.Int64("size", stat.Size())) return nil } @@ -316,9 +323,7 @@ func (d *DatalakeStorage) StatFile(ctx context.Context, filename string) (*BlobI return nil, err } - // Check the response status code - if resp.StatusCode() != fasthttp.StatusOK { - var err = fmt.Errorf("unexpected status code: %d", resp.StatusCode()) + if err := okResponse(resp); err != nil { logRequestError(logger, err, "bad status code", resp) return nil, err } @@ -346,7 +351,7 @@ func (d *DatalakeStorage) SetParent(ctx context.Context, filename, parent string req.SetRequestURI(d.baseURL + "/blob/" + d.workspace + "/" + objectKey + "/parent") req.Header.SetMethod(fasthttp.MethodPatch) req.Header.Add("Authorization", "Bearer "+d.token) - req.Header.Add("Content-Type", "application/json") + req.Header.SetContentType("application/json") body := map[string]any{ "parent": parentKey, @@ -365,15 +370,20 @@ func (d *DatalakeStorage) SetParent(ctx context.Context, filename, parent string return err } - // Check the response status code - var statusOK = resp.StatusCode() >= 200 && resp.StatusCode() < 300 - if !statusOK { - var err = fmt.Errorf("unexpected status code: %d", resp.StatusCode()) - logger.Debug("bad status code", zap.Error(err), zap.Int("status", resp.StatusCode()), zap.String("response", resp.String())) + if err := okResponse(resp); err != nil { + logRequestError(logger, err, "bad status code", resp) return err } - logger.Debug("finished") + return nil +} + +func okResponse(res *fasthttp.Response) error { + var statusOK = res.StatusCode() >= 200 && res.StatusCode() < 300 + + if !statusOK { + return fmt.Errorf("unexpected status code: %d", res.StatusCode()) + } return nil } diff --git a/internal/pkg/uploader/uploader.go b/internal/pkg/uploader/uploader.go index 957d889f61..1120751029 100644 --- a/internal/pkg/uploader/uploader.go +++ b/internal/pkg/uploader/uploader.go @@ -35,6 +35,8 @@ import ( // See at https://man7.org/linux/man-pages/man7/inotify.7.html const inotifyCloseWrite uint32 = 0x8 // IN_CLOSE_WRITE const inotifyMovedTo uint32 = 0x80 // IN_MOVED_TO +const inotifyDelete uint32 = 0x200 // IN_DELETE +const inotifyMovedFrom uint32 = 0x40 // IN_MOVED_FROM // Uploader represents file uploader type Uploader interface { @@ -250,9 +252,12 @@ func (u *uploaderImpl) uploadAndDelete(f string) { } // Check if the file exists - _, err := os.Stat(f) - if err != nil { - logger.Debug("file does not exist") + if _, err := os.Stat(f); err != nil { + if os.IsNotExist(err) { + logger.Debug("file does not exist", zap.Error(err)) + } else { + logger.Error("failed to stat file", zap.Error(err)) + } return } @@ -300,6 +305,7 @@ func (u *uploaderImpl) uploadAndDelete(f string) { } } +// startWatch watches for changes in the directory and uploads created files func (u *uploaderImpl) startWatch(ready chan<- struct{}) { defer close(u.watcherDoneCh) @@ -317,7 +323,7 @@ func (u *uploaderImpl) startWatch(ready chan<- struct{}) { } }() - if err := watcher.AddWatch(u.options.Dir, inotifyCloseWrite|inotifyMovedTo); err != nil { + if err := watcher.AddWatch(u.options.Dir, inotifyCloseWrite|inotifyMovedTo|inotifyDelete|inotifyMovedFrom); err != nil { logger.Error("can not start watching", zap.Error(err)) close(ready) return @@ -348,8 +354,19 @@ func (u *uploaderImpl) startWatch(ready chan<- struct{}) { if strings.HasSuffix(event.Name, ".tmp") { continue } + if event.Mask&(inotifyDelete|inotifyMovedFrom) != 0 { + logger.Debug("file deleted or moved away", zap.String("event", event.Name), zap.Uint32("mask", event.Mask)) + continue + } + logger.Debug("received an event", zap.String("event", event.Name), zap.Uint32("mask", event.Mask)) + if _, err := os.Stat(event.Name); os.IsNotExist(err) { + logger.Warn("file does not exist", zap.String("file", event.Name)) + // wait a bit for file operations to complete + time.Sleep(100 * time.Millisecond) + } + u.filesCh <- event.Name case err, ok := <-watcher.Error: if !ok { From c910668a86ee59b9a265dd87c1ea5f6abe4b0a34 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Mon, 23 Jun 2025 00:05:59 +0700 Subject: [PATCH 36/52] fix: wait until uploader finishes Signed-off-by: Alexander Onnikov --- internal/pkg/mediaconvert/transcoder.go | 3 +- internal/pkg/storage/datalake.go | 2 - internal/pkg/uploader/uploader.go | 126 +++++++++++++++--------- 3 files changed, 79 insertions(+), 52 deletions(-) diff --git a/internal/pkg/mediaconvert/transcoder.go b/internal/pkg/mediaconvert/transcoder.go index fca9d02353..46b8206083 100644 --- a/internal/pkg/mediaconvert/transcoder.go +++ b/internal/pkg/mediaconvert/transcoder.go @@ -85,6 +85,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er } defer func() { + logger.Debug("remove temporary folder") if err = os.RemoveAll(destinationFolder); err != nil { logger.Error("failed to cleanup temporary folder", zap.Error(err)) } @@ -218,7 +219,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er } logger.Debug("phase 8: schedule cleanup") - go uploader.Stop() + uploader.Stop() logger.Debug("phase 9: try to set metadata") diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index aaf18ac273..33d8b9e159 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -21,10 +21,8 @@ import ( "io" "mime/multipart" "net/textproto" - "net/url" "os" "path/filepath" - "strconv" "strings" "time" diff --git a/internal/pkg/uploader/uploader.go b/internal/pkg/uploader/uploader.go index 1120751029..e14782f0a7 100644 --- a/internal/pkg/uploader/uploader.go +++ b/internal/pkg/uploader/uploader.go @@ -34,9 +34,9 @@ import ( // See at https://man7.org/linux/man-pages/man7/inotify.7.html const inotifyCloseWrite uint32 = 0x8 // IN_CLOSE_WRITE +const inotifyMovedFrom uint32 = 0x40 // IN_MOVED_FROM const inotifyMovedTo uint32 = 0x80 // IN_MOVED_TO const inotifyDelete uint32 = 0x200 // IN_DELETE -const inotifyMovedFrom uint32 = 0x40 // IN_MOVED_FROM // Uploader represents file uploader type Uploader interface { @@ -104,50 +104,66 @@ func New(ctx context.Context, s storage.Storage, opts Options) Uploader { } func (u *uploaderImpl) Stop() { + u.logger.Info("stopping upload") u.stop(false) } func (u *uploaderImpl) Cancel() { + u.logger.Info("canceling upload") u.stop(true) } -func (u *uploaderImpl) scanInitialFiles() { - u.workerWaitGroup.Add(1) +func (u *uploaderImpl) scanFiles() { + logger := u.logger.With(zap.String("dir", u.options.Dir)) - go func() { - defer u.workerWaitGroup.Done() + logger.Info("scan files") + files, err := os.ReadDir(u.options.Dir) + if err != nil { + logger.Error("failed to read files", zap.Error(err)) + return + } - logger := u.logger.With(zap.String("dir", u.options.Dir)) - - logger.Info("initial file scan") - initFiles, err := os.ReadDir(u.options.Dir) - if err != nil { - logger.Error("failed to read initial files", zap.Error(err)) - return + count := 0 + for _, f := range files { + if f.IsDir() { + continue } - for _, f := range initFiles { - if f.IsDir() { - continue - } + var filePath = filepath.Join(u.options.Dir, f.Name()) - // Ignore source file - var filePath = filepath.Join(u.options.Dir, f.Name()) - if filePath == u.options.SourceFile { - continue - } - u.filesCh <- filePath + // Ignore source file + if filePath == u.options.SourceFile { + continue } - logger.Info("initial file scan complete", zap.Int("count", len(initFiles))) - }() + if _, uploaded := u.sentFiles.Load(filePath); uploaded { + logger.Debug("file already uploaded", zap.String("file", filePath)) + continue + } + + u.filesCh <- filePath + count++ + } + + logger.Info("scan complete", zap.Int("count", count)) } func (u *uploaderImpl) stop(rollback bool) { + // Stop watching for new files close(u.watcherStopCh) <-u.watcherDoneCh - u.logger.Debug("file watch stopped") + // Scan remaining files in the directory + u.scanFiles() + + // Close filesCh so no new files added + close(u.filesCh) + + // Wait for all workers to finish processing + u.workerWaitGroup.Wait() + u.logger.Debug("workers done") + + // Perform rollback if rollback { u.logger.Debug("starting rollback...") var i uint32 @@ -161,15 +177,25 @@ func (u *uploaderImpl) stop(rollback bool) { }) u.logger.Debug("rollback done") } - close(u.filesCh) - u.workerWaitGroup.Wait() - u.logger.Debug("workers done") u.uploadCancel() - _ = os.RemoveAll(u.options.Dir) + + remainingFiles, err := os.ReadDir(u.options.Dir) + if err != nil && !os.IsNotExist(err) { + u.logger.Error("failed to read dir", zap.Error(err)) + } + // log remaining files + if len(remainingFiles) > 0 { + files := make([]string, 0, len(remainingFiles)) + for _, entry := range remainingFiles { + files = append(files, entry.Name()) + } + u.logger.Info("remaining files", zap.Int("count", len(files)), zap.Any("files", files)) + } + u.sentFiles.Clear() - u.logger.Debug("finish done", zap.Bool("cancel", rollback)) + u.logger.Debug("stopped", zap.Bool("rollback", rollback)) } func (u *uploaderImpl) Start() { @@ -180,7 +206,7 @@ func (u *uploaderImpl) Start() { <-watcherReady - u.scanInitialFiles() + u.scanFiles() } func (u *uploaderImpl) startWorkers() { @@ -251,8 +277,7 @@ func (u *uploaderImpl) uploadAndDelete(f string) { return } - // Check if the file exists - if _, err := os.Stat(f); err != nil { + if err := waitFileExists(f); os.IsNotExist(err) { if os.IsNotExist(err) { logger.Debug("file does not exist", zap.Error(err)) } else { @@ -342,18 +367,12 @@ func (u *uploaderImpl) startWatch(ready chan<- struct{}) { logger.Error("file channel was closed") return } - if !strings.Contains(event.Name, u.options.Dir) { - continue - } - if event.Name == u.options.Dir { - continue - } - if event.Name == u.options.SourceFile { - continue - } - if strings.HasSuffix(event.Name, ".tmp") { + if event.Name == u.options.Dir || + event.Name == u.options.SourceFile || + strings.HasSuffix(event.Name, ".tmp") { continue } + if event.Mask&(inotifyDelete|inotifyMovedFrom) != 0 { logger.Debug("file deleted or moved away", zap.String("event", event.Name), zap.Uint32("mask", event.Mask)) continue @@ -361,12 +380,6 @@ func (u *uploaderImpl) startWatch(ready chan<- struct{}) { logger.Debug("received an event", zap.String("event", event.Name), zap.Uint32("mask", event.Mask)) - if _, err := os.Stat(event.Name); os.IsNotExist(err) { - logger.Warn("file does not exist", zap.String("file", event.Name)) - // wait a bit for file operations to complete - time.Sleep(100 * time.Millisecond) - } - u.filesCh <- event.Name case err, ok := <-watcher.Error: if !ok { @@ -376,3 +389,18 @@ func (u *uploaderImpl) startWatch(ready chan<- struct{}) { } } } + +func waitFileExists(file string) error { + var err error + + for range 10 { + stat, err := os.Stat(file) + if err == nil && stat.Size() > 0 { + return nil + } + + time.Sleep(50 * time.Millisecond) + } + + return err +} From 0755e927f56fb89c19d624336841810fb0801f66 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Mon, 23 Jun 2025 00:34:51 +0700 Subject: [PATCH 37/52] fix tests and lint issues Signed-off-by: Alexander Onnikov --- .golangci.yaml | 4 ++-- internal/pkg/mediaconvert/command.go | 24 ++++++++++++++++------- internal/pkg/mediaconvert/command_test.go | 12 ++++++++---- internal/pkg/mediaconvert/transcoder.go | 21 +++++++++++++------- internal/pkg/storage/datalake.go | 2 +- internal/pkg/uploader/uploader.go | 5 +++-- 6 files changed, 45 insertions(+), 23 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 5fd9dcf2b3..ea39029926 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -57,11 +57,11 @@ linters-settings: goimports: local-prefixes: github.com/networkservicemesh/sdk gocyclo: - min-complexity: 20 + min-complexity: 30 dupl: threshold: 150 funlen: - lines: 180 + lines: 200 statements: 100 goconst: min-len: 2 diff --git a/internal/pkg/mediaconvert/command.go b/internal/pkg/mediaconvert/command.go index 2efbf54cf7..88dbb7bb4a 100644 --- a/internal/pkg/mediaconvert/command.go +++ b/internal/pkg/mediaconvert/command.go @@ -29,18 +29,28 @@ import ( "go.uber.org/zap" ) +// LogLevel is ffmpeg log level type LogLevel string const ( - LogLevelQuiet LogLevel = "quiet" - LogLevelPanic LogLevel = "panic" - LogLevelFatal LogLevel = "fatal" - LogLevelError LogLevel = "error" + // LogLevelQuiet is quiet log level + LogLevelQuiet LogLevel = "quiet" + // LogLevelPanic is panic log level + LogLevelPanic LogLevel = "panic" + // LogLevelFatal is fatal log level + LogLevelFatal LogLevel = "fatal" + // LogLevelError is error log level + LogLevelError LogLevel = "error" + // LogLevelWarning is warning log level LogLevelWarning LogLevel = "warning" - LogLevelInfo LogLevel = "info" + // LogLevelInfo is info log level + LogLevelInfo LogLevel = "info" + // LogLevelVerbose is verbose log level LogLevelVerbose LogLevel = "verbose" - LogLevelDebug LogLevel = "debug" - LogLevelTrace LogLevel = "trace" + // LogLevelDebug is debug log level + LogLevelDebug LogLevel = "debug" + // LogLevelTrace is trace log level + LogLevelTrace LogLevel = "trace" ) // Options represents configuration for the ffmpeg command diff --git a/internal/pkg/mediaconvert/command_test.go b/internal/pkg/mediaconvert/command_test.go index 26beed3272..f8b36a7750 100644 --- a/internal/pkg/mediaconvert/command_test.go +++ b/internal/pkg/mediaconvert/command_test.go @@ -28,10 +28,11 @@ func Test_BuildVideoCommand_Scaling(t *testing.T) { Input: "pipe:0", UploadID: "1", Threads: 4, + LogLevel: mediaconvert.LogLevelDebug, ScalingLevels: []string{"720p", "480p"}, }) - const expected = `-threads 4 -i pipe:0 -map 0:v -vf scale=-2:720 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -map 0:v -vf scale=-2:480 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `-y -v debug -threads 4 -i pipe:0 -map 0:v -vf scale=-2:720 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -map 0:v -vf scale=-2:480 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(scaleCommand, " ")) } @@ -42,11 +43,12 @@ func Test_BuildVideoCommand_Scaling_NoRaw(t *testing.T) { Input: "pipe:0", UploadID: "1", Threads: 4, + LogLevel: mediaconvert.LogLevelDebug, Level: "720p", ScalingLevels: []string{"720p", "480p"}, }) - const expected = `-threads 4 -i pipe:0 -map 0:v -vf scale=-2:480 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `-y -v debug -threads 4 -i pipe:0 -map 0:v -vf scale=-2:480 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(scaleCommand, " ")) } @@ -57,11 +59,12 @@ func Test_BuildVideoCommand_Raw_NoTranscode(t *testing.T) { Input: "pipe:0", UploadID: "1", Threads: 4, + LogLevel: mediaconvert.LogLevelDebug, Level: resconv.Level("651:490"), Transcode: false, }) - const expected = `"-threads 4 -i pipe:0 -c:a copy -c:v copy -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `"-y -v debug -threads 4 -i pipe:0 -c:a copy -c:v copy -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(rawCommand, " ")) } @@ -72,11 +75,12 @@ func Test_BuildVideoCommand_Raw_Transcode(t *testing.T) { Input: "pipe:0", UploadID: "1", Threads: 4, + LogLevel: mediaconvert.LogLevelDebug, Level: resconv.Level("651:490"), Transcode: true, }) - const expected = `-threads 4 -i pipe:0 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `-y -v debug -threads 4 -i pipe:0 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(rawCommand, " ")) } diff --git a/internal/pkg/mediaconvert/transcoder.go b/internal/pkg/mediaconvert/transcoder.go index 46b8206083..9667073bf0 100644 --- a/internal/pkg/mediaconvert/transcoder.go +++ b/internal/pkg/mediaconvert/transcoder.go @@ -44,6 +44,7 @@ type Transcoder struct { logger *zap.Logger } +// Command represents a ffmpeg command type Command struct { cmd *exec.Cmd stdoutBuf bytes.Buffer @@ -186,7 +187,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er if cmdErr != nil { logger.Error("can not create a new command", zap.Error(cmdErr), zap.Strings("args", args)) go uploader.Cancel() - return nil, errors.Wrapf(err, "can not create a new command") + return nil, errors.Wrapf(cmdErr, "can not create a new command") } var command = Command{ @@ -209,13 +210,19 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er logger.Debug("phase 7: wait for the result") for _, cmd := range cmds { - if err = cmd.cmd.Wait(); err != nil { - logger.Error("can not wait for command end ", zap.Error(err)) - os.Stdout.Write(cmd.stdoutBuf.Bytes()) - os.Stderr.Write(cmd.stderrBuf.Bytes()) - go uploader.Cancel() - return nil, errors.Wrapf(err, "can not wait for command end") + if err = cmd.cmd.Wait(); err == nil { + continue } + + logger.Error("can not wait for command end ", zap.Error(err)) + if _, err = os.Stdout.Write(cmd.stdoutBuf.Bytes()); err != nil { + logger.Error("can not write stdout ", zap.Error(err)) + } + if _, err = os.Stderr.Write(cmd.stderrBuf.Bytes()); err != nil { + logger.Error("can not write stderr", zap.Error(err)) + } + go uploader.Cancel() + return nil, errors.Wrapf(err, "can not wait for command end") } logger.Debug("phase 8: schedule cleanup") diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index 33d8b9e159..95c8755569 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -285,7 +285,7 @@ func (d *DatalakeStorage) GetFile(ctx context.Context, filename, destination str defer func() { _ = file.Close() }() - if err := resp.BodyWriteTo(file); err != nil { + if err = resp.BodyWriteTo(file); err != nil { logger.Debug("can't write to file", zap.Error(err)) return err } diff --git a/internal/pkg/uploader/uploader.go b/internal/pkg/uploader/uploader.go index e14782f0a7..4e1515c80f 100644 --- a/internal/pkg/uploader/uploader.go +++ b/internal/pkg/uploader/uploader.go @@ -277,7 +277,7 @@ func (u *uploaderImpl) uploadAndDelete(f string) { return } - if err := waitFileExists(f); os.IsNotExist(err) { + if err := waitFileExists(f); err != nil { if os.IsNotExist(err) { logger.Debug("file does not exist", zap.Error(err)) } else { @@ -392,9 +392,10 @@ func (u *uploaderImpl) startWatch(ready chan<- struct{}) { func waitFileExists(file string) error { var err error + var stat os.FileInfo for range 10 { - stat, err := os.Stat(file) + stat, err = os.Stat(file) if err == nil && stat.Size() > 0 { return nil } From cf09ed49b579514148b82b4b4712b7c5b0250ade Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Tue, 24 Jun 2025 09:57:21 +0700 Subject: [PATCH 38/52] fix: adjust transcode result Signed-off-by: Alexander Onnikov --- internal/pkg/mediaconvert/scheduler.go | 20 +++++++++++++++----- internal/pkg/mediaconvert/transcoder.go | 11 ++++++++--- internal/pkg/queue/queue.go | 8 ++++---- internal/pkg/queue/worker.go | 8 +++++++- 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/internal/pkg/mediaconvert/scheduler.go b/internal/pkg/mediaconvert/scheduler.go index 5f33537360..da3190276b 100644 --- a/internal/pkg/mediaconvert/scheduler.go +++ b/internal/pkg/mediaconvert/scheduler.go @@ -47,7 +47,7 @@ type Task struct { // TaskResult represents transcoding task result type TaskResult struct { - Source string `json:"source"` + Playlist string `json:"playlist"` Thumbnail string `json:"thumbnail"` Width int `json:"width"` Height int `json:"height"` @@ -258,19 +258,29 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { logger.Debug("phase 9: try to set metadata") if metaProvider, ok := remoteStorage.(storage.MetaProvider); ok { - var hls = TaskResult{ + var result = TaskResult{ Width: videoStream.Width, Height: videoStream.Height, - Source: task.ID + "_master.m3u8", + Playlist: task.ID + "_master.m3u8", Thumbnail: task.ID + ".jpg", } - logger.Debug("applying metadata", zap.String("url", hls.Source), zap.String("thumbnail", hls.Thumbnail), zap.String("source", task.Source)) + logger.Debug( + "applying metadata", + zap.String("url", result.Playlist), + zap.String("thumbnail", result.Thumbnail), + zap.String("source", task.Source), + ) err = metaProvider.PatchMeta( ctx, task.Source, &storage.Metadata{ - "hls": hls, + "hls": map[string]any{ + "source": result.Playlist, + "thumbnail": result.Thumbnail, + }, + "width": result.Width, + "height": result.Height, }, ) if err != nil { diff --git a/internal/pkg/mediaconvert/transcoder.go b/internal/pkg/mediaconvert/transcoder.go index 9667073bf0..2ca4e3eb35 100644 --- a/internal/pkg/mediaconvert/transcoder.go +++ b/internal/pkg/mediaconvert/transcoder.go @@ -233,14 +233,14 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er var result = TaskResult{ Width: videoStream.Width, Height: videoStream.Height, - Source: task.ID + "_master.m3u8", + Playlist: task.ID + "_master.m3u8", Thumbnail: task.ID + ".jpg", } if metaProvider, ok := remoteStorage.(storage.MetaProvider); ok { logger.Debug( "applying metadata", - zap.String("url", result.Source), + zap.String("url", result.Playlist), zap.String("thumbnail", result.Thumbnail), zap.String("source", task.Source), ) @@ -248,7 +248,12 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er ctx, task.Source, &storage.Metadata{ - "hls": result, + "hls": map[string]any{ + "source": result.Playlist, + "thumbnail": result.Thumbnail, + }, + "width": result.Width, + "height": result.Height, }, ) if err != nil { diff --git a/internal/pkg/queue/queue.go b/internal/pkg/queue/queue.go index 64389e8335..cb5c674519 100644 --- a/internal/pkg/queue/queue.go +++ b/internal/pkg/queue/queue.go @@ -39,10 +39,10 @@ type TranscodeRequest struct { // TranscodeResult represents transcode result. type TranscodeResult struct { - BlobID string - WorkspaceUUID string - Playlist string - Thumbnail string + BlobID string `json:"blobId"` + WorkspaceUUID string `json:"workspaceUuid"` + Playlist string `json:"playlist"` + Thumbnail string `json:"thumbnail"` } // ConsumerOptions represents options for the consumer diff --git a/internal/pkg/queue/worker.go b/internal/pkg/queue/worker.go index 0137f48be9..43e72c822d 100644 --- a/internal/pkg/queue/worker.go +++ b/internal/pkg/queue/worker.go @@ -112,7 +112,13 @@ func (w *Worker) processMessage(ctx context.Context, msg kafka.Message, logger * res, err := transcoder.Transcode(ctx, &task) if err == nil { - if err = w.producer.Send(ctx, req.WorkspaceUUID, res); err != nil { + result := TranscodeResult{ + BlobID: req.BlobID, + WorkspaceUUID: req.WorkspaceUUID, + Playlist: res.Playlist, + Thumbnail: res.Thumbnail, + } + if err = w.producer.Send(ctx, req.WorkspaceUUID, result); err != nil { logger.Error("failed to send transcode result", zap.Error(err)) } } From 4bc1ea0a078632d57b9481c9dd524c832962e8c5 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Tue, 24 Jun 2025 11:09:01 +0700 Subject: [PATCH 39/52] fix: proper upload rollback Signed-off-by: Alexander Onnikov --- internal/pkg/uploader/uploader.go | 42 +++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/internal/pkg/uploader/uploader.go b/internal/pkg/uploader/uploader.go index 4e1515c80f..aad4594e16 100644 --- a/internal/pkg/uploader/uploader.go +++ b/internal/pkg/uploader/uploader.go @@ -165,17 +165,7 @@ func (u *uploaderImpl) stop(rollback bool) { // Perform rollback if rollback { - u.logger.Debug("starting rollback...") - var i uint32 - u.sentFiles.Range(func(key, _ any) bool { - i++ - var filename = key.(string) - u.workersCh[i%u.options.WorkerCount] <- func() { - u.deleteRemoteFile(filename) - } - return true - }) - u.logger.Debug("rollback done") + u.uploadRollback() } u.uploadCancel() @@ -198,6 +188,36 @@ func (u *uploaderImpl) stop(rollback bool) { u.logger.Debug("stopped", zap.Bool("rollback", rollback)) } +func (u *uploaderImpl) uploadRollback() { + u.logger.Debug("starting rollback...") + + // Create a separate worker pool for rollback + var rollbackWg sync.WaitGroup + rollbackCh := make(chan string, u.options.BufferSize) + + // Start rollback workers + for range u.options.WorkerCount { + rollbackWg.Add(1) + go func() { + defer rollbackWg.Done() + for filename := range rollbackCh { + u.deleteRemoteFile(filename) + } + }() + } + + // Send files to rollback + u.sentFiles.Range(func(key, _ any) bool { + rollbackCh <- key.(string) + return true + }) + + // Close channel and wait for rollback to complete + close(rollbackCh) + rollbackWg.Wait() + u.logger.Debug("rollback done") +} + func (u *uploaderImpl) Start() { watcherReady := make(chan struct{}) From b4949c0a369496fd64cf2594343b95edc4590749 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Tue, 24 Jun 2025 11:59:50 +0700 Subject: [PATCH 40/52] fix: nil pointer dereference Signed-off-by: Alexander Onnikov --- internal/pkg/mediaconvert/transcoder.go | 29 +++++++++++++------------ 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/internal/pkg/mediaconvert/transcoder.go b/internal/pkg/mediaconvert/transcoder.go index 2ca4e3eb35..6f476eee8d 100644 --- a/internal/pkg/mediaconvert/transcoder.go +++ b/internal/pkg/mediaconvert/transcoder.go @@ -200,29 +200,30 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er cmd.Stderr = io.MultiWriter(os.Stderr, &command.stderrBuf) cmds = append(cmds, command) - if err = cmd.Start(); err != nil { - logger.Error("can not start a command", zap.Error(err), zap.Strings("args", args)) + if startErr := cmd.Start(); startErr != nil { + logger.Error("can not start a command", zap.Error(startErr), zap.Strings("args", args)) go uploader.Cancel() - return nil, errors.Wrapf(err, "can not start a command") + return nil, errors.Wrapf(startErr, "can not start a command") } } logger.Debug("phase 7: wait for the result") for _, cmd := range cmds { - if err = cmd.cmd.Wait(); err == nil { + var cmdErr error + if cmdErr = cmd.cmd.Wait(); cmdErr == nil { continue } - logger.Error("can not wait for command end ", zap.Error(err)) - if _, err = os.Stdout.Write(cmd.stdoutBuf.Bytes()); err != nil { - logger.Error("can not write stdout ", zap.Error(err)) + logger.Error("can not wait for command end ", zap.Error(cmdErr)) + if _, writeErr := os.Stdout.Write(cmd.stdoutBuf.Bytes()); writeErr != nil { + logger.Error("can not write stdout ", zap.Error(writeErr)) } - if _, err = os.Stderr.Write(cmd.stderrBuf.Bytes()); err != nil { - logger.Error("can not write stderr", zap.Error(err)) + if _, writeErr := os.Stderr.Write(cmd.stderrBuf.Bytes()); writeErr != nil { + logger.Error("can not write stderr", zap.Error(writeErr)) } - go uploader.Cancel() - return nil, errors.Wrapf(err, "can not wait for command end") + uploader.Cancel() + return nil, errors.Wrapf(cmdErr, "can not wait for command end") } logger.Debug("phase 8: schedule cleanup") @@ -244,7 +245,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er zap.String("thumbnail", result.Thumbnail), zap.String("source", task.Source), ) - err = metaProvider.PatchMeta( + metaErr := metaProvider.PatchMeta( ctx, task.Source, &storage.Metadata{ @@ -256,8 +257,8 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er "height": result.Height, }, ) - if err != nil { - logger.Error("can not patch the source file", zap.Error(err)) + if metaErr != nil { + logger.Error("can not patch the source file", zap.Error(metaErr)) } } From 960eceb4af3e50950a0d7cb91f7fb39f1491aae7 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Tue, 24 Jun 2025 16:15:02 +0700 Subject: [PATCH 41/52] fix: ensure correct ffmpeg commands Signed-off-by: Alexander Onnikov --- .golangci.yaml | 4 ++-- internal/pkg/mediaconvert/command.go | 15 +++++++++++++- internal/pkg/mediaconvert/command_test.go | 22 +++++++++++++++++---- internal/pkg/mediaconvert/transcoder.go | 19 +++++++++++------- internal/pkg/queue/worker.go | 2 +- internal/pkg/uploader/uploader.go | 24 ++++++++++++----------- 6 files changed, 60 insertions(+), 26 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index ea39029926..3a6223134d 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -61,8 +61,8 @@ linters-settings: dupl: threshold: 150 funlen: - lines: 200 - statements: 100 + lines: 240 + statements: 120 goconst: min-len: 2 min-occurrences: 2 diff --git a/internal/pkg/mediaconvert/command.go b/internal/pkg/mediaconvert/command.go index 88dbb7bb4a..3b02624031 100644 --- a/internal/pkg/mediaconvert/command.go +++ b/internal/pkg/mediaconvert/command.go @@ -84,6 +84,8 @@ func buildCommonCommand(opts *Options) []string { var result = []string{ "-y", // Overwrite output files without asking. "-v", string(opts.LogLevel), + "-err_detect", "ignore_err", + "-fflags", "+discardcorrupt", "-threads", fmt.Sprint(opts.Threads), "-i", opts.Input, } @@ -113,6 +115,8 @@ func BuildAudioCommand(opts *Options) []string { func BuildRawVideoCommand(opts *Options) []string { if opts.Transcode { return append(buildCommonCommand(opts), + "-map", "0:v:0", + "-map", "0:a?", "-c:a", "aac", "-c:v", "libx264", "-preset", "veryfast", @@ -149,6 +153,14 @@ func BuildThumbnailCommand(opts *Options) []string { // BuildScalingVideoCommand returns flags for ffmpeg for video scaling func BuildScalingVideoCommand(opts *Options) []string { + if len(opts.ScalingLevels) == 0 { + return []string{} + } + + if len(opts.ScalingLevels) == 1 && opts.ScalingLevels[0] == opts.Level { + return []string{} + } + var result = buildCommonCommand(opts) for _, level := range opts.ScalingLevels { @@ -157,7 +169,8 @@ func BuildScalingVideoCommand(opts *Options) []string { } result = append(result, - "-map", "0:v", + "-map", "0:v:0", + "-map", "0:a?", "-vf", "scale=-2:"+level[:len(level)-1], "-c:a", "aac", "-c:v", "libx264", diff --git a/internal/pkg/mediaconvert/command_test.go b/internal/pkg/mediaconvert/command_test.go index f8b36a7750..0b83a79ab6 100644 --- a/internal/pkg/mediaconvert/command_test.go +++ b/internal/pkg/mediaconvert/command_test.go @@ -32,7 +32,7 @@ func Test_BuildVideoCommand_Scaling(t *testing.T) { ScalingLevels: []string{"720p", "480p"}, }) - const expected = `-y -v debug -threads 4 -i pipe:0 -map 0:v -vf scale=-2:720 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -map 0:v -vf scale=-2:480 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -vf scale=-2:720 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -map 0:v:0 -map 0:a? -vf scale=-2:480 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(scaleCommand, " ")) } @@ -48,7 +48,7 @@ func Test_BuildVideoCommand_Scaling_NoRaw(t *testing.T) { ScalingLevels: []string{"720p", "480p"}, }) - const expected = `-y -v debug -threads 4 -i pipe:0 -map 0:v -vf scale=-2:480 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -vf scale=-2:480 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(scaleCommand, " ")) } @@ -64,7 +64,7 @@ func Test_BuildVideoCommand_Raw_NoTranscode(t *testing.T) { Transcode: false, }) - const expected = `"-y -v debug -threads 4 -i pipe:0 -c:a copy -c:v copy -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `"-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -c:a copy -c:v copy -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(rawCommand, " ")) } @@ -80,7 +80,21 @@ func Test_BuildVideoCommand_Raw_Transcode(t *testing.T) { Transcode: true, }) - const expected = `-y -v debug -threads 4 -i pipe:0 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` require.Contains(t, expected, strings.Join(rawCommand, " ")) } + +func Test_BuildVideoCommand_Scaling_Small(t *testing.T) { + var scaleCommand = mediaconvert.BuildScalingVideoCommand(&mediaconvert.Options{ + OutputDir: "test", + Input: "pipe:0", + UploadID: "1", + Threads: 4, + LogLevel: mediaconvert.LogLevelDebug, + Level: "360p", + ScalingLevels: []string{"360p"}, + }) + + require.Empty(t, scaleCommand) +} diff --git a/internal/pkg/mediaconvert/transcoder.go b/internal/pkg/mediaconvert/transcoder.go index 6f476eee8d..7d8c0316d4 100644 --- a/internal/pkg/mediaconvert/transcoder.go +++ b/internal/pkg/mediaconvert/transcoder.go @@ -47,8 +47,8 @@ type Transcoder struct { // Command represents a ffmpeg command type Command struct { cmd *exec.Cmd - stdoutBuf bytes.Buffer - stderrBuf bytes.Buffer + stdoutBuf *bytes.Buffer + stderrBuf *bytes.Buffer } // NewTranscoder creates a new instance of task transcoder @@ -183,6 +183,11 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er var cmds []Command for _, args := range argsSlice { + if len(args) == 0 { + logger.Debug("skip empty command") + continue + } + cmd, cmdErr := newFfmpegCommand(ctx, nil, args) if cmdErr != nil { logger.Error("can not create a new command", zap.Error(cmdErr), zap.Strings("args", args)) @@ -192,12 +197,12 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er var command = Command{ cmd: cmd, - stdoutBuf: bytes.Buffer{}, - stderrBuf: bytes.Buffer{}, + stdoutBuf: &bytes.Buffer{}, + stderrBuf: &bytes.Buffer{}, } - cmd.Stdout = io.MultiWriter(os.Stdout, &command.stdoutBuf) - cmd.Stderr = io.MultiWriter(os.Stderr, &command.stderrBuf) + cmd.Stdout = io.MultiWriter(os.Stdout, command.stdoutBuf) + cmd.Stderr = io.MultiWriter(os.Stderr, command.stderrBuf) cmds = append(cmds, command) if startErr := cmd.Start(); startErr != nil { @@ -215,7 +220,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er continue } - logger.Error("can not wait for command end ", zap.Error(cmdErr)) + logger.Error("can not wait for command end", zap.Error(cmdErr), zap.String("cmd", cmd.cmd.String())) if _, writeErr := os.Stdout.Write(cmd.stdoutBuf.Bytes()); writeErr != nil { logger.Error("can not write stdout ", zap.Error(writeErr)) } diff --git a/internal/pkg/queue/worker.go b/internal/pkg/queue/worker.go index 43e72c822d..7c9dae02b4 100644 --- a/internal/pkg/queue/worker.go +++ b/internal/pkg/queue/worker.go @@ -85,7 +85,7 @@ func (w *Worker) fetchAndProcessMessage(ctx context.Context) error { err = w.processMessage(ctx, msg, logger) if err != nil { w.logger.Error("failed to process message", zap.Error(err)) - return err + return fmt.Errorf("process message: %w", err) } return nil diff --git a/internal/pkg/uploader/uploader.go b/internal/pkg/uploader/uploader.go index aad4594e16..baffcaa82e 100644 --- a/internal/pkg/uploader/uploader.go +++ b/internal/pkg/uploader/uploader.go @@ -297,6 +297,13 @@ func (u *uploaderImpl) uploadAndDelete(f string) { return } + // Check if the file has already been uploaded + _, ok := u.sentFiles.Load(f) + if ok && !u.shouldDeleteOnStop(f) { + logger.Debug("file already uploaded") + return + } + if err := waitFileExists(f); err != nil { if os.IsNotExist(err) { logger.Debug("file does not exist", zap.Error(err)) @@ -306,18 +313,11 @@ func (u *uploaderImpl) uploadAndDelete(f string) { return } - // Check if the file has already been uploaded - var _, ok = u.sentFiles.Load(f) - if ok && !u.shouldDeleteOnStop(f) { - logger.Debug("file already uploaded") - return - } - for attempt := range u.options.RetryCount { logger = logger.With(zap.Int("attempt", attempt)) - var ctx, cancel = context.WithTimeout(u.uploadCtx, u.options.Timeout) - var err = u.storage.PutFile(ctx, f) - cancel() + var putCtx, putCancel = context.WithTimeout(u.uploadCtx, u.options.Timeout) + var err = u.storage.PutFile(putCtx, f) + putCancel() if err != nil { logger.Error("attempt failed", zap.Error(err)) @@ -328,7 +328,9 @@ func (u *uploaderImpl) uploadAndDelete(f string) { // Update the file's parent if SourceFile is set if u.options.Source != "" { - err = u.storage.SetParent(ctx, f, u.options.Source) + var setParentCtx, setParentCancel = context.WithTimeout(u.uploadCtx, u.options.Timeout) + err = u.storage.SetParent(setParentCtx, f, u.options.Source) + setParentCancel() if err != nil { logger.Error("can not set blob parent", zap.Error(err), zap.String("filename", f), zap.String("source", u.options.Source)) } From 86ad2921f20529ed98b081e344c27acc9b724299 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Tue, 24 Jun 2025 18:00:18 +0700 Subject: [PATCH 42/52] fix: extract command executor to separate class Signed-off-by: Alexander Onnikov --- internal/pkg/mediaconvert/executor.go | 98 ++++++++++++++ internal/pkg/mediaconvert/executor_test.go | 142 +++++++++++++++++++++ internal/pkg/mediaconvert/transcoder.go | 50 +------- 3 files changed, 247 insertions(+), 43 deletions(-) create mode 100644 internal/pkg/mediaconvert/executor.go create mode 100644 internal/pkg/mediaconvert/executor_test.go diff --git a/internal/pkg/mediaconvert/executor.go b/internal/pkg/mediaconvert/executor.go new file mode 100644 index 0000000000..6385160aa7 --- /dev/null +++ b/internal/pkg/mediaconvert/executor.go @@ -0,0 +1,98 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package mediaconvert + +import ( + "bytes" + "context" + "io" + "os" + "os/exec" + "sync" + + "github.com/hcengineering/stream/internal/pkg/log" + "go.uber.org/zap" +) + +// CommandExecutor executes multiple commands in parallel +type CommandExecutor interface { + Execute(commands []*exec.Cmd) error +} + +type commandExecutor struct { + logger *zap.Logger +} + +var _ CommandExecutor = (*commandExecutor)(nil) + +// NewCommandExecutor creates a new instance of command executor +func NewCommandExecutor(ctx context.Context) CommandExecutor { + return &commandExecutor{ + logger: log.FromContext(ctx), + } +} + +// Execute executes multiple commands in parallel +func (e *commandExecutor) Execute(commands []*exec.Cmd) error { + logger := e.logger + errCh := make(chan error, len(commands)) + + var mu sync.Mutex + var wg sync.WaitGroup + for _, cmd := range commands { + wg.Add(1) + + go func(cmd *exec.Cmd) { + defer wg.Done() + + var stdoutBuf = &bytes.Buffer{} + var stderrBuf = &bytes.Buffer{} + + if logger.Core().Enabled(zap.DebugLevel) { + cmd.Stdout = io.MultiWriter(os.Stdout, stdoutBuf) + cmd.Stderr = io.MultiWriter(os.Stderr, stderrBuf) + } else { + cmd.Stdout = stdoutBuf + cmd.Stderr = stderrBuf + } + + logger.Info("run command", zap.String("cmd", cmd.String())) + if err := cmd.Run(); err != nil { + errCh <- err + + // Lock so only on goroutine can write to stdout/stderr at the same time + mu.Lock() + defer mu.Unlock() + logger.Error("can not wait for command end", zap.Error(err), zap.String("cmd", cmd.String())) + if _, writeErr := os.Stdout.Write(stdoutBuf.Bytes()); writeErr != nil { + logger.Error("can not write stdout ", zap.Error(writeErr)) + } + if _, writeErr := os.Stderr.Write(stderrBuf.Bytes()); writeErr != nil { + logger.Error("can not write stderr", zap.Error(writeErr)) + } + } + }(cmd) + } + + wg.Wait() + close(errCh) + + for err := range errCh { + return err + } + + return nil +} diff --git a/internal/pkg/mediaconvert/executor_test.go b/internal/pkg/mediaconvert/executor_test.go new file mode 100644 index 0000000000..3f30a82cf9 --- /dev/null +++ b/internal/pkg/mediaconvert/executor_test.go @@ -0,0 +1,142 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package mediaconvert_test + +import ( + "context" + "os/exec" + "testing" + "time" + + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/mediaconvert" + "github.com/stretchr/testify/assert" +) + +func TestCommandExecutor_Execute_Success(t *testing.T) { + tests := []struct { + name string + commands []*exec.Cmd + }{ + { + name: "single command", + commands: []*exec.Cmd{ + exec.Command("echo", "test"), + }, + }, + { + name: "multiple commands", + commands: []*exec.Cmd{ + exec.Command("echo", "test1"), + exec.Command("echo", "test2"), + exec.Command("echo", "test3"), + }, + }, + { + name: "empty commands", + commands: []*exec.Cmd{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + ctx = log.WithFields(ctx) + executor := mediaconvert.NewCommandExecutor(ctx) + + err := executor.Execute(tt.commands) + assert.NoError(t, err) + }) + } +} + +func TestCommandExecutor_Execute_Error(t *testing.T) { + tests := []struct { + name string + commands []*exec.Cmd + expectedError bool + }{ + { + name: "single failing command", + commands: []*exec.Cmd{ + exec.Command("false"), + }, + expectedError: true, + }, + { + name: "mixed success and failure", + commands: []*exec.Cmd{ + exec.Command("echo", "success"), + exec.Command("false"), + exec.Command("echo", "another success"), + }, + expectedError: true, + }, + { + name: "non-existent command", + commands: []*exec.Cmd{ + exec.Command("this-command-does-not-exist"), + }, + expectedError: true, + }, + { + name: "multiple failures", + commands: []*exec.Cmd{ + exec.Command("false"), + exec.Command("false"), + exec.Command("false"), + }, + expectedError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + ctx = log.WithFields(ctx) + executor := mediaconvert.NewCommandExecutor(ctx) + + err := executor.Execute(tt.commands) + if tt.expectedError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestCommandExecutor_Execute_Parallel(t *testing.T) { + // Create commands that sleep for different durations + commands := []*exec.Cmd{ + exec.Command("sleep", "0.1"), + exec.Command("sleep", "0.1"), + exec.Command("sleep", "0.1"), + exec.Command("sleep", "0.1"), + } + + ctx := context.Background() + ctx = log.WithFields(ctx) + executor := mediaconvert.NewCommandExecutor(ctx) + + start := time.Now() + err := executor.Execute(commands) + duration := time.Since(start) + + assert.NoError(t, err) + // If commands run in parallel, total time should be close to 0.1s, not 0.4s + assert.Less(t, duration, 200*time.Millisecond) +} diff --git a/internal/pkg/mediaconvert/transcoder.go b/internal/pkg/mediaconvert/transcoder.go index 7d8c0316d4..f85d034532 100644 --- a/internal/pkg/mediaconvert/transcoder.go +++ b/internal/pkg/mediaconvert/transcoder.go @@ -16,10 +16,8 @@ package mediaconvert import ( - "bytes" "context" "fmt" - "io" "os" "os/exec" "path/filepath" @@ -44,13 +42,6 @@ type Transcoder struct { logger *zap.Logger } -// Command represents a ffmpeg command -type Command struct { - cmd *exec.Cmd - stdoutBuf *bytes.Buffer - stderrBuf *bytes.Buffer -} - // NewTranscoder creates a new instance of task transcoder func NewTranscoder(ctx context.Context, cfg *config.Config) *Transcoder { var p = &Transcoder{ @@ -180,8 +171,8 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er BuildRawVideoCommand(&opts), BuildScalingVideoCommand(&opts), } - var cmds []Command + var cmds []*exec.Cmd for _, args := range argsSlice { if len(args) == 0 { logger.Debug("skip empty command") @@ -195,46 +186,19 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er return nil, errors.Wrapf(cmdErr, "can not create a new command") } - var command = Command{ - cmd: cmd, - stdoutBuf: &bytes.Buffer{}, - stderrBuf: &bytes.Buffer{}, - } - - cmd.Stdout = io.MultiWriter(os.Stdout, command.stdoutBuf) - cmd.Stderr = io.MultiWriter(os.Stderr, command.stderrBuf) - - cmds = append(cmds, command) - if startErr := cmd.Start(); startErr != nil { - logger.Error("can not start a command", zap.Error(startErr), zap.Strings("args", args)) - go uploader.Cancel() - return nil, errors.Wrapf(startErr, "can not start a command") - } + cmds = append(cmds, cmd) } - logger.Debug("phase 7: wait for the result") - - for _, cmd := range cmds { - var cmdErr error - if cmdErr = cmd.cmd.Wait(); cmdErr == nil { - continue - } - - logger.Error("can not wait for command end", zap.Error(cmdErr), zap.String("cmd", cmd.cmd.String())) - if _, writeErr := os.Stdout.Write(cmd.stdoutBuf.Bytes()); writeErr != nil { - logger.Error("can not write stdout ", zap.Error(writeErr)) - } - if _, writeErr := os.Stderr.Write(cmd.stderrBuf.Bytes()); writeErr != nil { - logger.Error("can not write stderr", zap.Error(writeErr)) - } + executor := NewCommandExecutor(ctx) + if execErr := executor.Execute(cmds); execErr != nil { uploader.Cancel() - return nil, errors.Wrapf(cmdErr, "can not wait for command end") + return nil, errors.Wrapf(execErr, "can not execute command") } - logger.Debug("phase 8: schedule cleanup") + logger.Debug("phase 7: schedule cleanup") uploader.Stop() - logger.Debug("phase 9: try to set metadata") + logger.Debug("phase 8: try to set metadata") var result = TaskResult{ Width: videoStream.Width, From 0cb756d440f8699bc190f2dfde88de928aa793a9 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Thu, 26 Jun 2025 20:04:19 +0700 Subject: [PATCH 43/52] feat: fast live recording Signed-off-by: Alexander Onnikov --- internal/pkg/api/v1/transcoding/handler.go | 2 +- internal/pkg/config/config.go | 18 +- internal/pkg/manifest/hls.go | 13 +- internal/pkg/manifest/hls_test.go | 20 ++- internal/pkg/mediaconvert/command.go | 153 ++++++++--------- internal/pkg/mediaconvert/command_test.go | 98 ++++++----- internal/pkg/mediaconvert/coordinator.go | 117 +++++++++---- internal/pkg/mediaconvert/multipart.go | 188 +++++++++++++++++++++ internal/pkg/mediaconvert/scheduler.go | 41 +++-- internal/pkg/mediaconvert/strategy.go | 67 ++++++++ internal/pkg/mediaconvert/stream.go | 172 ++++++++++++++----- internal/pkg/mediaconvert/transcoder.go | 35 ++-- internal/pkg/profile/profile.go | 185 ++++++++++++++++++++ internal/pkg/profile/profile_test.go | 173 +++++++++++++++++++ internal/pkg/queue/queue.go | 6 + internal/pkg/queue/worker.go | 2 +- internal/pkg/storage/datalake.go | 137 +++++++++++++++ internal/pkg/storage/storage.go | 14 ++ 18 files changed, 1169 insertions(+), 272 deletions(-) create mode 100644 internal/pkg/mediaconvert/multipart.go create mode 100644 internal/pkg/mediaconvert/strategy.go create mode 100644 internal/pkg/profile/profile.go create mode 100644 internal/pkg/profile/profile_test.go diff --git a/internal/pkg/api/v1/transcoding/handler.go b/internal/pkg/api/v1/transcoding/handler.go index c3a3ecfc92..b9871f7e12 100644 --- a/internal/pkg/api/v1/transcoding/handler.go +++ b/internal/pkg/api/v1/transcoding/handler.go @@ -68,7 +68,7 @@ func (t *trascodeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } -// NewHandler creates a new trnascoding http handler, requires context and config. +// NewHandler creates a new transcoding http handler, requires context and config. func NewHandler(ctx context.Context, cfg *config.Config) http.Handler { return &trascodeHandler{ scheduler: mediaconvert.NewScheduler(ctx, cfg), diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index f182a61075..ca0e4aef22 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -25,15 +25,15 @@ import ( // Config represents configuration for the huly-stream application. type Config struct { - SentryDsn string `split_words:"true" default:"" desc:"sentry dsn value"` - LogLevel string `split_words:"true" default:"debug" desc:"sets log level for the application"` - ServerSecret string `split_words:"true" default:"" desc:"server secret required to generate and verify tokens"` - PprofEnabled bool `split_words:"true" default:"true" desc:"starts profile server on localhost:6060 if true"` - Insecure bool `split_words:"true" default:"false" desc:"ignores authorization check if true"` - ServeURL string `split_words:"true" desc:"listen on url" default:"0.0.0.0:1080"` - EndpointURL *url.URL `split_words:"true" default:"s3://127.0.0.1:9000" desc:"S3 or Datalake endpoint, example: s3://my-ip-address, datalake://my-ip-address"` - MaxParallelScalingCount int `split_words:"true" default:"2" desc:"how much parallel scaling can be processed"` - MaxThreadCount int `split_words:"true" default:"4" desc:"max number of threads for transcoder"` + SentryDsn string `split_words:"true" default:"" desc:"sentry dsn value"` + LogLevel string `split_words:"true" default:"debug" desc:"sets log level for the application"` + ServerSecret string `split_words:"true" default:"" desc:"server secret required to generate and verify tokens"` + PprofEnabled bool `split_words:"true" default:"true" desc:"starts profile server on localhost:6060 if true"` + Insecure bool `split_words:"true" default:"false" desc:"ignores authorization check if true"` + ServeURL string `split_words:"true" desc:"listen on url" default:"0.0.0.0:1080"` + EndpointURL *url.URL `split_words:"true" default:"s3://127.0.0.1:9000" desc:"S3 or Datalake endpoint, example: s3://my-ip-address, datalake://my-ip-address"` + MaxParallelTranscodingCount int `split_words:"true" default:"2" desc:"how much parallel transcodings can be processed"` + MaxThreadCount int `split_words:"true" default:"4" desc:"max number of threads for transcoder"` QueueConfig string `split_words:"true" default:"" desc:"queue config"` Region string `split_words:"true" default:"" desc:"service region"` diff --git a/internal/pkg/manifest/hls.go b/internal/pkg/manifest/hls.go index dcb421fe50..7b2934ea46 100644 --- a/internal/pkg/manifest/hls.go +++ b/internal/pkg/manifest/hls.go @@ -18,13 +18,12 @@ import ( "fmt" "os" "path/filepath" - "strings" - "github.com/hcengineering/stream/internal/pkg/resconv" + "github.com/hcengineering/stream/internal/pkg/profile" ) // GenerateHLSPlaylist generates master file for master files for resolution levels -func GenerateHLSPlaylist(levels []string, outputPath, uploadID string) error { +func GenerateHLSPlaylist(profiles []profile.VideoProfile, outputPath, uploadID string) error { p := filepath.Join(outputPath, uploadID, fmt.Sprintf("%v_master.m3u8", uploadID)) d := filepath.Dir(p) _ = os.MkdirAll(d, os.ModePerm) @@ -40,16 +39,16 @@ func GenerateHLSPlaylist(levels []string, outputPath, uploadID string) error { return err } - for _, res := range levels { - var bandwidth = resconv.Bandwidth(res) - var resolution = strings.ReplaceAll(resconv.Resolution(res), ":", "x") + for _, profile := range profiles { + var bandwidth = profile.Bandwidth + var resolution = fmt.Sprintf("%vx%v", profile.Width, profile.Height) _, err = file.WriteString(fmt.Sprintf("#EXT-X-STREAM-INF:BANDWIDTH=%d,RESOLUTION=%v\n", bandwidth, resolution)) if err != nil { return err } - _, err = file.WriteString(fmt.Sprintf("%s_%s_master.m3u8\n", uploadID, res)) + _, err = file.WriteString(fmt.Sprintf("%s_%s_master.m3u8\n", uploadID, profile.Name)) if err != nil { return err } diff --git a/internal/pkg/manifest/hls_test.go b/internal/pkg/manifest/hls_test.go index a37c63b874..5ad6dd6758 100644 --- a/internal/pkg/manifest/hls_test.go +++ b/internal/pkg/manifest/hls_test.go @@ -19,14 +19,24 @@ import ( "testing" "github.com/hcengineering/stream/internal/pkg/manifest" + "github.com/hcengineering/stream/internal/pkg/profile" "github.com/stretchr/testify/require" ) func TestGenerateHLSPlaylist(t *testing.T) { - resolutions := []string{"320p", "480p", "720p", "1080p", "4k", "8k"} + profiles := []profile.VideoProfile{ + profile.Profile360p, + profile.Profile480p, + profile.Profile720p, + profile.Profile1080p, + profile.Profile1440p, + } uploadID := "test123" + defer func() { + _ = os.RemoveAll(uploadID) + }() - err := manifest.GenerateHLSPlaylist(resolutions, "", uploadID) + err := manifest.GenerateHLSPlaylist(profiles, "", uploadID) require.NoError(t, err) outputPath := filepath.Join(uploadID, uploadID+"_master.m3u8") @@ -42,10 +52,8 @@ func TestGenerateHLSPlaylist(t *testing.T) { require.Contains(t, playlistContent, "#EXTM3U", "File must start with #EXTM3U") - for _, res := range resolutions { - expectedLine := uploadID + "_" + res + "_master.m3u8" + for _, prof := range profiles { + expectedLine := uploadID + "_" + prof.Name + "_master.m3u8" require.Contains(t, playlistContent, expectedLine, "Missing expected reference: "+expectedLine) } - - _ = os.RemoveAll(uploadID) } diff --git a/internal/pkg/mediaconvert/command.go b/internal/pkg/mediaconvert/command.go index 3b02624031..895b5e08c2 100644 --- a/internal/pkg/mediaconvert/command.go +++ b/internal/pkg/mediaconvert/command.go @@ -21,11 +21,13 @@ import ( "io" "os/exec" "path/filepath" + "strconv" "strings" "github.com/pkg/errors" "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/profile" "go.uber.org/zap" ) @@ -55,14 +57,12 @@ const ( // Options represents configuration for the ffmpeg command type Options struct { - Input string - OutputDir string - ScalingLevels []string - Level string - LogLevel LogLevel - Transcode bool - Threads int - UploadID string + Input string + OutputDir string + LogLevel LogLevel + Threads int + UploadID string + Profiles []profile.VideoProfile } func newFfmpegCommand(ctx context.Context, in io.Reader, args []string) (*exec.Cmd, error) { @@ -70,14 +70,13 @@ func newFfmpegCommand(ctx context.Context, in io.Reader, args []string) (*exec.C return nil, errors.New("ctx should not be nil") } + var cmd = exec.CommandContext(ctx, "ffmpeg", args...) + cmd.Stdin = in + var logger = log.FromContext(ctx).With(zap.String("func", "newFFMpegCommand")) + logger.Debug("prepared command: ", zap.String("cmd", cmd.String())) - logger.Debug("prepared command: ", zap.Strings("args", args)) - - var result = exec.CommandContext(ctx, "ffmpeg", args...) - result.Stdin = in - - return result, nil + return cmd, nil } func buildCommonCommand(opts *Options) []string { @@ -90,6 +89,7 @@ func buildCommonCommand(opts *Options) []string { "-i", opts.Input, } + // If input is a URL, add HTTP specific parameters if strings.HasPrefix(opts.Input, "http://") || strings.HasPrefix(opts.Input, "https://") { result = append(result, "-reconnect", "1", @@ -101,6 +101,49 @@ func buildCommonCommand(opts *Options) []string { return result } +func buildHLSCommand(profile profile.VideoProfile, opts *Options) []string { + return []string{ + "-f", "hls", + "-hls_time", "5", + // Use HLS flags + // - split_by_time + // Allow segments to start on frames other than key frames. + // This improves behavior on some players when the time between key frames is inconsistent, + // but may make things worse on others, and can cause some oddities during seeking. + // This flag should be used with the hls_time option. + // - temp_file + // Write segment data to filename.tmp and rename to filename only once the segment is complete. + "-hls_flags", "split_by_time+temp_file", + // Do not limit number of HLS segments + "-hls_list_size", "0", + "-hls_segment_filename", filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", profile.Name)), + } +} + +func buildVideoCommand(profile profile.VideoProfile, opts *Options) []string { + crf := profile.CRF + if crf == 0 { + crf = 23 + } + command := []string{ + // Transcode only first video and optionally audio stream + "-map", "0:v:0", + "-map", "0:a?", + // Set up codecs + "-c:a", profile.AudioCodec, + "-c:v", profile.VideoCodec, + "-preset", "veryfast", + "-crf", strconv.Itoa(crf), + "-g", "60", + } + + if profile.VideoCodec != "copy" && profile.Scale { + command = append(command, "-vf", "scale=-2:"+strconv.Itoa(profile.Height)) + } + + return command +} + // BuildAudioCommand returns flags for getting the audio from the input func BuildAudioCommand(opts *Options) []string { var commonPart = buildCommonCommand(opts) @@ -111,34 +154,19 @@ func BuildAudioCommand(opts *Options) []string { ) } -// BuildRawVideoCommand returns an extremely lightweight ffmpeg command for converting raw video without extra cost. -func BuildRawVideoCommand(opts *Options) []string { - if opts.Transcode { - return append(buildCommonCommand(opts), - "-map", "0:v:0", - "-map", "0:a?", - "-c:a", "aac", - "-c:v", "libx264", - "-preset", "veryfast", - "-crf", "23", - "-g", "60", - "-f", "hls", - "-hls_time", "5", - "-hls_flags", "split_by_time+temp_file", - "-hls_list_size", "0", - "-hls_segment_filename", filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", opts.Level)), - filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, opts.Level))) +// BuildVideoCommand returns ffmpeg command for converting video. +func BuildVideoCommand(opts *Options) []string { + if len(opts.Profiles) == 0 { + return []string{} } - return append(buildCommonCommand(opts), - "-c:a", "copy", // Copy audio stream - "-c:v", "copy", // Copy video stream - "-f", "hls", - "-hls_time", "5", - "-hls_flags", "split_by_time+temp_file", - "-hls_list_size", "0", - "-hls_segment_filename", filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", opts.Level)), - filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, opts.Level))) + var command = buildCommonCommand(opts) + for _, profile := range opts.Profiles { + command = append(command, buildVideoCommand(profile, opts)...) + command = append(command, buildHLSCommand(profile, opts)...) + command = append(command, filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, profile.Name))) + } + return command } // BuildThumbnailCommand creates a command that creates a thumbnail for the input video @@ -150,48 +178,3 @@ func BuildThumbnailCommand(opts *Options) []string { filepath.Join(opts.OutputDir, opts.UploadID, opts.UploadID+".jpg"), ) } - -// BuildScalingVideoCommand returns flags for ffmpeg for video scaling -func BuildScalingVideoCommand(opts *Options) []string { - if len(opts.ScalingLevels) == 0 { - return []string{} - } - - if len(opts.ScalingLevels) == 1 && opts.ScalingLevels[0] == opts.Level { - return []string{} - } - - var result = buildCommonCommand(opts) - - for _, level := range opts.ScalingLevels { - if level == opts.Level { - continue - } - - result = append(result, - "-map", "0:v:0", - "-map", "0:a?", - "-vf", "scale=-2:"+level[:len(level)-1], - "-c:a", "aac", - "-c:v", "libx264", - "-preset", "veryfast", - "-crf", "23", - "-g", "60", - "-f", "hls", - "-hls_time", "5", - // Use HLS flags - // - split_by_time - // Allow segments to start on frames other than key frames. - // This improves behavior on some players when the time between key frames is inconsistent, - // but may make things worse on others, and can cause some oddities during seeking. - // This flag should be used with the hls_time option. - // - temp_file - // Write segment data to filename.tmp and rename to filename only once the segment is complete. - "-hls_flags", "split_by_time+temp_file", - "-hls_list_size", "0", - "-hls_segment_filename", filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", level)), - filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, level))) - } - - return result -} diff --git a/internal/pkg/mediaconvert/command_test.go b/internal/pkg/mediaconvert/command_test.go index 0b83a79ab6..67beff10b4 100644 --- a/internal/pkg/mediaconvert/command_test.go +++ b/internal/pkg/mediaconvert/command_test.go @@ -18,83 +18,79 @@ import ( "testing" "github.com/hcengineering/stream/internal/pkg/mediaconvert" - "github.com/hcengineering/stream/internal/pkg/resconv" + "github.com/hcengineering/stream/internal/pkg/profile" "github.com/stretchr/testify/require" ) +func Test_BuildVideoCommand_Empty(t *testing.T) { + var profiles []profile.VideoProfile + + var rawCommand = mediaconvert.BuildVideoCommand(&mediaconvert.Options{ + OutputDir: "test", + Input: "pipe:0", + UploadID: "1", + Threads: 4, + LogLevel: mediaconvert.LogLevelDebug, + Profiles: profiles, + }) + + require.Empty(t, rawCommand) +} + func Test_BuildVideoCommand_Scaling(t *testing.T) { - var scaleCommand = mediaconvert.BuildScalingVideoCommand(&mediaconvert.Options{ - OutputDir: "test", - Input: "pipe:0", - UploadID: "1", - Threads: 4, - LogLevel: mediaconvert.LogLevelDebug, - ScalingLevels: []string{"720p", "480p"}, - }) + var profiles = []profile.VideoProfile{ + profile.Profile720p, + profile.Profile480p, + } - const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -vf scale=-2:720 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -map 0:v:0 -map 0:a? -vf scale=-2:480 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` - - require.Contains(t, expected, strings.Join(scaleCommand, " ")) -} - -func Test_BuildVideoCommand_Scaling_NoRaw(t *testing.T) { - var scaleCommand = mediaconvert.BuildScalingVideoCommand(&mediaconvert.Options{ - OutputDir: "test", - Input: "pipe:0", - UploadID: "1", - Threads: 4, - LogLevel: mediaconvert.LogLevelDebug, - Level: "720p", - ScalingLevels: []string{"720p", "480p"}, - }) - - const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -vf scale=-2:480 -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` - - require.Contains(t, expected, strings.Join(scaleCommand, " ")) -} - -func Test_BuildVideoCommand_Raw_NoTranscode(t *testing.T) { - var rawCommand = mediaconvert.BuildRawVideoCommand(&mediaconvert.Options{ + var scaleCommand = mediaconvert.BuildVideoCommand(&mediaconvert.Options{ OutputDir: "test", Input: "pipe:0", UploadID: "1", Threads: 4, LogLevel: mediaconvert.LogLevelDebug, - Level: resconv.Level("651:490"), - Transcode: false, + Profiles: profiles, }) - const expected = `"-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -c:a copy -c:v copy -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -c:a aac -c:v libx264 -preset veryfast -crf 25 -g 60 -vf scale=-2:720 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -map 0:v:0 -map 0:a? -c:a aac -c:v libx264 -preset veryfast -crf 27 -g 60 -vf scale=-2:480 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` - require.Contains(t, expected, strings.Join(rawCommand, " ")) + require.Contains(t, expected, strings.Join(scaleCommand, " ")) } -func Test_BuildVideoCommand_Raw_Transcode(t *testing.T) { - var rawCommand = mediaconvert.BuildRawVideoCommand(&mediaconvert.Options{ +func Test_BuildVideoCommand_Original(t *testing.T) { + var profiles = []profile.VideoProfile{ + profile.MakeProfileOriginal(640, 480), + } + + var rawCommand = mediaconvert.BuildVideoCommand(&mediaconvert.Options{ OutputDir: "test", Input: "pipe:0", UploadID: "1", Threads: 4, LogLevel: mediaconvert.LogLevelDebug, - Level: resconv.Level("651:490"), - Transcode: true, + Profiles: profiles, }) - const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -c:a copy -c:v copy -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_orig.ts test/1/1_orig_master.m3u8` require.Contains(t, expected, strings.Join(rawCommand, " ")) } -func Test_BuildVideoCommand_Scaling_Small(t *testing.T) { - var scaleCommand = mediaconvert.BuildScalingVideoCommand(&mediaconvert.Options{ - OutputDir: "test", - Input: "pipe:0", - UploadID: "1", - Threads: 4, - LogLevel: mediaconvert.LogLevelDebug, - Level: "360p", - ScalingLevels: []string{"360p"}, +func Test_BuildVideoCommand_OriginalT(t *testing.T) { + var profiles = []profile.VideoProfile{ + profile.MakeProfileOriginalT(640, 480), + } + + var rawCommand = mediaconvert.BuildVideoCommand(&mediaconvert.Options{ + OutputDir: "test", + Input: "pipe:0", + UploadID: "1", + Threads: 4, + LogLevel: mediaconvert.LogLevelDebug, + Profiles: profiles, }) - require.Empty(t, scaleCommand) + const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_orig.ts test/1/1_orig_master.m3u8` + + require.Contains(t, expected, strings.Join(rawCommand, " ")) } diff --git a/internal/pkg/mediaconvert/coordinator.go b/internal/pkg/mediaconvert/coordinator.go index d2721134ee..d20520a203 100644 --- a/internal/pkg/mediaconvert/coordinator.go +++ b/internal/pkg/mediaconvert/coordinator.go @@ -17,7 +17,11 @@ package mediaconvert import ( "context" + "fmt" "path/filepath" + "regexp" + "strconv" + "strings" "sync" "sync/atomic" "time" @@ -27,7 +31,6 @@ import ( "github.com/google/uuid" "github.com/hcengineering/stream/internal/pkg/config" "github.com/hcengineering/stream/internal/pkg/log" - "github.com/hcengineering/stream/internal/pkg/resconv" "github.com/hcengineering/stream/internal/pkg/sharedpipe" "github.com/hcengineering/stream/internal/pkg/storage" "github.com/hcengineering/stream/internal/pkg/uploader" @@ -40,7 +43,7 @@ type StreamCoordinator struct { conf *config.Config uploadOptions uploader.Options - activeScalling int32 + activeTranscoding int32 mainContext context.Context logger *zap.Logger @@ -49,6 +52,11 @@ type StreamCoordinator struct { cancels sync.Map } +var _ handler.DataStore = (*StreamCoordinator)(nil) +var _ handler.ConcaterDataStore = (*StreamCoordinator)(nil) +var _ handler.TerminaterDataStore = (*StreamCoordinator)(nil) +var _ handler.LengthDeferrerDataStore = (*StreamCoordinator)(nil) + // NewStreamCoordinator creates a new scheduler for transcode operations. func NewStreamCoordinator(ctx context.Context, c *config.Config) *StreamCoordinator { return &StreamCoordinator{ @@ -81,42 +89,64 @@ func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo done: make(chan struct{}), } - var scaling = resconv.SubLevels(info.MetaData["resolution"]) - var level = resconv.Level(info.MetaData["resolution"]) - var cost int64 - - for _, scale := range scaling { - cost += int64(resconv.Pixels(resconv.Resolution(scale))) - } - - if atomic.AddInt32(&s.activeScalling, 1) > int32(s.conf.MaxParallelScalingCount) { - atomic.AddInt32(&s.activeScalling, -1) + if atomic.AddInt32(&s.activeTranscoding, 1) > int32(s.conf.MaxParallelTranscodingCount) { + atomic.AddInt32(&s.activeTranscoding, -1) s.logger.Debug("run out of resources for scaling") - scaling = nil + // TODO do not transcode } + width, err := strconv.Atoi(info.MetaData["width"]) + if err != nil { + return nil, errors.Wrapf(err, "can not parse video width: %v", info.MetaData["width"]) + } + + height, err := strconv.Atoi(info.MetaData["height"]) + if err != nil { + return nil, errors.Wrapf(err, "can not parse video height: %v", info.MetaData["height"]) + } + + meta := VideoMeta{ + Width: width, + Height: height, + Codec: extractCodec(info.MetaData["contentType"]), + ContentType: extractContentType(info.MetaData["contentType"]), + } + profiles := FastTranscodingProfiles(meta) + var commandOptions = Options{ - Input: "pipe:0", - OutputDir: s.conf.OutputDir, - Threads: s.conf.MaxThreadCount, - UploadID: info.ID, - Transcode: true, - Level: level, - ScalingLevels: scaling, + Input: "pipe:0", + OutputDir: s.conf.OutputDir, + Threads: s.conf.MaxThreadCount, + UploadID: info.ID, + Profiles: profiles, } if s.conf.EndpointURL != nil { s.logger.Sugar().Debugf("initializing uploader for %v", info) + + // setup content uploader for transcoded outputs var opts = s.uploadOptions opts.Dir = filepath.Join(opts.Dir, info.ID) - var storage, err = storage.NewStorageByURL(s.mainContext, s.conf.Endpoint(), s.conf.EndpointURL.Scheme, info.MetaData["token"], info.MetaData["workspace"]) + // create storage backend + var stg, err = storage.NewStorageByURL(s.mainContext, s.conf.Endpoint(), s.conf.EndpointURL.Scheme, info.MetaData["token"], info.MetaData["workspace"]) if err != nil { - s.logger.Error("can not create storage by url") - return nil, err + s.logger.Error("can not create storage by url", zap.Error(err)) + return nil, errors.Wrapf(err, "can not create storage") } - var contentUploader = uploader.New(s.mainContext, storage, opts) + stream.storage = stg + // if storage supports multipart, initialize raw upload + if ms, ok := stg.(storage.MultipartStorage); ok { + multipart, err := NewMultipartUpload(s.mainContext, ms, info, meta.ContentType) + if err != nil { + s.logger.Error("multipart upload failed", zap.Error(err)) + return nil, errors.Wrapf(err, "multipart upload failed") + } + stream.multipart = multipart + } + // uploader for processed outputs + var contentUploader = uploader.New(s.mainContext, stg, opts) stream.contentUploader = contentUploader } @@ -127,10 +157,7 @@ func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo go func() { stream.commandGroup.Wait() - if scaling != nil { - atomic.AddInt32(&s.activeScalling, -1) - } - s.logger.Debug("returned capacity", zap.Int64("capacity", cost)) + atomic.AddInt32(&s.activeTranscoding, -1) close(stream.done) }() @@ -142,26 +169,26 @@ func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo // GetUpload returns current a worker based on upload id func (s *StreamCoordinator) GetUpload(ctx context.Context, id string) (upload handler.Upload, err error) { + logger := s.logger.With(zap.String("func", "GetUpload")).With(zap.String("id", id)) + if v, ok := s.streams.Load(id); ok { - s.logger.Debug("GetUpload: found stream by id", zap.String("id", id)) + logger.Debug("found stream") var w = v.(*Stream) s.manageTimeout(w) return w, nil } - s.logger.Debug("GetUpload: stream not found", zap.String("id", id)) - return nil, errors.New("bad id") + + logger.Warn("stream not found") + return nil, fmt.Errorf("stream not found: %v", id) } // AsTerminatableUpload returns tusd handler.TerminatableUpload func (s *StreamCoordinator) AsTerminatableUpload(upload handler.Upload) handler.TerminatableUpload { - var worker = upload.(*Stream) - s.logger.Debug("AsTerminatableUpload") - return worker + return upload.(*Stream) } // AsLengthDeclarableUpload returns tusd handler.LengthDeclarableUpload func (s *StreamCoordinator) AsLengthDeclarableUpload(upload handler.Upload) handler.LengthDeclarableUpload { - s.logger.Debug("AsLengthDeclarableUpload") return upload.(*Stream) } @@ -191,3 +218,23 @@ func (s *StreamCoordinator) manageTimeout(w *Stream) { } }() } + +func extractCodec(mimeType string) string { + codecRegex := regexp.MustCompile(`codecs["\s=]+([^",\s]+)`) + matches := codecRegex.FindStringSubmatch(mimeType) + codec := "unknown" + if len(matches) > 1 { + codec = matches[1] + } + + return codec +} + +func extractContentType(mimeType string) string { + contentType := "video/mp4" + parts := strings.Split(mimeType, ";") + if parts[0] != "" { + contentType = strings.TrimSpace(parts[0]) + } + return contentType +} diff --git a/internal/pkg/mediaconvert/multipart.go b/internal/pkg/mediaconvert/multipart.go new file mode 100644 index 0000000000..f29e35ef7f --- /dev/null +++ b/internal/pkg/mediaconvert/multipart.go @@ -0,0 +1,188 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package mediaconvert provides types and functions for video transcoding +package mediaconvert + +import ( + "bytes" + "context" + "time" + + "github.com/pkg/errors" + + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/storage" + "github.com/tus/tusd/v2/pkg/handler" + "go.uber.org/zap" +) + +// minimum size for multipart parts (backend requires >=5MiB for all but last part) +const minPartSize = 5 * 1024 * 1024 + +type MultipartUpload struct { + logger *zap.Logger + buffer *bytes.Buffer + info handler.FileInfo + + storage storage.MultipartStorage + objectName string + uploadID string + parts []storage.MultipartPart + nextPartNum int + + terminated bool + completed bool + + bytesWritten int64 + bytesUploaded int64 +} + +func NewMultipartUpload( + ctx context.Context, + multipartStorage storage.MultipartStorage, + info handler.FileInfo, + contentType string, +) (*MultipartUpload, error) { + objectName := info.ID + uploadID, err := multipartStorage.MultipartUploadStart(ctx, objectName, contentType) + if err != nil { + return nil, errors.Wrap(err, "failed to initialize multipart upload") + } + + logger := log.FromContext(ctx).With(zap.String("multipart", "upload"), zap.String("uploadID", uploadID)) + + return &MultipartUpload{ + logger: logger, + buffer: bytes.NewBuffer(nil), + info: info, + storage: multipartStorage, + objectName: objectName, + uploadID: uploadID, + parts: make([]storage.MultipartPart, 0), + nextPartNum: 1, + }, nil +} + +// Write writes chunk of data to the storage +func (w *MultipartUpload) Write(ctx context.Context, data []byte) error { + if w.terminated || w.completed { + return errors.New("upload already terminated or completed") + } + + if err := ctx.Err(); err != nil { + return err + } + + _, err := w.buffer.Write(data) + if err != nil { + return errors.Wrap(err, "failed to write to buffer") + } + w.bytesWritten += int64(len(data)) + + // flush parts of at least minPartSize + for w.buffer.Len() >= minPartSize { + partNum := w.nextPartNum + partData := w.buffer.Next(minPartSize) + + if err := ctx.Err(); err != nil { + return err + } + + part, err := w.storage.MultipartUploadPart(ctx, w.objectName, w.uploadID, partNum, partData) + if err != nil { + w.logger.Error("multipart upload part failed", zap.Error(err), zap.Int("partNumber", partNum)) + return errors.Wrap(err, "failed to upload part") + } + + w.bytesUploaded += int64(len(partData)) + w.parts = append(w.parts, *part) + w.nextPartNum++ + } + + return nil +} + +// Terminate cancels the upload +func (w *MultipartUpload) Terminate(ctx context.Context) error { + if w.terminated || w.completed { + return nil + } + w.terminated = true + + w.logger.Debug("terminating multipart upload", zap.Int("parts", len(w.parts))) + + // create new context in case the main context is cancelled + cancelCtx := ctx + if ctx.Err() != nil { + var cancel context.CancelFunc + cancelCtx, cancel = context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + } + + if err := w.storage.MultipartUploadCancel(cancelCtx, w.objectName, w.uploadID); err != nil { + w.logger.Error("multipart upload cancel failed", zap.Error(err)) + return errors.Wrap(err, "failed to cancel multipart upload") + } + + w.logger.Debug("multipart upload terminated", zap.Int("parts", len(w.parts))) + + return nil +} + +// Complete uploads last bytes and completes the upload +func (w *MultipartUpload) Complete(ctx context.Context) error { + if w.completed { + return nil + } + + if w.terminated { + return errors.New("cannot complete terminated upload") + } + + w.logger.Debug("finishing multipart upload", zap.Int("parts", len(w.parts))) + + // flush any remaining data as last part + if w.buffer.Len() > 0 { + partNum := w.nextPartNum + lastData := w.buffer.Bytes() + + part, err := w.storage.MultipartUploadPart(ctx, w.objectName, w.uploadID, partNum, lastData) + if err != nil { + w.logger.Error("multipart upload last part failed", zap.Error(err), zap.Int("partNumber", partNum)) + return errors.Wrap(err, "failed to upload last part") + } + + w.bytesUploaded += int64(len(lastData)) + w.parts = append(w.parts, *part) + } + + if len(w.parts) == 0 { + w.logger.Warn("cannot complete upload with no parts") + return errors.New("cannot complete upload with no parts") + } + + if err := w.storage.MultipartUploadComplete(ctx, w.objectName, w.uploadID, w.parts); err != nil { + w.logger.Error("multipart upload complete failed", zap.Error(err)) + return errors.Wrap(err, "failed to complete multipart upload") + } + + w.completed = true + w.logger.Info( + "multipart upload completed", + zap.Int64("bytesUploaded", w.bytesUploaded), + zap.Int64("bytesWritten", w.bytesWritten), + ) + + return nil +} diff --git a/internal/pkg/mediaconvert/scheduler.go b/internal/pkg/mediaconvert/scheduler.go index da3190276b..2f3e2f6f55 100644 --- a/internal/pkg/mediaconvert/scheduler.go +++ b/internal/pkg/mediaconvert/scheduler.go @@ -21,13 +21,13 @@ import ( "os" "os/exec" "path/filepath" + "strings" "time" "github.com/google/uuid" "github.com/hcengineering/stream/internal/pkg/config" "github.com/hcengineering/stream/internal/pkg/log" "github.com/hcengineering/stream/internal/pkg/manifest" - "github.com/hcengineering/stream/internal/pkg/resconv" "github.com/hcengineering/stream/internal/pkg/storage" "github.com/hcengineering/stream/internal/pkg/token" "github.com/hcengineering/stream/internal/pkg/uploader" @@ -94,7 +94,7 @@ func (p *Scheduler) start() { close(p.taskCh) }() - for range p.cfg.MaxParallelScalingCount { + for range p.cfg.MaxParallelTranscodingCount { go func() { for task := range p.taskCh { p.processTask(p.ctx, task) @@ -185,17 +185,24 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { logger.Debug("video stream found", zap.String("codec", videoStream.CodecName), zap.Int("width", videoStream.Width), zap.Int("height", videoStream.Height)) - var res = fmt.Sprintf("%v:%v", videoStream.Width, videoStream.Height) - var codec = videoStream.CodecName - var level = resconv.Level(res) + meta := VideoMeta{ + Width: videoStream.Width, + Height: videoStream.Height, + Codec: videoStream.CodecName, + ContentType: stat.Type, + } + + var profiles = DefaultTranscodingProfiles(meta) + var opts = Options{ - Input: sourceFilePath, - OutputDir: p.cfg.OutputDir, - Level: level, - Transcode: !IsHLSSupportedVideoCodec(codec), - ScalingLevels: append(resconv.SubLevels(res), level), - UploadID: task.ID, - Threads: p.cfg.MaxThreadCount, + Input: sourceFilePath, + OutputDir: p.cfg.OutputDir, + UploadID: task.ID, + Threads: p.cfg.MaxThreadCount, + Profiles: profiles, + // Level: level, + // Transcode: !IsHLSSupportedVideoCodec(codec), + // ScalingLevels: append(resconv.SubLevels(res), level), } logger.Debug("phase 5: start async upload process") @@ -210,7 +217,7 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { SourceFile: sourceFilePath, }) - err = manifest.GenerateHLSPlaylist(opts.ScalingLevels, p.cfg.OutputDir, opts.UploadID) + err = manifest.GenerateHLSPlaylist(profiles, p.cfg.OutputDir, opts.UploadID) if err != nil { logger.Error("can not generate hls playlist", zap.String("out", p.cfg.OutputDir), zap.String("uploadID", opts.UploadID)) _ = os.RemoveAll(destinationFolder) @@ -223,8 +230,7 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { var argsSlice = [][]string{ BuildThumbnailCommand(&opts), - BuildRawVideoCommand(&opts), - BuildScalingVideoCommand(&opts), + BuildVideoCommand(&opts), } var cmds []*exec.Cmd @@ -295,6 +301,11 @@ func IsHLSSupportedVideoCodec(codec string) bool { case "h264", "h265": return true default: + if strings.HasPrefix(codec, "avc1") { + return true + } else if strings.HasPrefix(codec, "av1") { + return true + } return false } } diff --git a/internal/pkg/mediaconvert/strategy.go b/internal/pkg/mediaconvert/strategy.go new file mode 100644 index 0000000000..387db9de3f --- /dev/null +++ b/internal/pkg/mediaconvert/strategy.go @@ -0,0 +1,67 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package mediaconvert + +import ( + "fmt" + + "github.com/hcengineering/stream/internal/pkg/profile" + "github.com/hcengineering/stream/internal/pkg/resconv" +) + +type VideoMeta struct { + Width int + Height int + Codec string + ContentType string +} + +// DefaultTranscodingProfiles uses original resolution and two more resolutions +func DefaultTranscodingProfiles(meta VideoMeta) []profile.VideoProfile { + var profiles = make([]profile.VideoProfile, 0) + + var res = fmt.Sprintf("%v:%v", meta.Width, meta.Height) + var sublevels = resconv.SubLevels(res) + + if IsHLSSupportedVideoCodec(meta.Codec) { + profile := profile.MakeProfileOriginal(meta.Width, meta.Height) + profiles = append(profiles, profile) + } else { + profile := profile.MakeProfileOriginalT(meta.Width, meta.Height) + profiles = append(profiles, profile) + } + + for _, level := range sublevels { + if profile, ok := profile.GetProfileByName(level); ok { + profiles = append(profiles, profile) + } + } + + return profiles +} + +// FastTranscodingProfiles uses fastest possible video profile +func FastTranscodingProfiles(meta VideoMeta) []profile.VideoProfile { + if IsHLSSupportedVideoCodec(meta.Codec) { + return []profile.VideoProfile{ + profile.MakeProfileOriginal(meta.Width, meta.Height), + } + } + + return []profile.VideoProfile{ + profile.Profile360p, + } +} diff --git a/internal/pkg/mediaconvert/stream.go b/internal/pkg/mediaconvert/stream.go index e41042d2e0..135ce98997 100644 --- a/internal/pkg/mediaconvert/stream.go +++ b/internal/pkg/mediaconvert/stream.go @@ -11,44 +11,69 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package mediaconvert provides types and functions for video trnascoding +// Package mediaconvert provides types and functions for video transcoding package mediaconvert import ( "context" "io" + "os/exec" "sync" "github.com/pkg/errors" "github.com/hcengineering/stream/internal/pkg/manifest" "github.com/hcengineering/stream/internal/pkg/sharedpipe" + "github.com/hcengineering/stream/internal/pkg/storage" "github.com/hcengineering/stream/internal/pkg/uploader" "github.com/tus/tusd/v2/pkg/handler" "go.uber.org/zap" ) -// Stream manages client's input and transcodes it based on the passsed configuration +// Stream manages client's input and transcodes it based on the passed configuration type Stream struct { contentUploader uploader.Uploader logger *zap.Logger info handler.FileInfo writer *sharedpipe.Writer reader *sharedpipe.Reader + storage storage.Storage + multipart *MultipartUpload commandGroup sync.WaitGroup done chan struct{} } -// WriteChunk calls when client sends a chunk of raw data +var _ handler.Upload = (*Stream)(nil) +var _ handler.ConcatableUpload = (*Stream)(nil) +var _ handler.TerminatableUpload = (*Stream)(nil) +var _ handler.LengthDeclarableUpload = (*Stream)(nil) + +// WriteChunk is called when client sends a chunk of raw data func (w *Stream) WriteChunk(ctx context.Context, _ int64, src io.Reader) (int64, error) { w.logger.Debug("Write Chunk start", zap.Int64("offset", w.info.Offset)) - var bytes, err = io.ReadAll(src) - _, _ = w.writer.Write(bytes) - var n = int64(len(bytes)) + data, err := io.ReadAll(src) + if err != nil { + return 0, err + } + // write into pipeline for transcoding + written, err := w.writer.Write(data) + if err != nil { + return int64(written), err + } + + n := int64(len(data)) w.info.Offset += n - w.logger.Debug("Write Chunk end", zap.Int64("offset", w.info.Offset), zap.Error(err)) - return n, err + + if w.multipart != nil { + if writeErr := w.multipart.Write(ctx, data); writeErr != nil { + w.logger.Error("multipart upload part failed", zap.Error(writeErr)) + return n, writeErr + } + } + + w.logger.Debug("write chunk end", zap.Int64("offset", w.info.Offset), zap.Error(err)) + return n, nil } // DeclareLength sets length of the video input @@ -73,14 +98,40 @@ func (w *Stream) GetReader(ctx context.Context) (io.ReadCloser, error) { // Terminate calls when upload has failed func (w *Stream) Terminate(ctx context.Context) error { - w.logger.Debug("Terminating...") + w.logger.Debug("terminate upload") + + // Close the writer first to signal EOF to all readers + if err := w.writer.Close(); err != nil { + w.logger.Error("failed to close writer", zap.Error(err)) + return err + } + + var wg sync.WaitGroup + + // cancel upload if in progress if w.contentUploader != nil { + wg.Add(1) go func() { + defer wg.Done() w.commandGroup.Wait() w.contentUploader.Cancel() }() } - return w.writer.Close() + + // cancel multipart upload if in progress + if w.multipart != nil { + wg.Add(1) + go func() { + defer wg.Done() + if err := w.multipart.Terminate(ctx); err != nil { + w.logger.Error("multipart upload cancel failed", zap.Error(err)) + } + }() + } + + wg.Wait() + + return nil } // ConcatUploads calls when upload resumed after fail @@ -94,16 +145,56 @@ func (w *Stream) ConcatUploads(ctx context.Context, partialUploads []handler.Upl // FinishUpload calls when upload finished without errors on the client side func (w *Stream) FinishUpload(ctx context.Context) error { - w.logger.Debug("finishing upload...") + w.logger.Debug("finish upload") + + // Close the writer first to signal EOF to all readers + if err := w.writer.Close(); err != nil { + w.logger.Error("failed to close writer", zap.Error(err)) + return err + } + + var wg sync.WaitGroup if w.contentUploader != nil { + wg.Add(1) go func() { + defer wg.Done() w.commandGroup.Wait() w.contentUploader.Stop() }() } - return w.writer.Close() + // finalize raw multipart stream if supported + if w.multipart != nil { + wg.Add(1) + go func() { + defer wg.Done() + if err := w.multipart.Complete(ctx); err != nil { + w.logger.Error("multipart upload complete failed", zap.Error(err)) + return + } + + if metaProvider, ok := w.storage.(storage.MetaProvider); ok { + metaErr := metaProvider.PatchMeta( + ctx, + w.info.ID, + &storage.Metadata{ + "hls": map[string]any{ + "source": w.info.ID + "_master.m3u8", + "thumbnail": w.info.ID + ".jpg", + }, + }, + ) + if metaErr != nil { + w.logger.Error("can not patch the source file", zap.Error(metaErr)) + } + } + }() + } + + wg.Wait() + + return nil } // AsConcatableUpload returns tusd handler.ConcatableUpload @@ -115,48 +206,39 @@ func (s *StreamCoordinator) AsConcatableUpload(upload handler.Upload) handler.Co func (w *Stream) start(ctx context.Context, options *Options) error { defer w.logger.Debug("start done") w.reader = w.writer.Transpile() - if err := manifest.GenerateHLSPlaylist(append(options.ScalingLevels, options.Level), options.OutputDir, options.UploadID); err != nil { + if err := manifest.GenerateHLSPlaylist(options.Profiles, options.OutputDir, options.UploadID); err != nil { return err } + var argsSlice = [][]string{ + BuildThumbnailCommand(options), + BuildVideoCommand(options), + } + + var cmds []*exec.Cmd + for idx, args := range argsSlice { + reader := w.reader + if idx > 0 { + reader = w.writer.Transpile() + } + + cmd, cmdErr := newFfmpegCommand(ctx, reader, args) + if cmdErr != nil { + w.logger.Error("can not create a new command", zap.Error(cmdErr), zap.Strings("args", args)) + return errors.Wrapf(cmdErr, "can not create a new command") + } + cmds = append(cmds, cmd) + } + w.commandGroup.Add(1) go func() { defer w.commandGroup.Done() - var logger = w.logger.With(zap.String("command", "raw")) - defer logger.Debug("done") - - var args = BuildRawVideoCommand(options) - var convertSourceCommand, err = newFfmpegCommand(ctx, w.reader, args) - if err != nil { - logger.Debug("can not start", zap.Error(err)) - } - err = convertSourceCommand.Run() - if err != nil { - logger.Debug("finished with error", zap.Error(err)) + executor := NewCommandExecutor(ctx) + if execErr := executor.Execute(cmds); execErr != nil { + w.logger.Error("can not execute command", zap.Error(execErr)) } }() - if len(options.ScalingLevels) > 0 { - w.commandGroup.Add(1) - var scalingCommandReader = w.writer.Transpile() - - go func() { - defer w.commandGroup.Done() - var logger = w.logger.With(zap.String("command", "scaling")) - defer logger.Debug("done") - - var args = BuildScalingVideoCommand(options) - var convertSourceCommand, err = newFfmpegCommand(ctx, scalingCommandReader, args) - if err != nil { - logger.Debug("can not start", zap.Error(err)) - } - err = convertSourceCommand.Run() - if err != nil { - logger.Debug("finished with error", zap.Error(err)) - } - }() - } - go w.contentUploader.Start() return nil diff --git a/internal/pkg/mediaconvert/transcoder.go b/internal/pkg/mediaconvert/transcoder.go index f85d034532..1972f171dd 100644 --- a/internal/pkg/mediaconvert/transcoder.go +++ b/internal/pkg/mediaconvert/transcoder.go @@ -26,7 +26,6 @@ import ( "github.com/hcengineering/stream/internal/pkg/config" "github.com/hcengineering/stream/internal/pkg/log" "github.com/hcengineering/stream/internal/pkg/manifest" - "github.com/hcengineering/stream/internal/pkg/resconv" "github.com/hcengineering/stream/internal/pkg/storage" "github.com/hcengineering/stream/internal/pkg/token" "github.com/hcengineering/stream/internal/pkg/uploader" @@ -119,7 +118,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er videoStream := probe.FirstVideoStream() if videoStream == nil { logger.Error("no video stream found", zap.String("filepath", sourceFilePath)) - return nil, errors.Wrapf(err, "no video stream found") + return nil, fmt.Errorf("no video stream found") } logger.Debug("video stream found", zap.String("codec", videoStream.CodecName), zap.Int("width", videoStream.Width), zap.Int("height", videoStream.Height)) @@ -129,19 +128,22 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er logger.Info("no audio stream found", zap.String("filepath", sourceFilePath)) } - var res = fmt.Sprintf("%v:%v", videoStream.Width, videoStream.Height) - var codec = videoStream.CodecName - var level = resconv.Level(res) - var sublevels = resconv.SubLevels(res) + meta := VideoMeta{ + Width: videoStream.Width, + Height: videoStream.Height, + Codec: videoStream.CodecName, + ContentType: stat.Type, + } + + var profiles = DefaultTranscodingProfiles(meta) + var opts = Options{ - Input: sourceFilePath, - OutputDir: p.cfg.OutputDir, - Level: level, - LogLevel: LogLevel(p.cfg.LogLevel), - Transcode: !IsHLSSupportedVideoCodec(codec), - ScalingLevels: append(sublevels, level), - UploadID: task.ID, - Threads: p.cfg.MaxThreadCount, + Input: sourceFilePath, + OutputDir: p.cfg.OutputDir, + LogLevel: LogLevel(p.cfg.LogLevel), + Profiles: profiles, + UploadID: task.ID, + Threads: p.cfg.MaxThreadCount, } logger.Debug("phase 5: start async upload process") @@ -156,7 +158,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er SourceFile: sourceFilePath, }) - err = manifest.GenerateHLSPlaylist(opts.ScalingLevels, p.cfg.OutputDir, opts.UploadID) + err = manifest.GenerateHLSPlaylist(profiles, p.cfg.OutputDir, opts.UploadID) if err != nil { logger.Error("can not generate hls playlist", zap.String("out", p.cfg.OutputDir), zap.String("uploadID", opts.UploadID)) return nil, errors.Wrapf(err, "can not generate hls playlist") @@ -168,8 +170,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er var argsSlice = [][]string{ BuildThumbnailCommand(&opts), - BuildRawVideoCommand(&opts), - BuildScalingVideoCommand(&opts), + BuildVideoCommand(&opts), } var cmds []*exec.Cmd diff --git a/internal/pkg/profile/profile.go b/internal/pkg/profile/profile.go new file mode 100644 index 0000000000..7716ef6914 --- /dev/null +++ b/internal/pkg/profile/profile.go @@ -0,0 +1,185 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package profile provides video profiles +package profile + +import ( + "fmt" + + "github.com/hcengineering/stream/internal/pkg/resconv" +) + +// VideoProfile represents a video profile +type VideoProfile struct { + Name string + Width int // 0 means keep original + Height int // 0 means keep original + Bandwidth int + Scale bool + + // Codec settings + VideoCodec string + AudioCodec string + + // Advanced encoding settings + CRF int // Constant Rate Factor (0-51 for x264) +} + +// profileOriginal is a profile for original video without transcoding +var profileOriginal = VideoProfile{ + Name: "orig", + Scale: false, + VideoCodec: "copy", + AudioCodec: "copy", + CRF: 23, +} + +// profileOriginalT is a profile for transcoding video in the original resolution +var profileOriginalT = VideoProfile{ + Name: "orig", + Scale: false, + VideoCodec: "libx264", + AudioCodec: "aac", + CRF: 23, +} + +// Profile360p is a profile for transcoding video in 360p +var Profile360p = VideoProfile{ + Name: "360p", + Scale: true, + Width: 640, + Height: 360, + Bandwidth: 500000, + VideoCodec: "libx264", + AudioCodec: "aac", + CRF: 28, +} + +// Profile480p is a profile for transcoding video in 480p +var Profile480p = VideoProfile{ + Name: "480p", + Scale: true, + Width: 854, + Height: 480, + Bandwidth: 2000000, + VideoCodec: "libx264", + AudioCodec: "aac", + CRF: 27, +} + +// Profile720p is a profile for transcoding video in 720p +var Profile720p = VideoProfile{ + Name: "720p", + Scale: true, + Width: 1280, + Height: 720, + Bandwidth: 5000000, + VideoCodec: "libx264", + AudioCodec: "aac", + CRF: 25, +} + +// Profile1080p is a profile for transcoding video in 1080p +var Profile1080p = VideoProfile{ + Name: "1080p", + Scale: true, + Width: 1920, + Height: 1080, + Bandwidth: 8000000, + VideoCodec: "libx264", + AudioCodec: "aac", + CRF: 23, +} + +// Profile1440p is a profile for transcoding video in 1440p +var Profile1440p = VideoProfile{ + Name: "1440p", + Scale: true, + Width: 2560, + Height: 1440, + Bandwidth: 12000000, + VideoCodec: "libx264", + AudioCodec: "aac", + CRF: 23, +} + +// Profile2160p is a profile for transcoding video in 2160p +var Profile2160p = VideoProfile{ + Name: "2160p", + Scale: true, + Width: 3840, + Height: 2160, + Bandwidth: 25000000, + VideoCodec: "libx264", // Consider libx265 + AudioCodec: "aac", + CRF: 22, +} + +// Profile4320p is a profile for transcoding video in 360p +var Profile4320p = VideoProfile{ + Name: "4320p", + Scale: true, + Width: 7680, + Height: 4320, + Bandwidth: 50000000, + VideoCodec: "libx264", // Consider libx265 + AudioCodec: "aac", + CRF: 22, +} + +var Profiles = map[string]VideoProfile{ + "360p": Profile360p, + "480p": Profile480p, + "720p": Profile720p, + "1080p": Profile1080p, + "1440p": Profile1440p, + "2160p": Profile2160p, + "4320p": Profile4320p, +} + +func MakeProfileOriginal(width, height int) VideoProfile { + resolution := fmt.Sprintf("%v:%v", width, height) + level := resconv.Level(resolution) + bandwidth := resconv.Bandwidth(level) + + profile := profileOriginal + //profile.Name = level + profile.Width = width + profile.Height = height + profile.Bandwidth = bandwidth + + return profile +} + +func MakeProfileOriginalT(width, height int) VideoProfile { + resolution := fmt.Sprintf("%v:%v", width, height) + level := resconv.Level(resolution) + bandwidth := resconv.Bandwidth(level) + + profile := profileOriginalT + //profile.Name = level + profile.Width = width + profile.Height = height + profile.Bandwidth = bandwidth + + return profile +} + +// GetProfileByName returns a VideoProfile by name +func GetProfileByName(name string) (VideoProfile, bool) { + profile, ok := Profiles[name] + return profile, ok +} diff --git a/internal/pkg/profile/profile_test.go b/internal/pkg/profile/profile_test.go new file mode 100644 index 0000000000..a4ea1448fd --- /dev/null +++ b/internal/pkg/profile/profile_test.go @@ -0,0 +1,173 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package profile_test + +import ( + "testing" + + "github.com/hcengineering/stream/internal/pkg/profile" + "github.com/stretchr/testify/assert" +) + +func TestGetProfileByName(t *testing.T) { + tests := []struct { + name string + expected profile.VideoProfile + }{ + { + name: "360p", + expected: profile.Profile360p, + }, + { + name: "480p", + expected: profile.Profile480p, + }, + { + name: "720p", + expected: profile.Profile720p, + }, + { + name: "1080p", + expected: profile.Profile1080p, + }, + { + name: "1440p", + expected: profile.Profile1440p, + }, + { + name: "2160p", + expected: profile.Profile2160p, + }, + { + name: "4320p", + expected: profile.Profile4320p, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + profile, ok := profile.GetProfileByName(tt.name) + assert.True(t, ok) + assert.Equal(t, tt.expected, profile) + }) + } +} + +func TestGetProfileByName_Failure(t *testing.T) { + tests := []struct { + name string + }{ + { + name: "foo", + }, + { + name: "original", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, ok := profile.GetProfileByName(tt.name) + assert.False(t, ok) + }) + } +} + +func TestMakeProfileOriginal(t *testing.T) { + tests := []struct { + name string + width int + height int + expected profile.VideoProfile + }{ + { + name: "480p", + width: 640, + height: 480, + expected: profile.VideoProfile{ + Name: "orig", + VideoCodec: "copy", + AudioCodec: "copy", + Width: 640, + Height: 480, + Bandwidth: 2000000, + }, + }, + { + name: "1440p", + width: 2000, + height: 1200, + expected: profile.VideoProfile{ + Name: "orig", + VideoCodec: "copy", + AudioCodec: "copy", + Width: 2000, + Height: 1200, + Bandwidth: 8000000, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + profile := profile.MakeProfileOriginal(tt.width, tt.height) + assert.Equal(t, tt.expected, profile) + }) + } +} + +func TestMakeProfileOriginalT(t *testing.T) { + tests := []struct { + name string + width int + height int + expected profile.VideoProfile + }{ + { + name: "720p", + width: 1280, + height: 720, + expected: profile.VideoProfile{ + Name: "orig", + VideoCodec: "libx264", + AudioCodec: "aac", + Width: 1280, + Height: 720, + Bandwidth: 5000000, + }, + }, + { + name: "2160p", + width: 3840, + height: 2160, + expected: profile.VideoProfile{ + Name: "orig", + VideoCodec: "libx264", + AudioCodec: "aac", + Width: 3840, + Height: 2160, + Bandwidth: 25000000, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + profile := profile.MakeProfileOriginalT(tt.width, tt.height) + assert.Equal(t, tt.expected, profile) + }) + } +} diff --git a/internal/pkg/queue/queue.go b/internal/pkg/queue/queue.go index cb5c674519..d9f6a54208 100644 --- a/internal/pkg/queue/queue.go +++ b/internal/pkg/queue/queue.go @@ -17,6 +17,7 @@ package queue import ( "context" "encoding/json" + "fmt" "time" "github.com/hcengineering/stream/internal/pkg/log" @@ -168,7 +169,12 @@ func NewProducer(ctx context.Context, options ProducerOptions) Producer { // Send sends a message to the queue topic func (p *TProducer) Send(ctx context.Context, workspaceID string, data any) error { + if data == nil { + return fmt.Errorf("event data is empty") + } + value, err := json.Marshal(data) + if err != nil { return err } diff --git a/internal/pkg/queue/worker.go b/internal/pkg/queue/worker.go index 7c9dae02b4..60def900d5 100644 --- a/internal/pkg/queue/worker.go +++ b/internal/pkg/queue/worker.go @@ -111,7 +111,7 @@ func (w *Worker) processMessage(ctx context.Context, msg kafka.Message, logger * transcoder := mediaconvert.NewTranscoder(ctx, w.cfg) res, err := transcoder.Transcode(ctx, &task) - if err == nil { + if res != nil { result := TranscodeResult{ BlobID: req.BlobID, WorkspaceUUID: req.WorkspaceUUID, diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index 95c8755569..4000168aca 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -21,8 +21,10 @@ import ( "io" "mime/multipart" "net/textproto" + "net/url" "os" "path/filepath" + "strconv" "strings" "time" @@ -376,6 +378,140 @@ func (d *DatalakeStorage) SetParent(ctx context.Context, filename, parent string return nil } +func (d *DatalakeStorage) MultipartUploadStart(ctx context.Context, objectName, contentType string) (string, error) { + var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("objectName", objectName)) + url := fmt.Sprintf("%v/upload/multipart/%v/%v", d.baseURL, d.workspace, objectName) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(url) + req.Header.SetMethod(fasthttp.MethodPost) + req.Header.Add("Authorization", "Bearer "+d.token) + req.Header.SetContentType(contentType) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + logRequestError(logger, err, "request failed", resp) + return "", err + } + + if err := okResponse(resp); err != nil { + logRequestError(logger, err, "bad status code", resp) + return "", err + } + + var result struct { + UploadID string `json:"uploadId"` + } + err := json.Unmarshal(resp.Body(), &result) + + return result.UploadID, err +} + +func (d *DatalakeStorage) MultipartUploadPart(ctx context.Context, objectName, uploadID string, partNumber int, data []byte) (*MultipartPart, error) { + var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("uploadID", uploadID), zap.Int("partNumber", partNumber)) + params := url.Values{} + params.Add("uploadId", uploadID) + params.Add("partNumber", strconv.Itoa(partNumber)) + url := fmt.Sprintf("%v/upload/multipart/%v/%v/part?%v", d.baseURL, d.workspace, objectName, params.Encode()) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(url) + req.Header.SetMethod(fasthttp.MethodPut) + req.Header.Add("Authorization", "Bearer "+d.token) + req.Header.SetContentType("application/octet-stream") + req.Header.SetContentLength(len(data)) + req.SetBody(data) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + logRequestError(logger, err, "request failed", resp) + return nil, err + } + + if err := okResponse(resp); err != nil { + logRequestError(logger, err, "bad status code", resp) + return nil, err + } + + var part MultipartPart + err := json.Unmarshal(resp.Body(), &part) + + return &part, err +} + +func (d *DatalakeStorage) MultipartUploadComplete(ctx context.Context, objectName, uploadID string, parts []MultipartPart) error { + var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("uploadID", uploadID), zap.String("objectName", objectName)) + params := url.Values{} + params.Add("uploadId", uploadID) + url := fmt.Sprintf("%v/upload/multipart/%v/%v/complete?%v", d.baseURL, d.workspace, objectName, params.Encode()) + + body, err := json.Marshal(map[string]any{ + "parts": parts, + }) + + if err != nil { + logger.Debug("can not encode body", zap.Error(err)) + return err + } + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(url) + req.Header.SetMethod(fasthttp.MethodPost) + req.Header.Add("Authorization", "Bearer "+d.token) + req.Header.SetContentType("application/json") + req.SetBody(body) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + logRequestError(logger, err, "request failed", resp) + return err + } + + if err := okResponse(resp); err != nil { + logRequestError(logger, err, "bad status code", resp) + return err + } + + return nil +} + +func (d *DatalakeStorage) MultipartUploadCancel(ctx context.Context, objectName, uploadID string) error { + var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("uploadID", uploadID)) + params := url.Values{} + params.Add("uploadId", uploadID) + url := fmt.Sprintf("%v/upload/multipart/%v/%v/abort?%v", d.baseURL, d.workspace, objectName, params.Encode()) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(url) + req.Header.SetMethod(fasthttp.MethodPost) + req.Header.Add("Authorization", "Bearer "+d.token) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + logRequestError(logger, err, "request failed", resp) + return err + } + + if err := okResponse(resp); err != nil { + logRequestError(logger, err, "bad status code", resp) + return err + } + + return nil +} + func okResponse(res *fasthttp.Response) error { var statusOK = res.StatusCode() >= 200 && res.StatusCode() < 300 @@ -397,4 +533,5 @@ func logRequestError(logger *zap.Logger, err error, msg string, res *fasthttp.Re } var _ Storage = (*DatalakeStorage)(nil) +var _ MultipartStorage = (*DatalakeStorage)(nil) var _ MetaProvider = (*DatalakeStorage)(nil) diff --git a/internal/pkg/storage/storage.go b/internal/pkg/storage/storage.go index 0feb587825..78d28cdf13 100644 --- a/internal/pkg/storage/storage.go +++ b/internal/pkg/storage/storage.go @@ -46,6 +46,20 @@ type Storage interface { SetParent(ctx context.Context, fileName string, parentName string) error } +// MultipartPart represents uploaded multipart part +type MultipartPart struct { + PartNumber int `json:"partNumber"` + ETag string `json:"etag"` +} + +// MultipartStorage represents multipart-based storage +type MultipartStorage interface { + MultipartUploadStart(ctx context.Context, objectName, contentType string) (string, error) + MultipartUploadPart(ctx context.Context, objectName, uploadID string, partNumber int, data []byte) (*MultipartPart, error) + MultipartUploadComplete(ctx context.Context, objectName, uploadID string, parts []MultipartPart) error + MultipartUploadCancel(ctx context.Context, objectName, uploadID string) error +} + // NewStorageByURL creates a new storage based on the type from the url scheme, for example "datalake://my-datalake-endpoint" func NewStorageByURL(ctx context.Context, u *url.URL, storageType, token, workspace string) (Storage, error) { if workspace == "" { From a1a5bac7b5e041f74244cca8aa23abe2de0e5746 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Fri, 27 Jun 2025 10:41:44 +0700 Subject: [PATCH 44/52] fix ci and fmt issues Signed-off-by: Alexander Onnikov --- .golangci.yaml | 4 ++-- internal/pkg/mediaconvert/command.go | 5 +++-- internal/pkg/mediaconvert/coordinator.go | 2 +- internal/pkg/mediaconvert/multipart.go | 4 +++- internal/pkg/mediaconvert/strategy.go | 1 + internal/pkg/profile/profile.go | 9 +++++---- internal/pkg/profile/profile_test.go | 4 ++++ internal/pkg/storage/datalake.go | 4 ++++ 8 files changed, 23 insertions(+), 10 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 3a6223134d..2972e64e43 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -59,13 +59,13 @@ linters-settings: gocyclo: min-complexity: 30 dupl: - threshold: 150 + threshold: 500 funlen: lines: 240 statements: 120 goconst: min-len: 2 - min-occurrences: 2 + min-occurrences: 3 depguard: rules: main: diff --git a/internal/pkg/mediaconvert/command.go b/internal/pkg/mediaconvert/command.go index 895b5e08c2..100eddb956 100644 --- a/internal/pkg/mediaconvert/command.go +++ b/internal/pkg/mediaconvert/command.go @@ -120,7 +120,7 @@ func buildHLSCommand(profile profile.VideoProfile, opts *Options) []string { } } -func buildVideoCommand(profile profile.VideoProfile, opts *Options) []string { +func buildVideoCommand(profile profile.VideoProfile) []string { crf := profile.CRF if crf == 0 { crf = 23 @@ -162,7 +162,7 @@ func BuildVideoCommand(opts *Options) []string { var command = buildCommonCommand(opts) for _, profile := range opts.Profiles { - command = append(command, buildVideoCommand(profile, opts)...) + command = append(command, buildVideoCommand(profile)...) command = append(command, buildHLSCommand(profile, opts)...) command = append(command, filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, profile.Name))) } @@ -172,6 +172,7 @@ func BuildVideoCommand(opts *Options) []string { // BuildThumbnailCommand creates a command that creates a thumbnail for the input video func BuildThumbnailCommand(opts *Options) []string { return append([]string{}, + "-y", // Overwrite output files without asking. "-i", opts.Input, "-vframes", "1", "-update", "1", diff --git a/internal/pkg/mediaconvert/coordinator.go b/internal/pkg/mediaconvert/coordinator.go index d20520a203..3c57fbf455 100644 --- a/internal/pkg/mediaconvert/coordinator.go +++ b/internal/pkg/mediaconvert/coordinator.go @@ -90,8 +90,8 @@ func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo } if atomic.AddInt32(&s.activeTranscoding, 1) > int32(s.conf.MaxParallelTranscodingCount) { - atomic.AddInt32(&s.activeTranscoding, -1) s.logger.Debug("run out of resources for scaling") + // atomic.AddInt32(&s.activeTranscoding, -1) // TODO do not transcode } diff --git a/internal/pkg/mediaconvert/multipart.go b/internal/pkg/mediaconvert/multipart.go index f29e35ef7f..b153f62fcb 100644 --- a/internal/pkg/mediaconvert/multipart.go +++ b/internal/pkg/mediaconvert/multipart.go @@ -30,6 +30,7 @@ import ( // minimum size for multipart parts (backend requires >=5MiB for all but last part) const minPartSize = 5 * 1024 * 1024 +// MultipartUpload uploads chunks via multipart upload type MultipartUpload struct { logger *zap.Logger buffer *bytes.Buffer @@ -48,6 +49,7 @@ type MultipartUpload struct { bytesUploaded int64 } +// NewMultipartUpload creates a new multipart upload func NewMultipartUpload( ctx context.Context, multipartStorage storage.MultipartStorage, @@ -122,7 +124,7 @@ func (w *MultipartUpload) Terminate(ctx context.Context) error { w.logger.Debug("terminating multipart upload", zap.Int("parts", len(w.parts))) - // create new context in case the main context is cancelled + // create new context in case the main context is canceled cancelCtx := ctx if ctx.Err() != nil { var cancel context.CancelFunc diff --git a/internal/pkg/mediaconvert/strategy.go b/internal/pkg/mediaconvert/strategy.go index 387db9de3f..3fff4a80aa 100644 --- a/internal/pkg/mediaconvert/strategy.go +++ b/internal/pkg/mediaconvert/strategy.go @@ -22,6 +22,7 @@ import ( "github.com/hcengineering/stream/internal/pkg/resconv" ) +// VideoMeta contains information about the video type VideoMeta struct { Width int Height int diff --git a/internal/pkg/profile/profile.go b/internal/pkg/profile/profile.go index 7716ef6914..c84a3437ef 100644 --- a/internal/pkg/profile/profile.go +++ b/internal/pkg/profile/profile.go @@ -53,7 +53,7 @@ var profileOriginalT = VideoProfile{ Scale: false, VideoCodec: "libx264", AudioCodec: "aac", - CRF: 23, + CRF: 23, } // Profile360p is a profile for transcoding video in 360p @@ -89,7 +89,7 @@ var Profile720p = VideoProfile{ Bandwidth: 5000000, VideoCodec: "libx264", AudioCodec: "aac", - CRF: 25, + CRF: 25, } // Profile1080p is a profile for transcoding video in 1080p @@ -140,6 +140,7 @@ var Profile4320p = VideoProfile{ CRF: 22, } +// Profiles is a map of all supported profiles var Profiles = map[string]VideoProfile{ "360p": Profile360p, "480p": Profile480p, @@ -150,13 +151,13 @@ var Profiles = map[string]VideoProfile{ "4320p": Profile4320p, } +// MakeProfileOriginal creates a profile for original video without transcoding func MakeProfileOriginal(width, height int) VideoProfile { resolution := fmt.Sprintf("%v:%v", width, height) level := resconv.Level(resolution) bandwidth := resconv.Bandwidth(level) profile := profileOriginal - //profile.Name = level profile.Width = width profile.Height = height profile.Bandwidth = bandwidth @@ -164,13 +165,13 @@ func MakeProfileOriginal(width, height int) VideoProfile { return profile } +// MakeProfileOriginalT creates a profile for transcoding video in the original resolution func MakeProfileOriginalT(width, height int) VideoProfile { resolution := fmt.Sprintf("%v:%v", width, height) level := resconv.Level(resolution) bandwidth := resconv.Bandwidth(level) profile := profileOriginalT - //profile.Name = level profile.Width = width profile.Height = height profile.Bandwidth = bandwidth diff --git a/internal/pkg/profile/profile_test.go b/internal/pkg/profile/profile_test.go index a4ea1448fd..1ad67d7959 100644 --- a/internal/pkg/profile/profile_test.go +++ b/internal/pkg/profile/profile_test.go @@ -104,6 +104,7 @@ func TestMakeProfileOriginal(t *testing.T) { Width: 640, Height: 480, Bandwidth: 2000000, + CRF: 23, }, }, { @@ -117,6 +118,7 @@ func TestMakeProfileOriginal(t *testing.T) { Width: 2000, Height: 1200, Bandwidth: 8000000, + CRF: 23, }, }, } @@ -147,6 +149,7 @@ func TestMakeProfileOriginalT(t *testing.T) { Width: 1280, Height: 720, Bandwidth: 5000000, + CRF: 23, }, }, { @@ -160,6 +163,7 @@ func TestMakeProfileOriginalT(t *testing.T) { Width: 3840, Height: 2160, Bandwidth: 25000000, + CRF: 23, }, }, } diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index 4000168aca..b9560e0710 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -378,6 +378,7 @@ func (d *DatalakeStorage) SetParent(ctx context.Context, filename, parent string return nil } +// MultipartUploadStart creates a new multipart upload func (d *DatalakeStorage) MultipartUploadStart(ctx context.Context, objectName, contentType string) (string, error) { var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("objectName", objectName)) url := fmt.Sprintf("%v/upload/multipart/%v/%v", d.baseURL, d.workspace, objectName) @@ -410,6 +411,7 @@ func (d *DatalakeStorage) MultipartUploadStart(ctx context.Context, objectName, return result.UploadID, err } +// MultipartUploadPart uploads a part of a multipart upload func (d *DatalakeStorage) MultipartUploadPart(ctx context.Context, objectName, uploadID string, partNumber int, data []byte) (*MultipartPart, error) { var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("uploadID", uploadID), zap.Int("partNumber", partNumber)) params := url.Values{} @@ -445,6 +447,7 @@ func (d *DatalakeStorage) MultipartUploadPart(ctx context.Context, objectName, u return &part, err } +// MultipartUploadComplete completes a multipart upload func (d *DatalakeStorage) MultipartUploadComplete(ctx context.Context, objectName, uploadID string, parts []MultipartPart) error { var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("uploadID", uploadID), zap.String("objectName", objectName)) params := url.Values{} @@ -484,6 +487,7 @@ func (d *DatalakeStorage) MultipartUploadComplete(ctx context.Context, objectNam return nil } +// MultipartUploadCancel cancels a multipart upload func (d *DatalakeStorage) MultipartUploadCancel(ctx context.Context, objectName, uploadID string) error { var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("uploadID", uploadID)) params := url.Values{} From 4c2e68af493cfeb5136144ebd8217a30a35de98b Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Mon, 30 Jun 2025 10:45:57 +0700 Subject: [PATCH 45/52] fix: use different paths for recording and transcoding (#21) Signed-off-by: Alexander Onnikov --- internal/pkg/mediaconvert/coordinator.go | 14 +++++++++---- internal/pkg/mediaconvert/transcoder.go | 26 +++++++++++++++--------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/internal/pkg/mediaconvert/coordinator.go b/internal/pkg/mediaconvert/coordinator.go index 3c57fbf455..519e0d51ba 100644 --- a/internal/pkg/mediaconvert/coordinator.go +++ b/internal/pkg/mediaconvert/coordinator.go @@ -38,9 +38,12 @@ import ( "go.uber.org/zap" ) +var recordingDir = "r" + // StreamCoordinator represents manager for streams. It creates a new stream for a client and manages it's life cycle. type StreamCoordinator struct { conf *config.Config + outputDir string uploadOptions uploader.Options activeTranscoding int32 @@ -59,18 +62,21 @@ var _ handler.LengthDeferrerDataStore = (*StreamCoordinator)(nil) // NewStreamCoordinator creates a new scheduler for transcode operations. func NewStreamCoordinator(ctx context.Context, c *config.Config) *StreamCoordinator { + outputDir := filepath.Join(c.OutputDir, recordingDir) + return &StreamCoordinator{ - conf: c, + conf: c, + outputDir: outputDir, uploadOptions: uploader.Options{ RetryDelay: time.Millisecond * 100, Timeout: c.Timeout, WorkerCount: uint32(c.MaxThreadCount), RetryCount: 10, BufferSize: 128, - Dir: c.OutputDir, + Dir: outputDir, }, mainContext: ctx, - logger: log.FromContext(ctx).With(zap.String("Scheduler", c.OutputDir)), + logger: log.FromContext(ctx).With(zap.String("Scheduler", outputDir)), } } @@ -115,7 +121,7 @@ func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo var commandOptions = Options{ Input: "pipe:0", - OutputDir: s.conf.OutputDir, + OutputDir: s.outputDir, Threads: s.conf.MaxThreadCount, UploadID: info.ID, Profiles: profiles, diff --git a/internal/pkg/mediaconvert/transcoder.go b/internal/pkg/mediaconvert/transcoder.go index 1972f171dd..cffa166776 100644 --- a/internal/pkg/mediaconvert/transcoder.go +++ b/internal/pkg/mediaconvert/transcoder.go @@ -34,19 +34,25 @@ import ( "gopkg.in/vansante/go-ffprobe.v2" ) +var transcodingDir = "t" + // Transcoder process one transcoding task type Transcoder struct { - ctx context.Context - cfg *config.Config - logger *zap.Logger + ctx context.Context + cfg *config.Config + outputDir string + logger *zap.Logger } // NewTranscoder creates a new instance of task transcoder func NewTranscoder(ctx context.Context, cfg *config.Config) *Transcoder { + outputDir := filepath.Join(cfg.OutputDir, transcodingDir) + var p = &Transcoder{ - cfg: cfg, - ctx: ctx, - logger: log.FromContext(ctx).With(zap.String("transcoding", "transcoder")), + cfg: cfg, + ctx: ctx, + outputDir: outputDir, + logger: log.FromContext(ctx).With(zap.String("transcoding", "transcoder")), } return p @@ -67,7 +73,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er } logger.Debug("phase 2: preparing fs") - var destinationFolder = filepath.Join(p.cfg.OutputDir, task.ID) + var destinationFolder = filepath.Join(p.outputDir, task.ID) var _, filename = filepath.Split(task.Source) err = os.MkdirAll(destinationFolder, os.ModePerm) if err != nil { @@ -139,7 +145,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er var opts = Options{ Input: sourceFilePath, - OutputDir: p.cfg.OutputDir, + OutputDir: p.outputDir, LogLevel: LogLevel(p.cfg.LogLevel), Profiles: profiles, UploadID: task.ID, @@ -158,9 +164,9 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er SourceFile: sourceFilePath, }) - err = manifest.GenerateHLSPlaylist(profiles, p.cfg.OutputDir, opts.UploadID) + err = manifest.GenerateHLSPlaylist(profiles, p.outputDir, opts.UploadID) if err != nil { - logger.Error("can not generate hls playlist", zap.String("out", p.cfg.OutputDir), zap.String("uploadID", opts.UploadID)) + logger.Error("can not generate hls playlist", zap.String("out", p.outputDir), zap.String("uploadID", opts.UploadID)) return nil, errors.Wrapf(err, "can not generate hls playlist") } From 54d792ed7bd5c86e979c24792877da5222b3423e Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Tue, 1 Jul 2025 16:56:22 +0700 Subject: [PATCH 46/52] fix: provide service name in token extra Signed-off-by: Alexander Onnikov --- internal/pkg/mediaconvert/scheduler.go | 2 +- internal/pkg/mediaconvert/transcoder.go | 2 +- internal/pkg/token/token.go | 21 +++++++++------------ internal/pkg/token/token_test.go | 12 ++++-------- 4 files changed, 15 insertions(+), 22 deletions(-) diff --git a/internal/pkg/mediaconvert/scheduler.go b/internal/pkg/mediaconvert/scheduler.go index 2f3e2f6f55..08e9b18dbc 100644 --- a/internal/pkg/mediaconvert/scheduler.go +++ b/internal/pkg/mediaconvert/scheduler.go @@ -111,7 +111,7 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { defer logger.Debug("finished") logger.Debug("phase 1: get a token") - var tokenString, err = token.NewToken(p.cfg.ServerSecret, task.Workspace, "stream", "datalake") + var tokenString, err = token.NewToken(p.cfg.ServerSecret, task.Workspace, "stream") if err != nil { logger.Error("can not create token", zap.Error(err)) return diff --git a/internal/pkg/mediaconvert/transcoder.go b/internal/pkg/mediaconvert/transcoder.go index cffa166776..4d7231dc93 100644 --- a/internal/pkg/mediaconvert/transcoder.go +++ b/internal/pkg/mediaconvert/transcoder.go @@ -66,7 +66,7 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er defer logger.Debug("finished") logger.Debug("phase 1: get a token") - var tokenString, err = token.NewToken(p.cfg.ServerSecret, task.Workspace, "stream", "datalake") + var tokenString, err = token.NewToken(p.cfg.ServerSecret, task.Workspace, "stream") if err != nil { logger.Error("can not create token", zap.Error(err)) return nil, errors.Wrapf(err, "can not create token") diff --git a/internal/pkg/token/token.go b/internal/pkg/token/token.go index 1f81a76484..74da213b74 100644 --- a/internal/pkg/token/token.go +++ b/internal/pkg/token/token.go @@ -16,7 +16,6 @@ package token import ( "fmt" - "time" "github.com/golang-jwt/jwt/v5" "github.com/google/uuid" @@ -24,21 +23,19 @@ import ( // Token represents Claims for the platform token type Token struct { - jwt.RegisteredClaims - Account string `json:"account"` - Workspace string `json:"workspace,omitempty"` - Extra map[string]interface{} `json:"extra,omitempty"` + jwt.MapClaims + Account string `json:"account"` + Workspace string `json:"workspace,omitempty"` + Extra map[string]any `json:"extra,omitempty"` } // NewToken creates a new platform token -func NewToken(serverSecret, workspace, issuer, audience string) (string, error) { +func NewToken(serverSecret, workspace, service string) (string, error) { var res = Token{ Account: uuid.NewString(), Workspace: workspace, - RegisteredClaims: jwt.RegisteredClaims{ - Issuer: issuer, - Audience: jwt.ClaimStrings{audience}, - ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Hour * 12)), + Extra: map[string]any{ + "service": service, }, } return res.Encode(serverSecret) @@ -58,7 +55,7 @@ func (t *Token) Encode(serverSecret string) (string, error) { // Decode decodes a token by a passed configuration func Decode(secretKey, tokenString string) (*Token, error) { - token, err := jwt.ParseWithClaims(tokenString, &Token{}, func(token *jwt.Token) (interface{}, error) { + token, err := jwt.ParseWithClaims(tokenString, &Token{}, func(token *jwt.Token) (any, error) { if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) } @@ -73,5 +70,5 @@ func Decode(secretKey, tokenString string) (*Token, error) { return claims, nil } - return nil, fmt.Errorf("invalid token: can't parse claims") + return nil, fmt.Errorf("invalid token") } diff --git a/internal/pkg/token/token_test.go b/internal/pkg/token/token_test.go index 93d97bdb53..7c46879efb 100644 --- a/internal/pkg/token/token_test.go +++ b/internal/pkg/token/token_test.go @@ -15,26 +15,22 @@ package token_test import ( "testing" - "time" - "github.com/golang-jwt/jwt/v5" "github.com/hcengineering/stream/internal/pkg/token" "github.com/stretchr/testify/require" ) func Test_GenerateSimpleServiceToken(t *testing.T) { - var _, err = token.NewToken("secret", "ws", "issuer", "aud") + var _, err = token.NewToken("secret", "ws", "stream") require.NoError(t, err) } func Test_ParseSimpleServiceToken(t *testing.T) { const secret = "secret" - tokenString, err := token.NewToken(secret, "ws", "issuer", "aud") + tokenString, err := token.NewToken(secret, "ws", "stream") require.NoError(t, err) tok, err := token.Decode(secret, tokenString) require.NoError(t, err) - require.Equal(t, tok.Issuer, "issuer") - require.Equal(t, tok.Audience, jwt.ClaimStrings{"aud"}) - require.Equal(t, tok.Workspace, "ws") - require.True(t, tok.ExpiresAt.After(time.Now())) + require.Equal(t, "ws", tok.Workspace) + require.Equal(t, "stream", tok.Extra["service"]) } From a68925459af13c8f6db236effb684311f4b79ecc Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Tue, 1 Jul 2025 23:51:15 +0700 Subject: [PATCH 47/52] fix: do not cache m3u8 files Signed-off-by: Alexander Onnikov --- internal/pkg/storage/datalake.go | 5 ++++- internal/pkg/storage/s3.go | 2 +- internal/pkg/storage/storage.go | 7 ++++++- internal/pkg/uploader/uploader.go | 5 ++++- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index b9560e0710..e76e0b1a87 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -84,7 +84,7 @@ func getObjectKeyFromPath(s string) string { } // PutFile uploads file to the datalake -func (d *DatalakeStorage) PutFile(ctx context.Context, fileName string) error { +func (d *DatalakeStorage) PutFile(ctx context.Context, fileName string, options PutOptions) error { // #nosec file, err := os.Open(fileName) if err != nil { @@ -127,6 +127,9 @@ func (d *DatalakeStorage) PutFile(ctx context.Context, fileName string) error { req.Header.SetMethod(fasthttp.MethodPost) req.Header.Add("Authorization", "Bearer "+d.token) req.Header.SetContentType(writer.FormDataContentType()) + if options.NoCache { + req.Header.Add("Cache-Control", "max-age=0, must-revalidate") + } req.SetBody(body.Bytes()) if err := d.client.Do(req, res); err != nil { diff --git a/internal/pkg/storage/s3.go b/internal/pkg/storage/s3.go index e4f3050d09..aea0adec5b 100644 --- a/internal/pkg/storage/s3.go +++ b/internal/pkg/storage/s3.go @@ -84,7 +84,7 @@ func (u *S3Storage) DeleteFile(ctx context.Context, fileName string) error { } // PutFile uploads file to the s3 storage -func (u *S3Storage) PutFile(ctx context.Context, fileName string) error { +func (u *S3Storage) PutFile(ctx context.Context, fileName string, options PutOptions) error { var _, objectKey = filepath.Split(fileName) var logger = u.logger.With(zap.String("upload", u.bucketName), zap.String("fileName", fileName)) diff --git a/internal/pkg/storage/storage.go b/internal/pkg/storage/storage.go index 78d28cdf13..cd9c9b8aac 100644 --- a/internal/pkg/storage/storage.go +++ b/internal/pkg/storage/storage.go @@ -37,9 +37,14 @@ type BlobInfo struct { ETag string } +// PutOptions represents options for the PutFile operation +type PutOptions struct { + NoCache bool +} + // Storage represents file-based storage type Storage interface { - PutFile(ctx context.Context, fileName string) error + PutFile(ctx context.Context, fileName string, options PutOptions) error DeleteFile(ctx context.Context, fileName string) error GetFile(ctx context.Context, fileName, destination string) error StatFile(ctx context.Context, fileName string) (*BlobInfo, error) diff --git a/internal/pkg/uploader/uploader.go b/internal/pkg/uploader/uploader.go index baffcaa82e..2bdee38c76 100644 --- a/internal/pkg/uploader/uploader.go +++ b/internal/pkg/uploader/uploader.go @@ -316,7 +316,10 @@ func (u *uploaderImpl) uploadAndDelete(f string) { for attempt := range u.options.RetryCount { logger = logger.With(zap.Int("attempt", attempt)) var putCtx, putCancel = context.WithTimeout(u.uploadCtx, u.options.Timeout) - var err = u.storage.PutFile(putCtx, f) + var putOptions = storage.PutOptions{ + NoCache: u.shouldDeleteOnStop(f), + } + var err = u.storage.PutFile(putCtx, f, putOptions) putCancel() if err != nil { From 0e4b200720d1420e68ba83dede591a752b3e5596 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Thu, 3 Jul 2025 18:39:06 +0700 Subject: [PATCH 48/52] UBERF-12153 Transcode only created blobs Signed-off-by: Alexander Onnikov --- internal/pkg/queue/queue.go | 10 ++++++---- internal/pkg/queue/worker.go | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/pkg/queue/queue.go b/internal/pkg/queue/queue.go index d9f6a54208..b79786f67d 100644 --- a/internal/pkg/queue/queue.go +++ b/internal/pkg/queue/queue.go @@ -36,14 +36,16 @@ type TranscodeRequest struct { BlobID string WorkspaceUUID string ContentType string + Source map[string]any } // TranscodeResult represents transcode result. type TranscodeResult struct { - BlobID string `json:"blobId"` - WorkspaceUUID string `json:"workspaceUuid"` - Playlist string `json:"playlist"` - Thumbnail string `json:"thumbnail"` + BlobID string `json:"blobId"` + WorkspaceUUID string `json:"workspaceUuid"` + Source map[string]any `json:"source"` + Playlist string `json:"playlist"` + Thumbnail string `json:"thumbnail"` } // ConsumerOptions represents options for the consumer diff --git a/internal/pkg/queue/worker.go b/internal/pkg/queue/worker.go index 60def900d5..a80b4c702a 100644 --- a/internal/pkg/queue/worker.go +++ b/internal/pkg/queue/worker.go @@ -115,6 +115,7 @@ func (w *Worker) processMessage(ctx context.Context, msg kafka.Message, logger * result := TranscodeResult{ BlobID: req.BlobID, WorkspaceUUID: req.WorkspaceUUID, + Source: req.Source, Playlist: res.Playlist, Thumbnail: res.Thumbnail, } From aeaf67a21218298bfc9d547eeee1b912696233de Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Tue, 8 Jul 2025 20:55:01 +0700 Subject: [PATCH 49/52] fix: set parent for recording blobs Signed-off-by: Alexander Onnikov --- internal/pkg/manifest/hls.go | 19 +++++++++++++++++-- internal/pkg/manifest/hls_test.go | 19 +++++++++++++++++-- internal/pkg/mediaconvert/command.go | 3 ++- internal/pkg/mediaconvert/command_test.go | 6 +++--- internal/pkg/mediaconvert/coordinator.go | 2 ++ internal/pkg/mediaconvert/scheduler.go | 4 ++-- internal/pkg/mediaconvert/stream.go | 4 ++-- internal/pkg/mediaconvert/transcoder.go | 4 ++-- internal/pkg/storage/datalake.go | 5 +++++ 9 files changed, 52 insertions(+), 14 deletions(-) diff --git a/internal/pkg/manifest/hls.go b/internal/pkg/manifest/hls.go index 7b2934ea46..cc263b5a16 100644 --- a/internal/pkg/manifest/hls.go +++ b/internal/pkg/manifest/hls.go @@ -24,7 +24,7 @@ import ( // GenerateHLSPlaylist generates master file for master files for resolution levels func GenerateHLSPlaylist(profiles []profile.VideoProfile, outputPath, uploadID string) error { - p := filepath.Join(outputPath, uploadID, fmt.Sprintf("%v_master.m3u8", uploadID)) + p := filepath.Join(outputPath, uploadID, MasterPlaylistFileName(uploadID)) d := filepath.Dir(p) _ = os.MkdirAll(d, os.ModePerm) // #nosec @@ -48,7 +48,7 @@ func GenerateHLSPlaylist(profiles []profile.VideoProfile, outputPath, uploadID s return err } - _, err = file.WriteString(fmt.Sprintf("%s_%s_master.m3u8\n", uploadID, profile.Name)) + _, err = file.WriteString(PlaylistFileName(uploadID, profile.Name) + "\n") if err != nil { return err } @@ -56,3 +56,18 @@ func GenerateHLSPlaylist(profiles []profile.VideoProfile, outputPath, uploadID s return nil } + +// MasterPlaylistFileName generates master playlist file name +func MasterPlaylistFileName(source string) string { + return fmt.Sprintf("%s_master.m3u8", source) +} + +// PlaylistFileName generates profile playlist file name +func PlaylistFileName(source, profile string) string { + return fmt.Sprintf("%s_%s.m3u8", source, profile) +} + +// ThumbnailFileName generates thumbnail file name +func ThumbnailFileName(source string) string { + return fmt.Sprintf("%s.jpg", source) +} diff --git a/internal/pkg/manifest/hls_test.go b/internal/pkg/manifest/hls_test.go index 5ad6dd6758..b4f8d13e58 100644 --- a/internal/pkg/manifest/hls_test.go +++ b/internal/pkg/manifest/hls_test.go @@ -39,7 +39,7 @@ func TestGenerateHLSPlaylist(t *testing.T) { err := manifest.GenerateHLSPlaylist(profiles, "", uploadID) require.NoError(t, err) - outputPath := filepath.Join(uploadID, uploadID+"_master.m3u8") + outputPath := filepath.Join(uploadID, manifest.MasterPlaylistFileName(uploadID)) _, err = os.Stat(outputPath) require.NoError(t, err, "Master playlist file should exist") @@ -53,7 +53,22 @@ func TestGenerateHLSPlaylist(t *testing.T) { require.Contains(t, playlistContent, "#EXTM3U", "File must start with #EXTM3U") for _, prof := range profiles { - expectedLine := uploadID + "_" + prof.Name + "_master.m3u8" + expectedLine := manifest.PlaylistFileName(uploadID, prof.Name) require.Contains(t, playlistContent, expectedLine, "Missing expected reference: "+expectedLine) } } + +func TestMasterPlaylistFileName(t *testing.T) { + filename := manifest.MasterPlaylistFileName("example") + require.Equal(t, "example_master.m3u8", filename) +} + +func TestPlaylistFileName(t *testing.T) { + filename := manifest.PlaylistFileName("example", "720p") + require.Equal(t, "example_720p.m3u8", filename) +} + +func TestThumbnailFileName(t *testing.T) { + filename := manifest.ThumbnailFileName("example") + require.Equal(t, "example.jpg", filename) +} diff --git a/internal/pkg/mediaconvert/command.go b/internal/pkg/mediaconvert/command.go index 100eddb956..7017359282 100644 --- a/internal/pkg/mediaconvert/command.go +++ b/internal/pkg/mediaconvert/command.go @@ -27,6 +27,7 @@ import ( "github.com/pkg/errors" "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/manifest" "github.com/hcengineering/stream/internal/pkg/profile" "go.uber.org/zap" ) @@ -164,7 +165,7 @@ func BuildVideoCommand(opts *Options) []string { for _, profile := range opts.Profiles { command = append(command, buildVideoCommand(profile)...) command = append(command, buildHLSCommand(profile, opts)...) - command = append(command, filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_master.m3u8", opts.UploadID, profile.Name))) + command = append(command, filepath.Join(opts.OutputDir, opts.UploadID, manifest.PlaylistFileName(opts.UploadID, profile.Name))) } return command } diff --git a/internal/pkg/mediaconvert/command_test.go b/internal/pkg/mediaconvert/command_test.go index 67beff10b4..c516f89e74 100644 --- a/internal/pkg/mediaconvert/command_test.go +++ b/internal/pkg/mediaconvert/command_test.go @@ -52,7 +52,7 @@ func Test_BuildVideoCommand_Scaling(t *testing.T) { Profiles: profiles, }) - const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -c:a aac -c:v libx264 -preset veryfast -crf 25 -g 60 -vf scale=-2:720 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p_master.m3u8 -map 0:v:0 -map 0:a? -c:a aac -c:v libx264 -preset veryfast -crf 27 -g 60 -vf scale=-2:480 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p_master.m3u8` + const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -c:a aac -c:v libx264 -preset veryfast -crf 25 -g 60 -vf scale=-2:720 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p.m3u8 -map 0:v:0 -map 0:a? -c:a aac -c:v libx264 -preset veryfast -crf 27 -g 60 -vf scale=-2:480 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p.m3u8` require.Contains(t, expected, strings.Join(scaleCommand, " ")) } @@ -71,7 +71,7 @@ func Test_BuildVideoCommand_Original(t *testing.T) { Profiles: profiles, }) - const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -c:a copy -c:v copy -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_orig.ts test/1/1_orig_master.m3u8` + const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -c:a copy -c:v copy -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_orig.ts test/1/1_orig.m3u8` require.Contains(t, expected, strings.Join(rawCommand, " ")) } @@ -90,7 +90,7 @@ func Test_BuildVideoCommand_OriginalT(t *testing.T) { Profiles: profiles, }) - const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_orig.ts test/1/1_orig_master.m3u8` + const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_orig.ts test/1/1_orig.m3u8` require.Contains(t, expected, strings.Join(rawCommand, " ")) } diff --git a/internal/pkg/mediaconvert/coordinator.go b/internal/pkg/mediaconvert/coordinator.go index 519e0d51ba..cfad9916d4 100644 --- a/internal/pkg/mediaconvert/coordinator.go +++ b/internal/pkg/mediaconvert/coordinator.go @@ -31,6 +31,7 @@ import ( "github.com/google/uuid" "github.com/hcengineering/stream/internal/pkg/config" "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/manifest" "github.com/hcengineering/stream/internal/pkg/sharedpipe" "github.com/hcengineering/stream/internal/pkg/storage" "github.com/hcengineering/stream/internal/pkg/uploader" @@ -133,6 +134,7 @@ func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo // setup content uploader for transcoded outputs var opts = s.uploadOptions opts.Dir = filepath.Join(opts.Dir, info.ID) + opts.Source = manifest.MasterPlaylistFileName(info.ID) // create storage backend var stg, err = storage.NewStorageByURL(s.mainContext, s.conf.Endpoint(), s.conf.EndpointURL.Scheme, info.MetaData["token"], info.MetaData["workspace"]) diff --git a/internal/pkg/mediaconvert/scheduler.go b/internal/pkg/mediaconvert/scheduler.go index 08e9b18dbc..8235f44b02 100644 --- a/internal/pkg/mediaconvert/scheduler.go +++ b/internal/pkg/mediaconvert/scheduler.go @@ -267,8 +267,8 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { var result = TaskResult{ Width: videoStream.Width, Height: videoStream.Height, - Playlist: task.ID + "_master.m3u8", - Thumbnail: task.ID + ".jpg", + Playlist: manifest.MasterPlaylistFileName(task.ID), + Thumbnail: manifest.ThumbnailFileName(task.ID), } logger.Debug( diff --git a/internal/pkg/mediaconvert/stream.go b/internal/pkg/mediaconvert/stream.go index 135ce98997..95524361c9 100644 --- a/internal/pkg/mediaconvert/stream.go +++ b/internal/pkg/mediaconvert/stream.go @@ -180,8 +180,8 @@ func (w *Stream) FinishUpload(ctx context.Context) error { w.info.ID, &storage.Metadata{ "hls": map[string]any{ - "source": w.info.ID + "_master.m3u8", - "thumbnail": w.info.ID + ".jpg", + "source": manifest.MasterPlaylistFileName(w.info.ID), + "thumbnail": manifest.ThumbnailFileName(w.info.ID), }, }, ) diff --git a/internal/pkg/mediaconvert/transcoder.go b/internal/pkg/mediaconvert/transcoder.go index 4d7231dc93..a3fc1b738d 100644 --- a/internal/pkg/mediaconvert/transcoder.go +++ b/internal/pkg/mediaconvert/transcoder.go @@ -210,8 +210,8 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er var result = TaskResult{ Width: videoStream.Width, Height: videoStream.Height, - Playlist: task.ID + "_master.m3u8", - Thumbnail: task.ID + ".jpg", + Playlist: manifest.MasterPlaylistFileName(task.ID), + Thumbnail: manifest.ThumbnailFileName(task.ID), } if metaProvider, ok := remoteStorage.(storage.MetaProvider); ok { diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index e76e0b1a87..9207a88ed9 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -349,6 +349,11 @@ func (d *DatalakeStorage) SetParent(ctx context.Context, filename, parent string var objectKey = getObjectKeyFromPath(filename) var parentKey = getObjectKeyFromPath(parent) + if objectKey == parentKey { + // no need to set parent for itself + return nil + } + req := fasthttp.AcquireRequest() defer fasthttp.ReleaseRequest(req) req.SetRequestURI(d.baseURL + "/blob/" + d.workspace + "/" + objectKey + "/parent") From 36fe1deb355ffef326af2543676ee7ea5efc35b4 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Thu, 23 Oct 2025 12:10:38 +0700 Subject: [PATCH 50/52] Do not transcode while recording Signed-off-by: Alexander Onnikov --- internal/pkg/mediaconvert/coordinator.go | 29 +------ internal/pkg/mediaconvert/stream.go | 101 ++++------------------- 2 files changed, 18 insertions(+), 112 deletions(-) diff --git a/internal/pkg/mediaconvert/coordinator.go b/internal/pkg/mediaconvert/coordinator.go index cfad9916d4..bd93d8a61f 100644 --- a/internal/pkg/mediaconvert/coordinator.go +++ b/internal/pkg/mediaconvert/coordinator.go @@ -23,7 +23,6 @@ import ( "strconv" "strings" "sync" - "sync/atomic" "time" "github.com/pkg/errors" @@ -47,8 +46,6 @@ type StreamCoordinator struct { outputDir string uploadOptions uploader.Options - activeTranscoding int32 - mainContext context.Context logger *zap.Logger @@ -96,12 +93,6 @@ func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo done: make(chan struct{}), } - if atomic.AddInt32(&s.activeTranscoding, 1) > int32(s.conf.MaxParallelTranscodingCount) { - s.logger.Debug("run out of resources for scaling") - // atomic.AddInt32(&s.activeTranscoding, -1) - // TODO do not transcode - } - width, err := strconv.Atoi(info.MetaData["width"]) if err != nil { return nil, errors.Wrapf(err, "can not parse video width: %v", info.MetaData["width"]) @@ -118,15 +109,6 @@ func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo Codec: extractCodec(info.MetaData["contentType"]), ContentType: extractContentType(info.MetaData["contentType"]), } - profiles := FastTranscodingProfiles(meta) - - var commandOptions = Options{ - Input: "pipe:0", - OutputDir: s.outputDir, - Threads: s.conf.MaxThreadCount, - UploadID: info.ID, - Profiles: profiles, - } if s.conf.EndpointURL != nil { s.logger.Sugar().Debugf("initializing uploader for %v", info) @@ -153,22 +135,13 @@ func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo } stream.multipart = multipart } - // uploader for processed outputs - var contentUploader = uploader.New(s.mainContext, stg, opts) - stream.contentUploader = contentUploader } s.streams.Store(stream.info.ID, stream) - if err := stream.start(s.mainContext, &commandOptions); err != nil { + if err := stream.start(s.mainContext); err != nil { return nil, err } - go func() { - stream.commandGroup.Wait() - atomic.AddInt32(&s.activeTranscoding, -1) - close(stream.done) - }() - s.manageTimeout(stream) s.logger.Debug("NewUpload", zap.String("done", info.ID)) diff --git a/internal/pkg/mediaconvert/stream.go b/internal/pkg/mediaconvert/stream.go index 95524361c9..fbb671c911 100644 --- a/internal/pkg/mediaconvert/stream.go +++ b/internal/pkg/mediaconvert/stream.go @@ -17,31 +17,26 @@ package mediaconvert import ( "context" "io" - "os/exec" "sync" "github.com/pkg/errors" - "github.com/hcengineering/stream/internal/pkg/manifest" "github.com/hcengineering/stream/internal/pkg/sharedpipe" "github.com/hcengineering/stream/internal/pkg/storage" - "github.com/hcengineering/stream/internal/pkg/uploader" "github.com/tus/tusd/v2/pkg/handler" "go.uber.org/zap" ) // Stream manages client's input and transcodes it based on the passed configuration type Stream struct { - contentUploader uploader.Uploader - logger *zap.Logger - info handler.FileInfo - writer *sharedpipe.Writer - reader *sharedpipe.Reader - storage storage.Storage - multipart *MultipartUpload + logger *zap.Logger + info handler.FileInfo + writer *sharedpipe.Writer + reader *sharedpipe.Reader + storage storage.Storage + multipart *MultipartUpload - commandGroup sync.WaitGroup - done chan struct{} + done chan struct{} } var _ handler.Upload = (*Stream)(nil) @@ -108,16 +103,6 @@ func (w *Stream) Terminate(ctx context.Context) error { var wg sync.WaitGroup - // cancel upload if in progress - if w.contentUploader != nil { - wg.Add(1) - go func() { - defer wg.Done() - w.commandGroup.Wait() - w.contentUploader.Cancel() - }() - } - // cancel multipart upload if in progress if w.multipart != nil { wg.Add(1) @@ -131,6 +116,9 @@ func (w *Stream) Terminate(ctx context.Context) error { wg.Wait() + // Signal that the stream is done + close(w.done) + return nil } @@ -154,15 +142,7 @@ func (w *Stream) FinishUpload(ctx context.Context) error { } var wg sync.WaitGroup - - if w.contentUploader != nil { - wg.Add(1) - go func() { - defer wg.Done() - w.commandGroup.Wait() - w.contentUploader.Stop() - }() - } + var completeErr error // finalize raw multipart stream if supported if w.multipart != nil { @@ -171,30 +151,18 @@ func (w *Stream) FinishUpload(ctx context.Context) error { defer wg.Done() if err := w.multipart.Complete(ctx); err != nil { w.logger.Error("multipart upload complete failed", zap.Error(err)) + completeErr = err return } - - if metaProvider, ok := w.storage.(storage.MetaProvider); ok { - metaErr := metaProvider.PatchMeta( - ctx, - w.info.ID, - &storage.Metadata{ - "hls": map[string]any{ - "source": manifest.MasterPlaylistFileName(w.info.ID), - "thumbnail": manifest.ThumbnailFileName(w.info.ID), - }, - }, - ) - if metaErr != nil { - w.logger.Error("can not patch the source file", zap.Error(metaErr)) - } - } }() } wg.Wait() - return nil + // Signal that the stream is done + close(w.done) + + return completeErr } // AsConcatableUpload returns tusd handler.ConcatableUpload @@ -203,43 +171,8 @@ func (s *StreamCoordinator) AsConcatableUpload(upload handler.Upload) handler.Co return upload.(*Stream) } -func (w *Stream) start(ctx context.Context, options *Options) error { +func (w *Stream) start(ctx context.Context) error { defer w.logger.Debug("start done") w.reader = w.writer.Transpile() - if err := manifest.GenerateHLSPlaylist(options.Profiles, options.OutputDir, options.UploadID); err != nil { - return err - } - - var argsSlice = [][]string{ - BuildThumbnailCommand(options), - BuildVideoCommand(options), - } - - var cmds []*exec.Cmd - for idx, args := range argsSlice { - reader := w.reader - if idx > 0 { - reader = w.writer.Transpile() - } - - cmd, cmdErr := newFfmpegCommand(ctx, reader, args) - if cmdErr != nil { - w.logger.Error("can not create a new command", zap.Error(cmdErr), zap.Strings("args", args)) - return errors.Wrapf(cmdErr, "can not create a new command") - } - cmds = append(cmds, cmd) - } - - w.commandGroup.Add(1) - go func() { - defer w.commandGroup.Done() - executor := NewCommandExecutor(ctx) - if execErr := executor.Execute(cmds); execErr != nil { - w.logger.Error("can not execute command", zap.Error(execErr)) - } - }() - - go w.contentUploader.Start() - return nil } From bca70dea13caa9136128bcd05c48ac7013a831f7 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Fri, 24 Oct 2025 00:06:47 +0700 Subject: [PATCH 51/52] Open telemetry support Signed-off-by: Alexander Onnikov --- cmd/stream/main.go | 32 ++--- go.mod | 35 ++++- go.sum | 93 +++++++++++--- internal/pkg/config/config.go | 9 +- .../{mediaconvert => executor}/executor.go | 36 +++--- .../executor_test.go | 13 +- internal/pkg/mediaconvert/scheduler.go | 58 +++++---- internal/pkg/mediaconvert/stream.go | 24 ++++ internal/pkg/mediaconvert/transcoder.go | 12 +- internal/pkg/queue/worker.go | 8 +- internal/pkg/storage/datalake.go | 121 ++++++++++++++++-- internal/pkg/tracing/tracing.go | 53 ++++++++ internal/pkg/uploader/uploader.go | 30 +++-- 13 files changed, 402 insertions(+), 122 deletions(-) rename internal/pkg/{mediaconvert => executor}/executor.go (77%) rename internal/pkg/{mediaconvert => executor}/executor_test.go (89%) create mode 100644 internal/pkg/tracing/tracing.go diff --git a/cmd/stream/main.go b/cmd/stream/main.go index 37174f32af..3b00fd7d0c 100644 --- a/cmd/stream/main.go +++ b/cmd/stream/main.go @@ -16,6 +16,7 @@ package main import ( "context" + "errors" "net/http" "sync" @@ -24,8 +25,6 @@ import ( "syscall" "time" - "github.com/getsentry/sentry-go" - sentryhttp "github.com/getsentry/sentry-go/http" "go.uber.org/zap" "github.com/hcengineering/stream/internal/pkg/api/v1/recording" @@ -33,6 +32,7 @@ import ( "github.com/hcengineering/stream/internal/pkg/config" "github.com/hcengineering/stream/internal/pkg/log" "github.com/hcengineering/stream/internal/pkg/queue" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" ) func main() { @@ -53,16 +53,14 @@ func main() { } logger.Sugar().Debug("using config", zap.Any("config", cfg)) - if cfg.SentryDsn != "" { - if err := sentry.Init(sentry.ClientOptions{ - Dsn: cfg.SentryDsn, - Tags: map[string]string{"application": "stream"}, - }); err != nil { - logger.Sugar().Fatalf("sentry.Init: %s", err) - } - // ensure buffered events are sent before exit - defer sentry.Flush(2 * time.Second) + // Set up OpenTelemetry. + otelShutdown, err := setupOTelSDK(ctx, cfg) + if err != nil { + panic(err.Error()) } + defer func() { + err = errors.Join(err, otelShutdown(context.Background())) + }() var recordingHandler = recording.NewHandler(ctx, cfg) var transcodingHandler = transcoding.NewHandler(ctx, cfg) @@ -73,15 +71,11 @@ func main() { mux.Handle("/recording", http.StripPrefix("/recording", recordingHandler)) mux.Handle("/transcoding", http.StripPrefix("/transcoding", transcodingHandler)) - // wrap with Sentry HTTP handler if enabled + // Wrap handler with OpenTelemetry instrumentation var handler http.Handler = mux - if cfg.SentryDsn != "" { - sentryHandler := sentryhttp.New(sentryhttp.Options{ - Repanic: true, - WaitForDelivery: true, - Timeout: 2 * time.Second, - }) - handler = sentryHandler.Handle(mux) + if cfg.OtelEnabled && cfg.OtelTracesEnabled { + handler = otelhttp.NewHandler(handler, "http.server", + otelhttp.WithServerName(cfg.OtelServiceName)) } server := &http.Server{ diff --git a/go.mod b/go.mod index 701291bda0..4c0266ff8a 100644 --- a/go.mod +++ b/go.mod @@ -7,15 +7,27 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.29.6 github.com/aws/aws-sdk-go-v2/credentials v1.17.59 github.com/aws/aws-sdk-go-v2/service/s3 v1.77.0 - github.com/getsentry/sentry-go v0.31.1 github.com/golang-jwt/jwt/v5 v5.2.2 github.com/google/uuid v1.6.0 github.com/kelseyhightower/envconfig v1.4.0 github.com/pkg/errors v0.9.1 github.com/segmentio/kafka-go v0.4.48 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/tus/tusd/v2 v2.6.0 github.com/valyala/fasthttp v1.59.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 + go.opentelemetry.io/otel v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 + go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.14.0 + go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 + go.opentelemetry.io/otel/log v0.14.0 + go.opentelemetry.io/otel/sdk v1.38.0 + go.opentelemetry.io/otel/sdk/log v0.14.0 + go.opentelemetry.io/otel/sdk/metric v1.38.0 + go.opentelemetry.io/otel/trace v1.38.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20250215185904-eff6e970281f gopkg.in/vansante/go-ffprobe.v2 v2.2.1 @@ -38,14 +50,27 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect github.com/aws/smithy-go v1.22.3 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.40.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.25.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/grpc v1.75.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 3ca0c0a4f2..7a61312aa8 100644 --- a/go.sum +++ b/go.sum @@ -38,49 +38,56 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 h1:TzeR06UCMUq+KA3bDkujxK1GVGy github.com/aws/aws-sdk-go-v2/service/sts v1.33.14/go.mod h1:dspXf/oYWGWo6DEvj98wpaTeqt5+DMidZD0A9BYTizc= github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/getsentry/sentry-go v0.31.1 h1:ELVc0h7gwyhnXHDouXkhqTFSO5oslsRDk0++eyE0KJ4= -github.com/getsentry/sentry-go v0.31.1/go.mod h1:CYNcMMz73YigoHljQRG+qPF+eMq8gG72XcGN/p71BAY= -github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= -github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/segmentio/kafka-go v0.4.48 h1:9jyu9CWK4W5W+SroCe8EffbrRZVqAOkuaLd/ApID4Vs= github.com/segmentio/kafka-go v0.4.48/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tus/tusd/v2 v2.6.0 h1:Je243QDKnFTvm/WkLH2bd1oQ+7trolrflRWyuI0PdWI= github.com/tus/tusd/v2 v2.6.0/go.mod h1:1Eb1lBoSRBfYJ/mQfFVjyw8ZdNMdBqW17vgQKl3Ah9g= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -96,6 +103,42 @@ github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gi github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 h1:QQqYw3lkrzwVsoEX0w//EhH/TCnpRdEenKBOOEIMjWc= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0/go.mod h1:gSVQcr17jk2ig4jqJ2DX30IdWH251JcNAecvrqTxH1s= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 h1:Oe2z/BCg5q7k4iXC3cqJxKYg0ieRiOqF0cecFYdPTwk= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0/go.mod h1:ZQM5lAJpOsKnYagGg/zV2krVqTtaVdYdDkhMoX6Oalg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.14.0 h1:B/g+qde6Mkzxbry5ZZag0l7QrQBCtVm7lVjaLgmpje8= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.14.0/go.mod h1:mOJK8eMmgW6ocDJn6Bn11CcZ05gi3P8GylBXEkZtbgA= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0 h1:wm/Q0GAAykXv83wzcKzGGqAnnfLFyFe7RslekZuv+VI= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0/go.mod h1:ra3Pa40+oKjvYh+ZD3EdxFZZB0xdMfuileHAm4nNN7w= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 h1:kJxSDN4SgWWTjG/hPp3O7LCGLcHXFlvS2/FFOrwL+SE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0/go.mod h1:mgIOzS7iZeKJdeB8/NYHrJ48fdGc71Llo5bJ1J4DWUE= +go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM= +go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg= +go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= +go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -115,8 +158,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= -golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -128,8 +171,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -142,13 +185,23 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index ca0e4aef22..61cc586a52 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -25,7 +25,6 @@ import ( // Config represents configuration for the huly-stream application. type Config struct { - SentryDsn string `split_words:"true" default:"" desc:"sentry dsn value"` LogLevel string `split_words:"true" default:"debug" desc:"sets log level for the application"` ServerSecret string `split_words:"true" default:"" desc:"server secret required to generate and verify tokens"` PprofEnabled bool `split_words:"true" default:"true" desc:"starts profile server on localhost:6060 if true"` @@ -40,6 +39,14 @@ type Config struct { OutputDir string `split_words:"true" default:"/tmp/transcoing/" desc:"path to the directory with transcoding result."` Timeout time.Duration `default:"5m" desc:"timeout for the upload"` + + // OpenTelemetry configuration + OtelEnabled bool `split_words:"true" default:"true" desc:"enable OpenTelemetry"` + OtelServiceName string `split_words:"true" default:"stream" desc:"service name for OpenTelemetry"` + OtelServiceVersion string `split_words:"true" default:"1.0.0" desc:"service version for OpenTelemetry"` + OtelTracesEnabled bool `split_words:"true" default:"true" desc:"enable OpenTelemetry traces"` + OtelMetricsEnabled bool `split_words:"true" default:"true" desc:"enable OpenTelemetry metrics"` + OtelLogsEnabled bool `split_words:"true" default:"false" desc:"enable OpenTelemetry logs"` } // FromEnv creates new Config from env diff --git a/internal/pkg/mediaconvert/executor.go b/internal/pkg/executor/executor.go similarity index 77% rename from internal/pkg/mediaconvert/executor.go rename to internal/pkg/executor/executor.go index 6385160aa7..e0f2cc90d9 100644 --- a/internal/pkg/mediaconvert/executor.go +++ b/internal/pkg/executor/executor.go @@ -13,7 +13,7 @@ // limitations under the License. // -package mediaconvert +package executor import ( "bytes" @@ -24,30 +24,18 @@ import ( "sync" "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/tracing" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) -// CommandExecutor executes multiple commands in parallel -type CommandExecutor interface { - Execute(commands []*exec.Cmd) error -} +var tracer = otel.Tracer("executor") -type commandExecutor struct { - logger *zap.Logger -} - -var _ CommandExecutor = (*commandExecutor)(nil) - -// NewCommandExecutor creates a new instance of command executor -func NewCommandExecutor(ctx context.Context) CommandExecutor { - return &commandExecutor{ - logger: log.FromContext(ctx), - } -} - -// Execute executes multiple commands in parallel -func (e *commandExecutor) Execute(commands []*exec.Cmd) error { - logger := e.logger +// ExecuteCommands executes multiple commands in parallel +func ExecuteCommands(ctx context.Context, commands []*exec.Cmd) error { + logger := log.FromContext(ctx) errCh := make(chan error, len(commands)) var mu sync.Mutex @@ -58,6 +46,11 @@ func (e *commandExecutor) Execute(commands []*exec.Cmd) error { go func(cmd *exec.Cmd) { defer wg.Done() + _, span := tracer.Start(ctx, "cmd", trace.WithAttributes( + attribute.String("command", cmd.String()), + )) + defer span.End() + var stdoutBuf = &bytes.Buffer{} var stderrBuf = &bytes.Buffer{} @@ -71,6 +64,7 @@ func (e *commandExecutor) Execute(commands []*exec.Cmd) error { logger.Info("run command", zap.String("cmd", cmd.String())) if err := cmd.Run(); err != nil { + tracing.RecordError(span, err) errCh <- err // Lock so only on goroutine can write to stdout/stderr at the same time diff --git a/internal/pkg/mediaconvert/executor_test.go b/internal/pkg/executor/executor_test.go similarity index 89% rename from internal/pkg/mediaconvert/executor_test.go rename to internal/pkg/executor/executor_test.go index 3f30a82cf9..4e8bf3a4d3 100644 --- a/internal/pkg/mediaconvert/executor_test.go +++ b/internal/pkg/executor/executor_test.go @@ -13,7 +13,7 @@ // limitations under the License. // -package mediaconvert_test +package executor_test import ( "context" @@ -21,8 +21,8 @@ import ( "testing" "time" + "github.com/hcengineering/stream/internal/pkg/executor" "github.com/hcengineering/stream/internal/pkg/log" - "github.com/hcengineering/stream/internal/pkg/mediaconvert" "github.com/stretchr/testify/assert" ) @@ -55,9 +55,8 @@ func TestCommandExecutor_Execute_Success(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := context.Background() ctx = log.WithFields(ctx) - executor := mediaconvert.NewCommandExecutor(ctx) - err := executor.Execute(tt.commands) + err := executor.ExecuteCommands(ctx, tt.commands) assert.NoError(t, err) }) } @@ -107,9 +106,8 @@ func TestCommandExecutor_Execute_Error(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := context.Background() ctx = log.WithFields(ctx) - executor := mediaconvert.NewCommandExecutor(ctx) - err := executor.Execute(tt.commands) + err := executor.ExecuteCommands(ctx, tt.commands) if tt.expectedError { assert.Error(t, err) } else { @@ -130,10 +128,9 @@ func TestCommandExecutor_Execute_Parallel(t *testing.T) { ctx := context.Background() ctx = log.WithFields(ctx) - executor := mediaconvert.NewCommandExecutor(ctx) start := time.Now() - err := executor.Execute(commands) + err := executor.ExecuteCommands(ctx, commands) duration := time.Since(start) assert.NoError(t, err) diff --git a/internal/pkg/mediaconvert/scheduler.go b/internal/pkg/mediaconvert/scheduler.go index 8235f44b02..d41998f75d 100644 --- a/internal/pkg/mediaconvert/scheduler.go +++ b/internal/pkg/mediaconvert/scheduler.go @@ -26,15 +26,21 @@ import ( "github.com/google/uuid" "github.com/hcengineering/stream/internal/pkg/config" + "github.com/hcengineering/stream/internal/pkg/executor" "github.com/hcengineering/stream/internal/pkg/log" "github.com/hcengineering/stream/internal/pkg/manifest" "github.com/hcengineering/stream/internal/pkg/storage" "github.com/hcengineering/stream/internal/pkg/token" + "github.com/hcengineering/stream/internal/pkg/tracing" "github.com/hcengineering/stream/internal/pkg/uploader" + "github.com/pkg/errors" + "go.opentelemetry.io/otel" "go.uber.org/zap" "gopkg.in/vansante/go-ffprobe.v2" ) +var tracer = otel.Tracer("mediaconvert") + // Task represents transcoding task type Task struct { ID string @@ -97,14 +103,19 @@ func (p *Scheduler) start() { for range p.cfg.MaxParallelTranscodingCount { go func() { for task := range p.taskCh { - p.processTask(p.ctx, task) + err := tracing.WithSpan(p.ctx, tracer, "transcode", func(ctx context.Context) error { + return p.processTask(ctx, task) + }) + if err != nil { + p.logger.Error("failed to process task", zap.Error(err)) + } } }() } } // TODO: add a factory pattern to process tasks by different media type -func (p *Scheduler) processTask(ctx context.Context, task *Task) { +func (p *Scheduler) processTask(ctx context.Context, task *Task) error { var logger = p.logger.With(zap.String("task-id", task.ID)) logger.Debug("start") @@ -114,7 +125,7 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { var tokenString, err = token.NewToken(p.cfg.ServerSecret, task.Workspace, "stream") if err != nil { logger.Error("can not create token", zap.Error(err)) - return + return errors.Wrapf(err, "can not create token") } logger.Debug("phase 2: preparing fs") @@ -124,7 +135,7 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { err = os.MkdirAll(destinationFolder, os.ModePerm) if err != nil { logger.Error("can not create temporary folder", zap.Error(err)) - return + return errors.Wrapf(err, "can not create temporary folder") } defer func() { @@ -140,35 +151,37 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { if err != nil { logger.Error("can not create storage by url", zap.Error(err), zap.String("url", p.cfg.EndpointURL.String())) _ = os.RemoveAll(destinationFolder) - return + return err } stat, err := remoteStorage.StatFile(ctx, task.Source) if err != nil { logger.Error("can not stat a file", zap.Error(err), zap.String("filepath", task.Source)) _ = os.RemoveAll(destinationFolder) - return + return errors.Wrapf(err, "can not stat a file") } if !IsSupportedMediaType(stat.Type) { logger.Info("unsupported media type", zap.String("type", stat.Type)) _ = os.RemoveAll(destinationFolder) - return + return fmt.Errorf("unsupported media type: %s", stat.Type) } if err = remoteStorage.GetFile(ctx, task.Source, sourceFilePath); err != nil { logger.Error("can not download a file", zap.Error(err), zap.String("filepath", task.Source)) _ = os.RemoveAll(destinationFolder) // TODO: reschedule - return + return errors.Wrapf(err, "can not download a file") } logger.Debug("phase 4: prepare to transcode") - probe, err := ffprobe.ProbeURL(ctx, sourceFilePath) + probe, err := tracing.WithSpanResult(ctx, tracer, "ffprobe", func(ctx context.Context) (*ffprobe.ProbeData, error) { + return ffprobe.ProbeURL(ctx, sourceFilePath) + }) if err != nil { logger.Error("can not get probe for a file", zap.Error(err), zap.String("filepath", sourceFilePath)) _ = os.RemoveAll(destinationFolder) - return + return errors.Wrapf(err, "can not get ffprobe") } audioStream := probe.FirstAudioStream() @@ -180,7 +193,7 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { if videoStream == nil { logger.Error("no video stream found in the file", zap.String("filepath", sourceFilePath)) _ = os.RemoveAll(destinationFolder) - return + return fmt.Errorf("no video stream found") } logger.Debug("video stream found", zap.String("codec", videoStream.CodecName), zap.Int("width", videoStream.Width), zap.Int("height", videoStream.Height)) @@ -221,7 +234,7 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { if err != nil { logger.Error("can not generate hls playlist", zap.String("out", p.cfg.OutputDir), zap.String("uploadID", opts.UploadID)) _ = os.RemoveAll(destinationFolder) - return + return errors.Wrapf(err, "can not generate hls playlist") } go uploader.Start() @@ -239,23 +252,19 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { if cmdErr != nil { logger.Error("can not create a new command", zap.Error(cmdErr), zap.Strings("args", args)) go uploader.Cancel() - return + return errors.Wrapf(cmdErr, "can not create a new command") } cmds = append(cmds, cmd) - if err = cmd.Start(); err != nil { - logger.Error("can not start a command", zap.Error(err), zap.Strings("args", args)) - go uploader.Cancel() - return - } } logger.Debug("phase 7: wait for the result") - for _, cmd := range cmds { - if err = cmd.Wait(); err != nil { - logger.Error("can not wait for command end ", zap.Error(err)) - go uploader.Cancel() - return - } + execErr := tracing.WithSpan(ctx, tracer, "ffmpeg", func(ctx context.Context) error { + return executor.ExecuteCommands(ctx, cmds) + }) + if execErr != nil { + logger.Error("can not wait for command end ", zap.Error(execErr)) + go uploader.Cancel() + return errors.Wrapf(execErr, "can not execute command") } logger.Debug("phase 8: schedule cleanup") @@ -293,6 +302,7 @@ func (p *Scheduler) processTask(ctx context.Context, task *Task) { logger.Error("can not patch the source file", zap.Error(err)) } } + return nil } // IsHLSSupportedVideoCodec checks whether the codec is supported by HLS diff --git a/internal/pkg/mediaconvert/stream.go b/internal/pkg/mediaconvert/stream.go index fbb671c911..d32e05c2a6 100644 --- a/internal/pkg/mediaconvert/stream.go +++ b/internal/pkg/mediaconvert/stream.go @@ -20,9 +20,12 @@ import ( "sync" "github.com/pkg/errors" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "github.com/hcengineering/stream/internal/pkg/sharedpipe" "github.com/hcengineering/stream/internal/pkg/storage" + "github.com/hcengineering/stream/internal/pkg/tracing" "github.com/tus/tusd/v2/pkg/handler" "go.uber.org/zap" ) @@ -46,6 +49,12 @@ var _ handler.LengthDeclarableUpload = (*Stream)(nil) // WriteChunk is called when client sends a chunk of raw data func (w *Stream) WriteChunk(ctx context.Context, _ int64, src io.Reader) (int64, error) { + ctx, span := tracer.Start(ctx, "WriteChunk", trace.WithAttributes( + attribute.String("workspace", w.info.MetaData["workspace"]), + attribute.String("upload_id", w.info.ID), + )) + defer span.End() + w.logger.Debug("Write Chunk start", zap.Int64("offset", w.info.Offset)) data, err := io.ReadAll(src) if err != nil { @@ -93,10 +102,17 @@ func (w *Stream) GetReader(ctx context.Context) (io.ReadCloser, error) { // Terminate calls when upload has failed func (w *Stream) Terminate(ctx context.Context) error { + ctx, span := tracer.Start(ctx, "Terminate", trace.WithAttributes( + attribute.String("workspace", w.info.MetaData["workspace"]), + attribute.String("upload_id", w.info.ID), + )) + defer span.End() + w.logger.Debug("terminate upload") // Close the writer first to signal EOF to all readers if err := w.writer.Close(); err != nil { + tracing.RecordError(span, err) w.logger.Error("failed to close writer", zap.Error(err)) return err } @@ -133,10 +149,17 @@ func (w *Stream) ConcatUploads(ctx context.Context, partialUploads []handler.Upl // FinishUpload calls when upload finished without errors on the client side func (w *Stream) FinishUpload(ctx context.Context) error { + ctx, span := tracer.Start(ctx, "FinishUpload", trace.WithAttributes( + attribute.String("workspace", w.info.MetaData["workspace"]), + attribute.String("upload_id", w.info.ID), + )) + defer span.End() + w.logger.Debug("finish upload") // Close the writer first to signal EOF to all readers if err := w.writer.Close(); err != nil { + tracing.RecordError(span, err) w.logger.Error("failed to close writer", zap.Error(err)) return err } @@ -151,6 +174,7 @@ func (w *Stream) FinishUpload(ctx context.Context) error { defer wg.Done() if err := w.multipart.Complete(ctx); err != nil { w.logger.Error("multipart upload complete failed", zap.Error(err)) + tracing.RecordError(span, err) completeErr = err return } diff --git a/internal/pkg/mediaconvert/transcoder.go b/internal/pkg/mediaconvert/transcoder.go index a3fc1b738d..c0b4f9bc0a 100644 --- a/internal/pkg/mediaconvert/transcoder.go +++ b/internal/pkg/mediaconvert/transcoder.go @@ -24,10 +24,12 @@ import ( "time" "github.com/hcengineering/stream/internal/pkg/config" + "github.com/hcengineering/stream/internal/pkg/executor" "github.com/hcengineering/stream/internal/pkg/log" "github.com/hcengineering/stream/internal/pkg/manifest" "github.com/hcengineering/stream/internal/pkg/storage" "github.com/hcengineering/stream/internal/pkg/token" + "github.com/hcengineering/stream/internal/pkg/tracing" "github.com/hcengineering/stream/internal/pkg/uploader" "github.com/pkg/errors" "go.uber.org/zap" @@ -115,7 +117,9 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er } logger.Debug("phase 4: prepare to transcode") - probe, err := ffprobe.ProbeURL(ctx, sourceFilePath) + probe, err := tracing.WithSpanResult(ctx, tracer, "ffprobe", func(spanCtx context.Context) (*ffprobe.ProbeData, error) { + return ffprobe.ProbeURL(spanCtx, sourceFilePath) + }) if err != nil { logger.Error("can not get ffprobe", zap.Error(err), zap.String("filepath", sourceFilePath)) return nil, errors.Wrapf(err, "can not get ffprobe") @@ -196,8 +200,10 @@ func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, er cmds = append(cmds, cmd) } - executor := NewCommandExecutor(ctx) - if execErr := executor.Execute(cmds); execErr != nil { + execErr := tracing.WithSpan(ctx, tracer, "ffmpeg", func(ctx context.Context) error { + return executor.ExecuteCommands(ctx, cmds) + }) + if execErr != nil { uploader.Cancel() return nil, errors.Wrapf(execErr, "can not execute command") } diff --git a/internal/pkg/queue/worker.go b/internal/pkg/queue/worker.go index a80b4c702a..4fc0802f0b 100644 --- a/internal/pkg/queue/worker.go +++ b/internal/pkg/queue/worker.go @@ -21,11 +21,15 @@ import ( "github.com/hcengineering/stream/internal/pkg/config" "github.com/hcengineering/stream/internal/pkg/log" "github.com/hcengineering/stream/internal/pkg/mediaconvert" + "github.com/hcengineering/stream/internal/pkg/tracing" "github.com/pkg/errors" "github.com/segmentio/kafka-go" + "go.opentelemetry.io/otel" "go.uber.org/zap" ) +var tracer = otel.Tracer("worker") + // Worker is a queue processing worker type Worker struct { logger *zap.Logger @@ -109,7 +113,9 @@ func (w *Worker) processMessage(ctx context.Context, msg kafka.Message, logger * } transcoder := mediaconvert.NewTranscoder(ctx, w.cfg) - res, err := transcoder.Transcode(ctx, &task) + res, err := tracing.WithSpanResult(ctx, tracer, "transcode", func(spanCtx context.Context) (*mediaconvert.TaskResult, error) { + return transcoder.Transcode(spanCtx, &task) + }) if res != nil { result := TranscodeResult{ diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index 9207a88ed9..1101f49f51 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -29,11 +29,17 @@ import ( "time" "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/tracing" "github.com/pkg/errors" "github.com/valyala/fasthttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) +var tracer = otel.Tracer("storage.datalake") + type uploadResult struct { key string error string @@ -84,9 +90,15 @@ func getObjectKeyFromPath(s string) string { } // PutFile uploads file to the datalake -func (d *DatalakeStorage) PutFile(ctx context.Context, fileName string, options PutOptions) error { +func (d *DatalakeStorage) PutFile(ctx context.Context, filename string, options PutOptions) error { + ctx, span := tracer.Start(ctx, "datalake.put_file", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", getObjectKeyFromPath(filename)), + )) + defer span.End() + // #nosec - file, err := os.Open(fileName) + file, err := os.Open(filename) if err != nil { return err } @@ -94,8 +106,8 @@ func (d *DatalakeStorage) PutFile(ctx context.Context, fileName string, options _ = file.Close() }() - var objectKey = getObjectKeyFromPath(fileName) - var logger = d.logger.With(zap.String("upload", d.workspace), zap.String("fileName", fileName)) + var objectKey = getObjectKeyFromPath(filename) + var logger = d.logger.With(zap.String("upload", d.workspace), zap.String("fileName", filename)) logger.Debug("start") @@ -104,16 +116,19 @@ func (d *DatalakeStorage) PutFile(ctx context.Context, fileName string, options part, err := createFormFile(writer, "file", objectKey, getContentType(objectKey)) if err != nil { + tracing.RecordError(span, err) return errors.Wrapf(err, "failed to create form file") } _, err = io.Copy(part, file) if err != nil { + tracing.RecordError(span, err) return errors.Wrapf(err, "failed to copy file data") } err = writer.Close() if err != nil { + tracing.RecordError(span, err) return errors.Wrapf(err, "failed to close multipart writer") } @@ -133,32 +148,39 @@ func (d *DatalakeStorage) PutFile(ctx context.Context, fileName string, options req.SetBody(body.Bytes()) if err := d.client.Do(req, res); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "upload failed", res) return errors.Wrapf(err, "upload failed") } var result []uploadResult if err := json.Unmarshal(res.Body(), &result); err != nil { + tracing.RecordError(span, err) return errors.Wrapf(err, "parse error") } for _, res := range result { if res.error != "" { + tracing.RecordError(span, err) return fmt.Errorf("upload error: %v %v", res.key, res.error) } } - logger.Debug("uploaded") - return nil } // DeleteFile deletes file from the datalake -func (d *DatalakeStorage) DeleteFile(ctx context.Context, fileName string) error { - var logger = d.logger.With(zap.String("delete", d.workspace), zap.String("fileName", fileName)) +func (d *DatalakeStorage) DeleteFile(ctx context.Context, filename string) error { + ctx, span := tracer.Start(ctx, "datalake.delete_file", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", getObjectKeyFromPath(filename)), + )) + defer span.End() + + var logger = d.logger.With(zap.String("delete", d.workspace), zap.String("fileName", filename)) logger.Debug("start") - var objectKey = getObjectKeyFromPath(fileName) + var objectKey = getObjectKeyFromPath(filename) req := fasthttp.AcquireRequest() defer fasthttp.ReleaseRequest(req) @@ -171,22 +193,28 @@ func (d *DatalakeStorage) DeleteFile(ctx context.Context, fileName string) error req.Header.Add("Authorization", "Bearer "+d.token) if err := d.client.Do(req, res); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "delete failed", res) return errors.Wrapf(err, "delete failed") } if err := okResponse(res); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "bad status code", res) return err } - logger.Debug("deleted") - return nil } // PatchMeta patches metadata for the object func (d *DatalakeStorage) PatchMeta(ctx context.Context, filename string, md *Metadata) error { + ctx, span := tracer.Start(ctx, "datalake.patch_meta", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", getObjectKeyFromPath(filename)), + )) + defer span.End() + var logger = d.logger.With(zap.String("patch meta", d.workspace), zap.String("fileName", filename)) logger.Debug("start") defer logger.Debug("finished") @@ -211,11 +239,13 @@ func (d *DatalakeStorage) PatchMeta(ctx context.Context, filename string, md *Me defer fasthttp.ReleaseResponse(resp) if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "request failed", resp) return err } if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "bad status code", resp) return err } @@ -225,6 +255,12 @@ func (d *DatalakeStorage) PatchMeta(ctx context.Context, filename string, md *Me // GetMeta gets metadata related to the object func (d *DatalakeStorage) GetMeta(ctx context.Context, filename string) (*Metadata, error) { + ctx, span := tracer.Start(ctx, "datalake.put_file", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", getObjectKeyFromPath(filename)), + )) + defer span.End() + var logger = d.logger.With(zap.String("get meta", d.workspace), zap.String("fileName", filename)) logger.Debug("start") @@ -240,11 +276,13 @@ func (d *DatalakeStorage) GetMeta(ctx context.Context, filename string) (*Metada defer fasthttp.ReleaseResponse(resp) if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "request failed", resp) return nil, err } if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "bad status code", resp) return nil, err } @@ -257,6 +295,12 @@ func (d *DatalakeStorage) GetMeta(ctx context.Context, filename string) (*Metada // GetFile gets file from the storage func (d *DatalakeStorage) GetFile(ctx context.Context, filename, destination string) error { + ctx, span := tracer.Start(ctx, "datalake.get_file", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", getObjectKeyFromPath(filename)), + )) + defer span.End() + var logger = d.logger.With(zap.String("get", d.workspace), zap.String("fileName", filename), zap.String("destination", destination)) logger.Debug("start") @@ -272,11 +316,13 @@ func (d *DatalakeStorage) GetFile(ctx context.Context, filename, destination str defer fasthttp.ReleaseResponse(resp) if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "request failed", resp) return err } if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "bad status code", resp) return err } @@ -284,6 +330,7 @@ func (d *DatalakeStorage) GetFile(ctx context.Context, filename, destination str // #nosec file, err := os.Create(destination) if err != nil { + tracing.RecordError(span, err) logger.Debug("can't create a file", zap.Error(err)) return err } @@ -291,12 +338,14 @@ func (d *DatalakeStorage) GetFile(ctx context.Context, filename, destination str _ = file.Close() }() if err = resp.BodyWriteTo(file); err != nil { + tracing.RecordError(span, err) logger.Debug("can't write to file", zap.Error(err)) return err } stat, err := os.Stat(destination) if err != nil { + tracing.RecordError(span, err) logger.Error("can't stat the file", zap.Error(err)) return err } @@ -307,6 +356,12 @@ func (d *DatalakeStorage) GetFile(ctx context.Context, filename, destination str // StatFile gets file stat from the storage func (d *DatalakeStorage) StatFile(ctx context.Context, filename string) (*BlobInfo, error) { + ctx, span := tracer.Start(ctx, "datalake.stat_file", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", getObjectKeyFromPath(filename)), + )) + defer span.End() + var logger = d.logger.With(zap.String("head", d.workspace), zap.String("fileName", filename)) logger.Debug("start") @@ -322,11 +377,13 @@ func (d *DatalakeStorage) StatFile(ctx context.Context, filename string) (*BlobI defer fasthttp.ReleaseResponse(resp) if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "request failed", resp) return nil, err } if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "bad status code", resp) return nil, err } @@ -342,6 +399,12 @@ func (d *DatalakeStorage) StatFile(ctx context.Context, filename string) (*BlobI // SetParent updates blob parent reference func (d *DatalakeStorage) SetParent(ctx context.Context, filename, parent string) error { + ctx, span := tracer.Start(ctx, "datalake.set_parent", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", getObjectKeyFromPath(filename)), + )) + defer span.End() + var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("fileName", filename), zap.String("parent", parent)) logger.Debug("start") @@ -366,6 +429,7 @@ func (d *DatalakeStorage) SetParent(ctx context.Context, filename, parent string } if err := json.NewEncoder(req.BodyWriter()).Encode(body); err != nil { + tracing.RecordError(span, err) logger.Debug("can not encode body", zap.Error(err)) return err } @@ -374,11 +438,13 @@ func (d *DatalakeStorage) SetParent(ctx context.Context, filename, parent string defer fasthttp.ReleaseResponse(resp) if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "request failed", resp) return err } if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "bad status code", resp) return err } @@ -388,6 +454,12 @@ func (d *DatalakeStorage) SetParent(ctx context.Context, filename, parent string // MultipartUploadStart creates a new multipart upload func (d *DatalakeStorage) MultipartUploadStart(ctx context.Context, objectName, contentType string) (string, error) { + ctx, span := tracer.Start(ctx, "datalake.multipart_upload_start", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", objectName), + )) + defer span.End() + var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("objectName", objectName)) url := fmt.Sprintf("%v/upload/multipart/%v/%v", d.baseURL, d.workspace, objectName) @@ -402,11 +474,13 @@ func (d *DatalakeStorage) MultipartUploadStart(ctx context.Context, objectName, defer fasthttp.ReleaseResponse(resp) if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "request failed", resp) return "", err } if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "bad status code", resp) return "", err } @@ -421,6 +495,13 @@ func (d *DatalakeStorage) MultipartUploadStart(ctx context.Context, objectName, // MultipartUploadPart uploads a part of a multipart upload func (d *DatalakeStorage) MultipartUploadPart(ctx context.Context, objectName, uploadID string, partNumber int, data []byte) (*MultipartPart, error) { + ctx, span := tracer.Start(ctx, "datalake.multipart_upload_part", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", objectName), + attribute.Int("size", len(data)), + )) + defer span.End() + var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("uploadID", uploadID), zap.Int("partNumber", partNumber)) params := url.Values{} params.Add("uploadId", uploadID) @@ -440,11 +521,13 @@ func (d *DatalakeStorage) MultipartUploadPart(ctx context.Context, objectName, u defer fasthttp.ReleaseResponse(resp) if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "request failed", resp) return nil, err } if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "bad status code", resp) return nil, err } @@ -457,6 +540,12 @@ func (d *DatalakeStorage) MultipartUploadPart(ctx context.Context, objectName, u // MultipartUploadComplete completes a multipart upload func (d *DatalakeStorage) MultipartUploadComplete(ctx context.Context, objectName, uploadID string, parts []MultipartPart) error { + ctx, span := tracer.Start(ctx, "datalake.multipart_upload_complete", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", objectName), + )) + defer span.End() + var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("uploadID", uploadID), zap.String("objectName", objectName)) params := url.Values{} params.Add("uploadId", uploadID) @@ -483,11 +572,13 @@ func (d *DatalakeStorage) MultipartUploadComplete(ctx context.Context, objectNam defer fasthttp.ReleaseResponse(resp) if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "request failed", resp) return err } if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "bad status code", resp) return err } @@ -497,6 +588,12 @@ func (d *DatalakeStorage) MultipartUploadComplete(ctx context.Context, objectNam // MultipartUploadCancel cancels a multipart upload func (d *DatalakeStorage) MultipartUploadCancel(ctx context.Context, objectName, uploadID string) error { + ctx, span := tracer.Start(ctx, "datalake.multipart_upload_cancel", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", objectName), + )) + defer span.End() + var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("uploadID", uploadID)) params := url.Values{} params.Add("uploadId", uploadID) @@ -512,11 +609,13 @@ func (d *DatalakeStorage) MultipartUploadCancel(ctx context.Context, objectName, defer fasthttp.ReleaseResponse(resp) if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "request failed", resp) return err } if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) logRequestError(logger, err, "bad status code", resp) return err } diff --git a/internal/pkg/tracing/tracing.go b/internal/pkg/tracing/tracing.go new file mode 100644 index 0000000000..e9afbc55dd --- /dev/null +++ b/internal/pkg/tracing/tracing.go @@ -0,0 +1,53 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package tracing + +import ( + "context" + + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +// RecordError records an error into the provided span and sets its status to Error. +func RecordError(span trace.Span, err error) { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) +} + +// WithSpan wraps a function execution within a span. +func WithSpan(ctx context.Context, tracer trace.Tracer, name string, fn func(ctx context.Context) error) error { + ctx, span := tracer.Start(ctx, name) + defer span.End() + + err := fn(ctx) + if err != nil { + RecordError(span, err) + } + + return err +} + +// WithSpanResult wraps a function returning result execution within a span. +func WithSpanResult[Result any](ctx context.Context, tracer trace.Tracer, name string, fn func(ctx context.Context) (Result, error)) (Result, error) { + ctx, span := tracer.Start(ctx, name) + defer span.End() + + result, err := fn(ctx) + if err != nil { + RecordError(span, err) + } + + return result, err +} diff --git a/internal/pkg/uploader/uploader.go b/internal/pkg/uploader/uploader.go index 2bdee38c76..78df9f3d2b 100644 --- a/internal/pkg/uploader/uploader.go +++ b/internal/pkg/uploader/uploader.go @@ -28,10 +28,13 @@ import ( "github.com/hcengineering/stream/internal/pkg/log" "github.com/hcengineering/stream/internal/pkg/storage" "github.com/pkg/errors" + "go.opentelemetry.io/otel" "go.uber.org/zap" "k8s.io/utils/inotify" ) +var tracer = otel.Tracer("uploader") + // See at https://man7.org/linux/man-pages/man7/inotify.7.html const inotifyCloseWrite uint32 = 0x8 // IN_CLOSE_WRITE const inotifyMovedFrom uint32 = 0x40 // IN_MOVED_FROM @@ -93,7 +96,7 @@ func New(ctx context.Context, s storage.Storage, opts Options) Uploader { res.logger.Sugar().Debugf("uploader config is %v", opts) - res.uploadCtx, res.uploadCancel = context.WithCancel(context.Background()) + res.uploadCtx, res.uploadCancel = context.WithCancel(ctx) err := os.MkdirAll(opts.Dir, os.ModePerm) if err != nil { @@ -105,15 +108,18 @@ func New(ctx context.Context, s storage.Storage, opts Options) Uploader { func (u *uploaderImpl) Stop() { u.logger.Info("stopping upload") - u.stop(false) + u.stop(u.uploadCtx, false) } func (u *uploaderImpl) Cancel() { u.logger.Info("canceling upload") - u.stop(true) + u.stop(u.uploadCtx, true) } -func (u *uploaderImpl) scanFiles() { +func (u *uploaderImpl) scanFiles(ctx context.Context) { + _, span := tracer.Start(ctx, "scanFiles") + defer span.End() + logger := u.logger.With(zap.String("dir", u.options.Dir)) logger.Info("scan files") @@ -148,13 +154,16 @@ func (u *uploaderImpl) scanFiles() { logger.Info("scan complete", zap.Int("count", count)) } -func (u *uploaderImpl) stop(rollback bool) { +func (u *uploaderImpl) stop(ctx context.Context, rollback bool) { + ctx, span := tracer.Start(ctx, "stop") + defer span.End() + // Stop watching for new files close(u.watcherStopCh) <-u.watcherDoneCh // Scan remaining files in the directory - u.scanFiles() + u.scanFiles(ctx) // Close filesCh so no new files added close(u.filesCh) @@ -165,7 +174,7 @@ func (u *uploaderImpl) stop(rollback bool) { // Perform rollback if rollback { - u.uploadRollback() + u.uploadRollback(ctx) } u.uploadCancel() @@ -188,7 +197,10 @@ func (u *uploaderImpl) stop(rollback bool) { u.logger.Debug("stopped", zap.Bool("rollback", rollback)) } -func (u *uploaderImpl) uploadRollback() { +func (u *uploaderImpl) uploadRollback(ctx context.Context) { + _, span := tracer.Start(ctx, "uploadRollback") + defer span.End() + u.logger.Debug("starting rollback...") // Create a separate worker pool for rollback @@ -226,7 +238,7 @@ func (u *uploaderImpl) Start() { <-watcherReady - u.scanFiles() + u.scanFiles(u.uploadCtx) } func (u *uploaderImpl) startWorkers() { From b7b6930b0992ae48f08d37745d5e6e5e26517337 Mon Sep 17 00:00:00 2001 From: Alexander Onnikov Date: Fri, 24 Oct 2025 11:36:34 +0700 Subject: [PATCH 52/52] fix build (#26) Signed-off-by: Alexander Onnikov --- .github/workflows/main.yaml | 6 +- .golangci.yaml | 254 ++++++++++++----------- Dockerfile | 14 +- cmd/stream/main.go | 5 +- cmd/stream/otel.go | 179 ++++++++++++++++ internal/pkg/executor/executor.go | 1 + internal/pkg/manifest/hls.go | 2 +- internal/pkg/mediaconvert/coordinator.go | 2 +- internal/pkg/mediaconvert/stream.go | 2 +- internal/pkg/storage/datalake.go | 28 ++- internal/pkg/tracing/tracing.go | 1 + 11 files changed, 344 insertions(+), 150 deletions(-) create mode 100644 cmd/stream/otel.go diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 35e5f78798..0bc40be1df 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -63,9 +63,9 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: 1.24.1 + go-version: 1.24.4 - name: golangci-lint - uses: golangci/golangci-lint-action@v4 + uses: golangci/golangci-lint-action@v8 with: - version: v1.64.7 + version: v2.5.0 args: --timeout 3m --verbose diff --git a/.golangci.yaml b/.golangci.yaml index 2972e64e43..91947ae785 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,130 +1,11 @@ --- +version: "2" run: go: "1.23" timeout: 2m issues-exit-code: 1 tests: true -linters-settings: - goheader: - template: |- - Copyright © {{ mod-year-range }} Hardcore Engineering Inc. - - Licensed under the Eclipse Public License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. You may - obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - - See the License for the specific language governing permissions and - limitations under the License. - errcheck: - check-type-assertions: false - check-blank: false - govet: - enable: - - shadow - settings: - printf: - funcs: - - (github.com/sirupsen/logrus.FieldLogger).Infof - - (github.com/sirupsen/logrus.FieldLogger).Warnf - - (github.com/sirupsen/logrus.FieldLogger).Errorf - - (github.com/sirupsen/logrus.FieldLogger).Fatalf - revive: - confidence: 0.8 - rules: - - name: exported - - name: blank-imports - - name: context-as-argument - - name: context-keys-type - - name: dot-imports - - name: error-return - - name: error-strings - - name: error-naming - - name: exported - - name: increment-decrement - - name: package-comments - - name: range - - name: receiver-naming - - name: time-naming - - name: unexported-return - - name: indent-error-flow - - name: errorf - - name: superfluous-else - - name: unreachable-code - goimports: - local-prefixes: github.com/networkservicemesh/sdk - gocyclo: - min-complexity: 30 - dupl: - threshold: 500 - funlen: - lines: 240 - statements: 120 - goconst: - min-len: 2 - min-occurrences: 3 - depguard: - rules: - main: - deny: - - pkg: "errors" - desc: "Please use \"github.com/pkg/errors\" instead of \"errors\" in go imports" - misspell: - locale: US - unparam: - check-exported: false - nakedret: - max-func-lines: 30 - prealloc: - simple: true - range-loops: true - for-loops: false - gosec: - excludes: - - G115 - - G204 - - G301 - - G302 - - G306 - gocritic: - enabled-checks: - - appendCombine - - boolExprSimplify - - builtinShadow - - commentedOutCode - - commentedOutImport - - docStub - - dupImport - - emptyFallthrough - - emptyStringTest - - equalFold - - evalOrder - - hexLiteral - - importShadow - - indexAlloc - - initClause - - methodExprCall - - nestingReduce - - nilValReturn - - octalLiteral - - paramTypeCombine - - rangeExprCopy - - rangeValCopy - - regexpPattern - - sloppyReassign - - stringXbytes - - typeAssertChain - - typeUnparen - - unlabelStmt - - unnamedResult - - unnecessaryBlock - - weakCond - - yodaStyleExpr linters: - disable-all: true enable: - goheader - bodyclose @@ -138,23 +19,144 @@ linters: - goconst - gocritic - gocyclo - - gofmt - - goimports - revive - gosec - - gosimple - govet - ineffassign - misspell - nakedret - copyloopvar - staticcheck - - stylecheck - - typecheck - unconvert - unparam - whitespace + settings: + goheader: + template: |- + Copyright © {{ mod-year-range }} Hardcore Engineering Inc. + + Licensed under the Eclipse Public License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. You may + obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + See the License for the specific language governing permissions and + limitations under the License. + errcheck: + check-type-assertions: false + check-blank: false + govet: + enable: + - atomic + - bools + - defers + - unsafeptr + - waitgroup + disable: + - shadow + settings: + printf: + funcs: + - (github.com/sirupsen/logrus.FieldLogger).Infof + - (github.com/sirupsen/logrus.FieldLogger).Warnf + - (github.com/sirupsen/logrus.FieldLogger).Errorf + - (github.com/sirupsen/logrus.FieldLogger).Fatalf + revive: + confidence: 0.8 + rules: + - name: exported + - name: blank-imports + - name: context-as-argument + - name: context-keys-type + - name: dot-imports + - name: error-return + - name: error-strings + - name: error-naming + - name: exported + - name: increment-decrement + - name: package-comments + - name: range + - name: receiver-naming + - name: time-naming + - name: unexported-return + - name: indent-error-flow + - name: errorf + - name: superfluous-else + - name: unreachable-code + gocyclo: + min-complexity: 30 + dupl: + threshold: 500 + funlen: + lines: 240 + statements: 120 + goconst: + min-len: 2 + min-occurrences: 3 + depguard: + rules: + main: + deny: + - pkg: "errors" + desc: "Please use \"github.com/pkg/errors\" instead of \"errors\" in go imports" + misspell: + locale: US + unparam: + check-exported: false + nakedret: + max-func-lines: 30 + prealloc: + simple: true + range-loops: true + for-loops: false + gosec: + excludes: + - G115 + - G204 + - G301 + - G302 + - G306 + gocritic: + enabled-checks: + - appendCombine + - boolExprSimplify + - builtinShadow + - commentedOutCode + - commentedOutImport + - docStub + - dupImport + - emptyFallthrough + - emptyStringTest + - equalFold + - evalOrder + - hexLiteral + # - importShadow + - indexAlloc + - initClause + - methodExprCall + - nestingReduce + - nilValReturn + - octalLiteral + - paramTypeCombine + - rangeExprCopy + - rangeValCopy + - regexpPattern + - sloppyReassign + - stringXbytes + - typeAssertChain + - typeUnparen + - unlabelStmt + - unnamedResult + - unnecessaryBlock + - weakCond + - yodaStyleExpr +formatters: + enable: + - gofmt + - goimports issues: - exclude-use-default: false max-issues-per-linter: 0 max-same-issues: 0 diff --git a/Dockerfile b/Dockerfile index 1e7a5cd10b..297954361b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,14 +11,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.24.4 AS builder +FROM golang:1.24.4 AS base + +WORKDIR /app + ENV GO111MODULE=on ENV CGO_ENABLED=0 ENV GOBIN=/bin +ENV PATH=$PATH:$GOBIN ARG BUILDARCH=amd64 COPY . ./ +FROM base AS linter + +RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.5.0 + +RUN golangci-lint run --verbose + +FROM base AS builder + RUN set -xe && GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /go/bin/stream ./cmd/stream FROM alpine diff --git a/cmd/stream/main.go b/cmd/stream/main.go index 3b00fd7d0c..b01c6b9609 100644 --- a/cmd/stream/main.go +++ b/cmd/stream/main.go @@ -16,7 +16,6 @@ package main import ( "context" - "errors" "net/http" "sync" @@ -59,7 +58,9 @@ func main() { panic(err.Error()) } defer func() { - err = errors.Join(err, otelShutdown(context.Background())) + if err := otelShutdown(context.Background()); err != nil { + logger.Error("failed to close queue consumer", zap.Error(err)) + } }() var recordingHandler = recording.NewHandler(ctx, cfg) diff --git a/cmd/stream/otel.go b/cmd/stream/otel.go new file mode 100644 index 0000000000..1f16e0cbae --- /dev/null +++ b/cmd/stream/otel.go @@ -0,0 +1,179 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "time" + + "github.com/hcengineering/stream/internal/pkg/config" + "github.com/pkg/errors" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" + "go.opentelemetry.io/otel/exporters/stdout/stdoutmetric" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/log/global" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/log" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.37.0" +) + +// setupOTelSDK bootstraps the OpenTelemetry pipeline. +// If it does not return an error, make sure to call shutdown for proper cleanup. +func setupOTelSDK(ctx context.Context, cfg *config.Config) (func(context.Context) error, error) { + if !cfg.OtelEnabled { + return func(context.Context) error { return nil }, nil + } + + var shutdownFuncs []func(context.Context) error + + shutdown := func(ctx context.Context) error { + var shutdownErr error + for _, fn := range shutdownFuncs { + if err := fn(ctx); err != nil { + if shutdownErr == nil { + shutdownErr = err + } else { + shutdownErr = errors.Wrap(err, shutdownErr.Error()) + } + } + } + shutdownFuncs = nil + return shutdownErr + } + + res, err := newResource(cfg) + if err != nil { + return shutdown, err + } + + prop := newPropagator() + otel.SetTextMapPropagator(prop) + + if cfg.OtelTracesEnabled { + tracerProvider, tracerErr := newTracerProvider(ctx, res) + if tracerErr != nil { + return shutdown, tracerErr + } + shutdownFuncs = append(shutdownFuncs, tracerProvider.Shutdown) + otel.SetTracerProvider(tracerProvider) + } + + if cfg.OtelMetricsEnabled { + meterProvider, meterErr := newMeterProvider(ctx, res) + if meterErr != nil { + return shutdown, meterErr + } + shutdownFuncs = append(shutdownFuncs, meterProvider.Shutdown) + otel.SetMeterProvider(meterProvider) + } + + if cfg.OtelLogsEnabled { + loggerProvider, loggerErr := newLoggerProvider(ctx, res) + if loggerErr != nil { + return shutdown, loggerErr + } + shutdownFuncs = append(shutdownFuncs, loggerProvider.Shutdown) + global.SetLoggerProvider(loggerProvider) + } + + return shutdown, err +} + +func newResource(cfg *config.Config) (*resource.Resource, error) { + defaultRes := resource.Default() + + customRes := resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName(cfg.OtelServiceName), + semconv.ServiceVersion(cfg.OtelServiceVersion), + ) + + return resource.Merge(defaultRes, customRes) +} + +func newPropagator() propagation.TextMapPropagator { + return propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + ) +} + +func newTracerProvider(ctx context.Context, res *resource.Resource) (*trace.TracerProvider, error) { + var exporter trace.SpanExporter + var err error + + exporter, err = otlptracehttp.New(ctx) + if err != nil { + exporter, err = stdouttrace.New(stdouttrace.WithPrettyPrint()) + if err != nil { + return nil, err + } + } + + tracerProvider := trace.NewTracerProvider( + trace.WithResource(res), + trace.WithBatcher(exporter, + trace.WithBatchTimeout(5*time.Second), + ), + ) + return tracerProvider, nil +} + +func newMeterProvider(ctx context.Context, res *resource.Resource) (*metric.MeterProvider, error) { + var exporter metric.Exporter + var err error + + exporter, err = otlpmetrichttp.New(ctx) + if err != nil { + exporter, err = stdoutmetric.New() + if err != nil { + return nil, err + } + } + + meterProvider := metric.NewMeterProvider( + metric.WithResource(res), + metric.WithReader(metric.NewPeriodicReader(exporter, + metric.WithInterval(60*time.Second))), + ) + + return meterProvider, nil +} + +func newLoggerProvider(ctx context.Context, res *resource.Resource) (*log.LoggerProvider, error) { + var exporter log.Exporter + var err error + + exporter, err = otlploghttp.New(ctx) + if err != nil { + exporter, err = stdoutlog.New() + if err != nil { + return nil, err + } + } + + loggerProvider := log.NewLoggerProvider( + log.WithResource(res), + log.WithProcessor(log.NewBatchProcessor(exporter)), + ) + + return loggerProvider, nil +} diff --git a/internal/pkg/executor/executor.go b/internal/pkg/executor/executor.go index e0f2cc90d9..03050dfc9c 100644 --- a/internal/pkg/executor/executor.go +++ b/internal/pkg/executor/executor.go @@ -13,6 +13,7 @@ // limitations under the License. // +// Package executor provides command execution utilities. package executor import ( diff --git a/internal/pkg/manifest/hls.go b/internal/pkg/manifest/hls.go index cc263b5a16..9e8f27248d 100644 --- a/internal/pkg/manifest/hls.go +++ b/internal/pkg/manifest/hls.go @@ -43,7 +43,7 @@ func GenerateHLSPlaylist(profiles []profile.VideoProfile, outputPath, uploadID s var bandwidth = profile.Bandwidth var resolution = fmt.Sprintf("%vx%v", profile.Width, profile.Height) - _, err = file.WriteString(fmt.Sprintf("#EXT-X-STREAM-INF:BANDWIDTH=%d,RESOLUTION=%v\n", bandwidth, resolution)) + _, err = fmt.Fprintf(file, "#EXT-X-STREAM-INF:BANDWIDTH=%d,RESOLUTION=%v\n", bandwidth, resolution) if err != nil { return err } diff --git a/internal/pkg/mediaconvert/coordinator.go b/internal/pkg/mediaconvert/coordinator.go index bd93d8a61f..48c20b78f2 100644 --- a/internal/pkg/mediaconvert/coordinator.go +++ b/internal/pkg/mediaconvert/coordinator.go @@ -138,7 +138,7 @@ func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo } s.streams.Store(stream.info.ID, stream) - if err := stream.start(s.mainContext); err != nil { + if err := stream.start(); err != nil { return nil, err } diff --git a/internal/pkg/mediaconvert/stream.go b/internal/pkg/mediaconvert/stream.go index d32e05c2a6..a4201532ad 100644 --- a/internal/pkg/mediaconvert/stream.go +++ b/internal/pkg/mediaconvert/stream.go @@ -195,7 +195,7 @@ func (s *StreamCoordinator) AsConcatableUpload(upload handler.Upload) handler.Co return upload.(*Stream) } -func (w *Stream) start(ctx context.Context) error { +func (w *Stream) start() error { defer w.logger.Debug("start done") w.reader = w.writer.Transpile() return nil diff --git a/internal/pkg/storage/datalake.go b/internal/pkg/storage/datalake.go index 1101f49f51..b6e47d3de7 100644 --- a/internal/pkg/storage/datalake.go +++ b/internal/pkg/storage/datalake.go @@ -91,7 +91,7 @@ func getObjectKeyFromPath(s string) string { // PutFile uploads file to the datalake func (d *DatalakeStorage) PutFile(ctx context.Context, filename string, options PutOptions) error { - ctx, span := tracer.Start(ctx, "datalake.put_file", trace.WithAttributes( + _, span := tracer.Start(ctx, "datalake.put_file", trace.WithAttributes( attribute.String("workspace", d.workspace), attribute.String("object_key", getObjectKeyFromPath(filename)), )) @@ -120,14 +120,12 @@ func (d *DatalakeStorage) PutFile(ctx context.Context, filename string, options return errors.Wrapf(err, "failed to create form file") } - _, err = io.Copy(part, file) - if err != nil { + if _, err := io.Copy(part, file); err != nil { tracing.RecordError(span, err) return errors.Wrapf(err, "failed to copy file data") } - err = writer.Close() - if err != nil { + if err := writer.Close(); err != nil { tracing.RecordError(span, err) return errors.Wrapf(err, "failed to close multipart writer") } @@ -171,7 +169,7 @@ func (d *DatalakeStorage) PutFile(ctx context.Context, filename string, options // DeleteFile deletes file from the datalake func (d *DatalakeStorage) DeleteFile(ctx context.Context, filename string) error { - ctx, span := tracer.Start(ctx, "datalake.delete_file", trace.WithAttributes( + _, span := tracer.Start(ctx, "datalake.delete_file", trace.WithAttributes( attribute.String("workspace", d.workspace), attribute.String("object_key", getObjectKeyFromPath(filename)), )) @@ -209,7 +207,7 @@ func (d *DatalakeStorage) DeleteFile(ctx context.Context, filename string) error // PatchMeta patches metadata for the object func (d *DatalakeStorage) PatchMeta(ctx context.Context, filename string, md *Metadata) error { - ctx, span := tracer.Start(ctx, "datalake.patch_meta", trace.WithAttributes( + _, span := tracer.Start(ctx, "datalake.patch_meta", trace.WithAttributes( attribute.String("workspace", d.workspace), attribute.String("object_key", getObjectKeyFromPath(filename)), )) @@ -255,7 +253,7 @@ func (d *DatalakeStorage) PatchMeta(ctx context.Context, filename string, md *Me // GetMeta gets metadata related to the object func (d *DatalakeStorage) GetMeta(ctx context.Context, filename string) (*Metadata, error) { - ctx, span := tracer.Start(ctx, "datalake.put_file", trace.WithAttributes( + _, span := tracer.Start(ctx, "datalake.put_file", trace.WithAttributes( attribute.String("workspace", d.workspace), attribute.String("object_key", getObjectKeyFromPath(filename)), )) @@ -295,7 +293,7 @@ func (d *DatalakeStorage) GetMeta(ctx context.Context, filename string) (*Metada // GetFile gets file from the storage func (d *DatalakeStorage) GetFile(ctx context.Context, filename, destination string) error { - ctx, span := tracer.Start(ctx, "datalake.get_file", trace.WithAttributes( + _, span := tracer.Start(ctx, "datalake.get_file", trace.WithAttributes( attribute.String("workspace", d.workspace), attribute.String("object_key", getObjectKeyFromPath(filename)), )) @@ -356,7 +354,7 @@ func (d *DatalakeStorage) GetFile(ctx context.Context, filename, destination str // StatFile gets file stat from the storage func (d *DatalakeStorage) StatFile(ctx context.Context, filename string) (*BlobInfo, error) { - ctx, span := tracer.Start(ctx, "datalake.stat_file", trace.WithAttributes( + _, span := tracer.Start(ctx, "datalake.stat_file", trace.WithAttributes( attribute.String("workspace", d.workspace), attribute.String("object_key", getObjectKeyFromPath(filename)), )) @@ -399,7 +397,7 @@ func (d *DatalakeStorage) StatFile(ctx context.Context, filename string) (*BlobI // SetParent updates blob parent reference func (d *DatalakeStorage) SetParent(ctx context.Context, filename, parent string) error { - ctx, span := tracer.Start(ctx, "datalake.set_parent", trace.WithAttributes( + _, span := tracer.Start(ctx, "datalake.set_parent", trace.WithAttributes( attribute.String("workspace", d.workspace), attribute.String("object_key", getObjectKeyFromPath(filename)), )) @@ -454,7 +452,7 @@ func (d *DatalakeStorage) SetParent(ctx context.Context, filename, parent string // MultipartUploadStart creates a new multipart upload func (d *DatalakeStorage) MultipartUploadStart(ctx context.Context, objectName, contentType string) (string, error) { - ctx, span := tracer.Start(ctx, "datalake.multipart_upload_start", trace.WithAttributes( + _, span := tracer.Start(ctx, "datalake.multipart_upload_start", trace.WithAttributes( attribute.String("workspace", d.workspace), attribute.String("object_key", objectName), )) @@ -495,7 +493,7 @@ func (d *DatalakeStorage) MultipartUploadStart(ctx context.Context, objectName, // MultipartUploadPart uploads a part of a multipart upload func (d *DatalakeStorage) MultipartUploadPart(ctx context.Context, objectName, uploadID string, partNumber int, data []byte) (*MultipartPart, error) { - ctx, span := tracer.Start(ctx, "datalake.multipart_upload_part", trace.WithAttributes( + _, span := tracer.Start(ctx, "datalake.multipart_upload_part", trace.WithAttributes( attribute.String("workspace", d.workspace), attribute.String("object_key", objectName), attribute.Int("size", len(data)), @@ -540,7 +538,7 @@ func (d *DatalakeStorage) MultipartUploadPart(ctx context.Context, objectName, u // MultipartUploadComplete completes a multipart upload func (d *DatalakeStorage) MultipartUploadComplete(ctx context.Context, objectName, uploadID string, parts []MultipartPart) error { - ctx, span := tracer.Start(ctx, "datalake.multipart_upload_complete", trace.WithAttributes( + _, span := tracer.Start(ctx, "datalake.multipart_upload_complete", trace.WithAttributes( attribute.String("workspace", d.workspace), attribute.String("object_key", objectName), )) @@ -588,7 +586,7 @@ func (d *DatalakeStorage) MultipartUploadComplete(ctx context.Context, objectNam // MultipartUploadCancel cancels a multipart upload func (d *DatalakeStorage) MultipartUploadCancel(ctx context.Context, objectName, uploadID string) error { - ctx, span := tracer.Start(ctx, "datalake.multipart_upload_cancel", trace.WithAttributes( + _, span := tracer.Start(ctx, "datalake.multipart_upload_cancel", trace.WithAttributes( attribute.String("workspace", d.workspace), attribute.String("object_key", objectName), )) diff --git a/internal/pkg/tracing/tracing.go b/internal/pkg/tracing/tracing.go index e9afbc55dd..fee8f4a794 100644 --- a/internal/pkg/tracing/tracing.go +++ b/internal/pkg/tracing/tracing.go @@ -11,6 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package tracing provides utilities for OpenTelemetry tracing. package tracing import (