diff --git a/foundations/stream/.DS_Store b/foundations/stream/.DS_Store new file mode 100644 index 0000000000..0923990b38 Binary files /dev/null and b/foundations/stream/.DS_Store differ diff --git a/foundations/stream/.github/workflows/docker-push.yaml b/foundations/stream/.github/workflows/docker-push.yaml new file mode 100644 index 0000000000..eb2621cfd4 --- /dev/null +++ b/foundations/stream/.github/workflows/docker-push.yaml @@ -0,0 +1,44 @@ +--- +name: Docker push +on: + workflow_dispatch: + inputs: + version: + description: 'Version tag for the image' + required: false + default: 'latest' + type: string + +jobs: + push: + runs-on: ubuntu-latest + steps: + - name: "Checkout" + uses: actions/checkout@v4 + + - name: "Set up Docker Buildx" + uses: docker/setup-buildx-action@v1 + + - name: "Login to GitHub Container Registry" + uses: docker/login-action@v1 + with: + username: hardcoreeng + password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + + - name: Docker meta + id: metaci + uses: docker/metadata-action@v3 + with: + images: hardcoreeng/stream:${{ inputs.version }} + tags: | + type=ref,event=pr + type=sha,prefix= + + - name: "Build and push" + uses: docker/build-push-action@v2 + with: + file: Dockerfile + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: hardcoreeng/service_stream:${{ inputs.version }} diff --git a/foundations/stream/.github/workflows/main.yaml b/foundations/stream/.github/workflows/main.yaml new file mode 100644 index 0000000000..0bc40be1df --- /dev/null +++ b/foundations/stream/.github/workflows/main.yaml @@ -0,0 +1,71 @@ +--- +name: ci + +on: + push: + branches: + - main + - develop + pull_request: + +concurrency: + group: 'main' + cancel-in-progress: true + +jobs: + yamllint: + name: yamllint + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: yaml-lint + uses: ibiqlik/action-yamllint@v1 + with: + config_file: .github/yamllint.yaml + strict: true + build-and-test: + strategy: + matrix: + os: + - ubuntu + runs-on: ${{ matrix.os }}-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: 1.24.1 + - name: Build + run: go build -race ./... + - name: Test + run: go test -race ./... + checkgomod: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: 1.24.1 + - run: go mod tidy + - name: Check for changes in go.mod or go.sum + run: | + git diff --name-only --exit-code go.mod || ( echo "Run go mod tidy" && false ) + golangci-lint: + name: golangci-lint + runs-on: ubuntu-latest + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: 1.24.4 + - name: golangci-lint + uses: golangci/golangci-lint-action@v8 + with: + version: v2.5.0 + args: --timeout 3m --verbose diff --git a/foundations/stream/.github/yamllint.yaml b/foundations/stream/.github/yamllint.yaml new file mode 100644 index 0000000000..7c94f8d8fe --- /dev/null +++ b/foundations/stream/.github/yamllint.yaml @@ -0,0 +1,12 @@ +--- +extends: default + +yaml-files: + - '*.yaml' + - '*.yml' + +rules: + truthy: disable + line-length: disable + comments: + min-spaces-from-content: 1 diff --git a/foundations/stream/.gitignore b/foundations/stream/.gitignore new file mode 100644 index 0000000000..643a5c9e89 --- /dev/null +++ b/foundations/stream/.gitignore @@ -0,0 +1,8 @@ +# IntelliJ +.idea + +# VSCode +.vscode + +# Binary +stream diff --git a/foundations/stream/.golangci.yaml b/foundations/stream/.golangci.yaml new file mode 100644 index 0000000000..91947ae785 --- /dev/null +++ b/foundations/stream/.golangci.yaml @@ -0,0 +1,162 @@ +--- +version: "2" +run: + go: "1.23" + timeout: 2m + issues-exit-code: 1 + tests: true +linters: + enable: + - goheader + - bodyclose + - unused + - depguard + - dogsled + - dupl + - errcheck + - funlen + - gochecknoinits + - goconst + - gocritic + - gocyclo + - revive + - gosec + - govet + - ineffassign + - misspell + - nakedret + - copyloopvar + - staticcheck + - unconvert + - unparam + - whitespace + settings: + goheader: + template: |- + Copyright © {{ mod-year-range }} Hardcore Engineering Inc. + + Licensed under the Eclipse Public License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. You may + obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + See the License for the specific language governing permissions and + limitations under the License. + errcheck: + check-type-assertions: false + check-blank: false + govet: + enable: + - atomic + - bools + - defers + - unsafeptr + - waitgroup + disable: + - shadow + settings: + printf: + funcs: + - (github.com/sirupsen/logrus.FieldLogger).Infof + - (github.com/sirupsen/logrus.FieldLogger).Warnf + - (github.com/sirupsen/logrus.FieldLogger).Errorf + - (github.com/sirupsen/logrus.FieldLogger).Fatalf + revive: + confidence: 0.8 + rules: + - name: exported + - name: blank-imports + - name: context-as-argument + - name: context-keys-type + - name: dot-imports + - name: error-return + - name: error-strings + - name: error-naming + - name: exported + - name: increment-decrement + - name: package-comments + - name: range + - name: receiver-naming + - name: time-naming + - name: unexported-return + - name: indent-error-flow + - name: errorf + - name: superfluous-else + - name: unreachable-code + gocyclo: + min-complexity: 30 + dupl: + threshold: 500 + funlen: + lines: 240 + statements: 120 + goconst: + min-len: 2 + min-occurrences: 3 + depguard: + rules: + main: + deny: + - pkg: "errors" + desc: "Please use \"github.com/pkg/errors\" instead of \"errors\" in go imports" + misspell: + locale: US + unparam: + check-exported: false + nakedret: + max-func-lines: 30 + prealloc: + simple: true + range-loops: true + for-loops: false + gosec: + excludes: + - G115 + - G204 + - G301 + - G302 + - G306 + gocritic: + enabled-checks: + - appendCombine + - boolExprSimplify + - builtinShadow + - commentedOutCode + - commentedOutImport + - docStub + - dupImport + - emptyFallthrough + - emptyStringTest + - equalFold + - evalOrder + - hexLiteral + # - importShadow + - indexAlloc + - initClause + - methodExprCall + - nestingReduce + - nilValReturn + - octalLiteral + - paramTypeCombine + - rangeExprCopy + - rangeValCopy + - regexpPattern + - sloppyReassign + - stringXbytes + - typeAssertChain + - typeUnparen + - unlabelStmt + - unnamedResult + - unnecessaryBlock + - weakCond + - yodaStyleExpr +formatters: + enable: + - gofmt + - goimports +issues: + max-issues-per-linter: 0 + max-same-issues: 0 diff --git a/foundations/stream/Dockerfile b/foundations/stream/Dockerfile new file mode 100644 index 0000000000..297954361b --- /dev/null +++ b/foundations/stream/Dockerfile @@ -0,0 +1,48 @@ +# Copyright © 2025 Hardcore Engineering Inc. +# +# Licensed under the Eclipse Public License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. You may +# obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM golang:1.24.4 AS base + +WORKDIR /app + +ENV GO111MODULE=on +ENV CGO_ENABLED=0 +ENV GOBIN=/bin +ENV PATH=$PATH:$GOBIN +ARG BUILDARCH=amd64 + +COPY . ./ + +FROM base AS linter + +RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.5.0 + +RUN golangci-lint run --verbose + +FROM base AS builder + +RUN set -xe && GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /go/bin/stream ./cmd/stream + +FROM alpine + +RUN set -xe && apk add --no-cache ffmpeg +RUN apk add --no-cache ca-certificates jq bash \ + && addgroup -g 1000 stream \ + && adduser -u 1000 -G stream -s /bin/sh -D stream \ + && chown stream:stream /. +COPY --from=builder /go/bin/stream /stream + +EXPOSE 1080 +USER stream + +ENTRYPOINT ["/stream"] \ No newline at end of file diff --git a/foundations/stream/LICENSE b/foundations/stream/LICENSE new file mode 100644 index 0000000000..e48e096345 --- /dev/null +++ b/foundations/stream/LICENSE @@ -0,0 +1,277 @@ +Eclipse Public License - v 2.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + + a) in the case of the initial Contributor, the initial content + Distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + i) changes to the Program, and + ii) additions to the Program; + where such changes and/or additions to the Program originate from + and are Distributed by that particular Contributor. A Contribution + "originates" from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include changes or additions to the Program that + are not Modified Works. + +"Contributor" means any person or entity that Distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which +are necessarily infringed by the use or sale of its Contribution alone +or when combined with the Program. + +"Program" means the Contributions Distributed in accordance with this +Agreement. + +"Recipient" means anyone who receives the Program under this Agreement +or any Secondary License (as applicable), including Contributors. + +"Derivative Works" shall mean any work, whether in Source Code or other +form, that is based on (or derived from) the Program and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. + +"Modified Works" shall mean any work in Source Code or other form that +results from an addition to, deletion from, or modification of the +contents of the Program, including, for purposes of clarity any new file +in Source Code form that contains any contents of the Program. Modified +Works shall not include works that contain only declarations, +interfaces, types, classes, structures, or files of the Program solely +in each case in order to link to, bind by name, or subclass the Program +or Modified Works thereof. + +"Distribute" means the acts of a) distributing or b) making available +in any manner that enables the transfer of a copy. + +"Source Code" means the form of a Program preferred for making +modifications, including but not limited to software source code, +documentation source, and configuration files. + +"Secondary License" means either the GNU General Public License, +Version 2.0, or any later versions of that license, including any +exceptions or additional permissions as identified by the initial +Contributor. + +2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free copyright + license to reproduce, prepare Derivative Works of, publicly display, + publicly perform, Distribute and sublicense the Contribution of such + Contributor, if any, and such Derivative Works. + + b) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free patent + license under Licensed Patents to make, use, sell, offer to sell, + import and otherwise transfer the Contribution of such Contributor, + if any, in Source Code or other form. This patent license shall + apply to the combination of the Contribution and the Program if, at + the time the Contribution is added by the Contributor, such addition + of the Contribution causes such combination to be covered by the + Licensed Patents. The patent license shall not apply to any other + combinations which include the Contribution. No hardware per se is + licensed hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the + rights and licenses granted hereunder, each Recipient hereby + assumes sole responsibility to secure any other intellectual + property rights needed, if any. For example, if a third party + patent license is required to allow Recipient to Distribute the + Program, it is Recipient's responsibility to acquire that license + before distributing the Program. + + d) Each Contributor represents that to its knowledge it has + sufficient copyright rights in its Contribution, if any, to grant + the copyright license set forth in this Agreement. + + e) Notwithstanding the terms of any Secondary License, no + Contributor makes additional grants to any Recipient (other than + those set forth in this Agreement) as a result of such Recipient's + receipt of the Program under the terms of a Secondary License + (if permitted under the terms of Section 3). + +3. REQUIREMENTS + +3.1 If a Contributor Distributes the Program in any form, then: + + a) the Program must also be made available as Source Code, in + accordance with section 3.2, and the Contributor must accompany + the Program with a statement that the Source Code for the Program + is available under this Agreement, and informs Recipients how to + obtain it in a reasonable manner on or through a medium customarily + used for software exchange; and + + b) the Contributor may Distribute the Program under a license + different than this Agreement, provided that such license: + i) effectively disclaims on behalf of all other Contributors all + warranties and conditions, express and implied, including + warranties or conditions of title and non-infringement, and + implied warranties or conditions of merchantability and fitness + for a particular purpose; + + ii) effectively excludes on behalf of all other Contributors all + liability for damages, including direct, indirect, special, + incidental and consequential damages, such as lost profits; + + iii) does not attempt to limit or alter the recipients' rights + in the Source Code under section 3.2; and + + iv) requires any subsequent distribution of the Program by any + party to be under a license that satisfies the requirements + of this section 3. + +3.2 When the Program is Distributed as Source Code: + + a) it must be made available under this Agreement, or if the + Program (i) is combined with other material in a separate file or + files made available under a Secondary License, and (ii) the initial + Contributor attached to the Source Code the notice described in + Exhibit A of this Agreement, then the Program may be made available + under the terms of such Secondary Licenses, and + + b) a copy of this Agreement must be included with each copy of + the Program. + +3.3 Contributors may not remove or alter any copyright, patent, +trademark, attribution notices, disclaimers of warranty, or limitations +of liability ("notices") contained within the Program from any copy of +the Program which they Distribute, provided that Contributors may add +their own appropriate notices. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities +with respect to end users, business partners and the like. While this +license is intended to facilitate the commercial use of the Program, +the Contributor who includes the Program in a commercial product +offering should do so in a manner which does not create potential +liability for other Contributors. Therefore, if a Contributor includes +the Program in a commercial product offering, such Contributor +("Commercial Contributor") hereby agrees to defend and indemnify every +other Contributor ("Indemnified Contributor") against any losses, +damages and costs (collectively "Losses") arising from claims, lawsuits +and other legal actions brought by a third party against the Indemnified +Contributor to the extent caused by the acts or omissions of such +Commercial Contributor in connection with its distribution of the Program +in a commercial product offering. The obligations in this section do not +apply to any claims or Losses relating to any actual or alleged +intellectual property infringement. In order to qualify, an Indemnified +Contributor must: a) promptly notify the Commercial Contributor in +writing of such claim, and b) allow the Commercial Contributor to control, +and cooperate with the Commercial Contributor in, the defense and any +related settlement negotiations. The Indemnified Contributor may +participate in any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial +product offering, Product X. That Contributor is then a Commercial +Contributor. If that Commercial Contributor then makes performance +claims, or offers warranties related to Product X, those performance +claims and warranties are such Commercial Contributor's responsibility +alone. Under this section, the Commercial Contributor would have to +defend claims against the other Contributors related to those performance +claims and warranties, and if a court requires any other Contributor to +pay any damages as a result, the Commercial Contributor must pay +those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN "AS IS" +BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF +TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR +PURPOSE. Each Recipient is solely responsible for determining the +appropriateness of using and distributing the Program and assumes all +risks associated with its exercise of rights under this Agreement, +including but not limited to the risks and costs of program errors, +compliance with applicable laws, damage to or loss of data, programs +or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS +SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST +PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE +EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under +applicable law, it shall not affect the validity or enforceability of +the remainder of the terms of this Agreement, and without further +action by the parties hereto, such provision shall be reformed to the +minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity +(including a cross-claim or counterclaim in a lawsuit) alleging that the +Program itself (excluding combinations of the Program with other software +or hardware) infringes such Recipient's patent(s), then such Recipient's +rights granted under Section 2(b) shall terminate as of the date such +litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it +fails to comply with any of the material terms or conditions of this +Agreement and does not cure such failure in a reasonable period of +time after becoming aware of such noncompliance. If all Recipient's +rights under this Agreement terminate, Recipient agrees to cease use +and distribution of the Program as soon as reasonably practicable. +However, Recipient's obligations under this Agreement and any licenses +granted by Recipient relating to the Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, +but in order to avoid inconsistency the Agreement is copyrighted and +may only be modified in the following manner. The Agreement Steward +reserves the right to publish new versions (including revisions) of +this Agreement from time to time. No one other than the Agreement +Steward has the right to modify this Agreement. The Eclipse Foundation +is the initial Agreement Steward. The Eclipse Foundation may assign the +responsibility to serve as the Agreement Steward to a suitable separate +entity. Each new version of the Agreement will be given a distinguishing +version number. The Program (including Contributions) may always be +Distributed subject to the version of the Agreement under which it was +received. In addition, after a new version of the Agreement is published, +Contributor may elect to Distribute the Program (including its +Contributions) under the new version. + +Except as expressly stated in Sections 2(a) and 2(b) above, Recipient +receives no rights or licenses to the intellectual property of any +Contributor under this Agreement, whether expressly, by implication, +estoppel or otherwise. All rights in the Program not expressly granted +under this Agreement are reserved. Nothing in this Agreement is intended +to be enforceable by any entity that is not a Contributor or Recipient. +No third-party beneficiary rights are created under this Agreement. + +Exhibit A - Form of Secondary Licenses Notice + +"This Source Code may also be made available under the following +Secondary Licenses when the conditions for such availability set forth +in the Eclipse Public License, v. 2.0 are satisfied: {name license(s), +version(s), and exceptions or additional permissions here}." + + Simply including a copy of this Agreement, including this Exhibit A + is not sufficient to license the Source Code under Secondary Licenses. + + If it is not possible or desirable to put the notice in a particular + file, then You may include the notice in a location (such as a LICENSE + file in a relevant directory) where a recipient would be likely to + look for such a notice. + + You may add additional accurate notices of copyright ownership. diff --git a/foundations/stream/README.md b/foundations/stream/README.md new file mode 100644 index 0000000000..a1e12967c0 --- /dev/null +++ b/foundations/stream/README.md @@ -0,0 +1,149 @@ +# Stream + +[![X (formerly Twitter) Follow](https://img.shields.io/twitter/follow/huly_io?style=for-the-badge)](https://x.com/huly_io) +![GitHub License](https://img.shields.io/github/license/hcengineering/platform?style=for-the-badge) + +## About + +The Stream is a high-performance HTTP-based transcoding service. *Stream* supports the **TUS protocol**, enabling +reliable, resumable transcoding. Designed for seamless and consistent media processing, it supports advanced transcoding +features with robust integration options. + +--- + +## Features + +### TUS Protocol Support + +- **Resumable transcoding**: Leveraging the TUS protocol, *Stream* ensures reliable and efficient transcoding bucket + processing. + +#### Input Support + +- **Supported Input Formats**: + - `mp4` + - `webm` + +#### Output Options + +- **Supported Output Formats**: + - `hls` + +#### Upload options + +- **s3 Upload**: Direct upload to S3 storage. +- **datalake Upload**: Upload to datalake storage. + +#### Key Functionalities + +- **Live transcoding with minimal upload time**: Transcoding results are available after stream completion. +- **Transcoding Cancellation**: Cancel or pause ongoing transcoding in real-time. +- **Transcoding Resumption**: Resume incomplete transcoding tasks efficiently. + +### Transcoding scheduling + +## Installation + +### Prerequisites + +- [Go](https://golang.org/dl/) (v1.23+ recommended) +- [ffmpeg](https://www.ffmpeg.org/download.html) (ensure it’s installed and available in your system's PATH) + +### Steps + +1. Install dependencies: + +```bash +go mod tidy +``` + +2. Build the service: + +```bash +docker build . -t hcengineering/stream:latest +``` + +--- + +## Configuration + +### App Env Configuration + +The following environment variables can be used: + +``` +KEY TYPE DEFAULT DESCRIPTION +STREAM_LOG_LEVEL String debug sets log level for the application +STREAM_SERVER_SECRET String server secret required to generate and verify tokens +STREAM_PPROF_ENABLED True or False true starts profile server on localhost:6060 if true +STREAM_INSECURE True or False false ignores authorization check if true +STREAM_SERVE_URL String 0.0.0.0:1080 listens on URL +STREAM_ENDPOINT_URL URL s3://127.0.0.1:9000 S3 or Datalake endpoint, example: s3://my-ip-address, datalake://my-ip-address +STREAM_MAX_PARALLEL_SCALING_COUNT Integer 2 how much parallel scaling can be processed +STREAM_MAX_THREAD_COUNT Integer 4 max number of threads for transcoder +STREAM_OUTPUT_DIR String /tmp/transcoding/ path to the directory where transcoded files are stored +STREAM_SENTRY_DSN String "" sentry dsn for error tracking +``` + +### Metadata + +**resolution:** if passed, set the resolution for the output, for example, 'resolution: 1920:1080'. + +**token:** must be provided to be authorized in the Huly's datalake service. + +**workspace:** required for uploading content to the datalake storage. + +#### S3 Env configuration + +if you're working with S3 storage type, these envs must be provided: +**AWS_ACCESS_KEY_ID** +**AWS_SECRET_ACCESS_KEY** + +## Usage + +The service exposes an HTTP API. + +Below are some examples of how to interact with it. + +### Transcode via TUS + +```bash +curl -X POST http://localhost:1080/recording \ + -H "Tus-Resumable: 1.0.0" \ + -H "Upload-Length: " \ + --data-binary @path/to/your/file.mp4 +``` + +Note: A TUS client is required to play with the service locally. +You can use the tus-js-client example with [video](https://github.com/tus/tus-js-client/blob/main/demos/browser/video.html). + +### Schedule a transcoding + +```bash +curl -X POST http://localhost:1080/transcoding \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer " \ + -d '{ + "source": "", + "format": "hls", + "workspace": "test" + }' +``` + +## Contributing + +We welcome contributions! To get started: + +1. Fork the repository. +2. Create a new branch for your feature or bug fix. +3. Submit a pull request describing your changes. + +--- + +## License + +This project is licensed under the [MIT License](LICENSE). + +--- + +Enjoy seamless transcoding with *Stream*! 🚀 \ No newline at end of file diff --git a/foundations/stream/cmd/stream/main.go b/foundations/stream/cmd/stream/main.go new file mode 100644 index 0000000000..b01c6b9609 --- /dev/null +++ b/foundations/stream/cmd/stream/main.go @@ -0,0 +1,155 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main provides huly-stream entry point function +package main + +import ( + "context" + "net/http" + "sync" + + "os" + "os/signal" + "syscall" + "time" + + "go.uber.org/zap" + + "github.com/hcengineering/stream/internal/pkg/api/v1/recording" + "github.com/hcengineering/stream/internal/pkg/api/v1/transcoding" + "github.com/hcengineering/stream/internal/pkg/config" + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/queue" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" +) + +func main() { + var ctx, cancel = signal.NotifyContext( + context.Background(), + os.Interrupt, + syscall.SIGHUP, + syscall.SIGTERM, + syscall.SIGQUIT, + ) + defer cancel() + ctx = log.WithFields(ctx) + + var logger = log.FromContext(ctx) + var cfg, err = config.FromEnv() + if err != nil { + panic(err.Error()) + } + logger.Sugar().Debug("using config", zap.Any("config", cfg)) + + // Set up OpenTelemetry. + otelShutdown, err := setupOTelSDK(ctx, cfg) + if err != nil { + panic(err.Error()) + } + defer func() { + if err := otelShutdown(context.Background()); err != nil { + logger.Error("failed to close queue consumer", zap.Error(err)) + } + }() + + var recordingHandler = recording.NewHandler(ctx, cfg) + var transcodingHandler = transcoding.NewHandler(ctx, cfg) + + // setup HTTP routes + mux := http.NewServeMux() + mux.Handle("/recording/", http.StripPrefix("/recording/", recordingHandler)) + mux.Handle("/recording", http.StripPrefix("/recording", recordingHandler)) + mux.Handle("/transcoding", http.StripPrefix("/transcoding", transcodingHandler)) + + // Wrap handler with OpenTelemetry instrumentation + var handler http.Handler = mux + if cfg.OtelEnabled && cfg.OtelTracesEnabled { + handler = otelhttp.NewHandler(handler, "http.server", + otelhttp.WithServerName(cfg.OtelServiceName)) + } + + server := &http.Server{ + Addr: cfg.ServeURL, + Handler: handler, + ReadHeaderTimeout: 15 * time.Second, + } + + var wg sync.WaitGroup + + wg.Add(1) + go func() { + defer wg.Done() + + logger.Info("http server started", zap.String("ServeURL", cfg.ServeURL)) + defer logger.Info("http server finished") + + // #nosec + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + logger.Error("http server error", zap.Error(err)) + cancel() + } + }() + + config := queue.ParseConfig(cfg.QueueConfig, "stream", cfg.Region) + logger.Info("using queue config", zap.Any("config", config)) + + consumerOptions := queue.ConsumerOptions{ + Topic: queue.TopicTranscodeRequest, + Group: "stream", + Config: config, + } + consumer := queue.NewConsumer(ctx, consumerOptions) + + producerOptions := queue.ProducerOptions{ + Topic: queue.TopicTranscodeResult, + Group: "stream", + Config: config, + RetryCount: 3, + } + producer := queue.NewProducer(ctx, producerOptions) + + wg.Add(1) + go func() { + defer wg.Done() + + logger.Info("queue worker started") + defer logger.Info("queue worker finished") + + worker := queue.NewWorker(ctx, consumer, producer, cfg) + if err := worker.Start(ctx); err != nil { + logger.Error("queue worker error", zap.Error(err)) + cancel() + } + }() + + <-ctx.Done() + + // Stop HTTP server to prevent accepting new connections + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer shutdownCancel() + + if err := server.Shutdown(shutdownCtx); err != nil { + logger.Error("http server shutdown error", zap.Error(err)) + } + + // Wait for all workers to complete + wg.Wait() + + // Close the consumer + if err := consumer.Close(); err != nil { + logger.Error("failed to close queue consumer", zap.Error(err)) + } + + logger.Info("shutdown complete") +} diff --git a/foundations/stream/cmd/stream/otel.go b/foundations/stream/cmd/stream/otel.go new file mode 100644 index 0000000000..1f16e0cbae --- /dev/null +++ b/foundations/stream/cmd/stream/otel.go @@ -0,0 +1,179 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "time" + + "github.com/hcengineering/stream/internal/pkg/config" + "github.com/pkg/errors" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" + "go.opentelemetry.io/otel/exporters/stdout/stdoutmetric" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/log/global" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/log" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.37.0" +) + +// setupOTelSDK bootstraps the OpenTelemetry pipeline. +// If it does not return an error, make sure to call shutdown for proper cleanup. +func setupOTelSDK(ctx context.Context, cfg *config.Config) (func(context.Context) error, error) { + if !cfg.OtelEnabled { + return func(context.Context) error { return nil }, nil + } + + var shutdownFuncs []func(context.Context) error + + shutdown := func(ctx context.Context) error { + var shutdownErr error + for _, fn := range shutdownFuncs { + if err := fn(ctx); err != nil { + if shutdownErr == nil { + shutdownErr = err + } else { + shutdownErr = errors.Wrap(err, shutdownErr.Error()) + } + } + } + shutdownFuncs = nil + return shutdownErr + } + + res, err := newResource(cfg) + if err != nil { + return shutdown, err + } + + prop := newPropagator() + otel.SetTextMapPropagator(prop) + + if cfg.OtelTracesEnabled { + tracerProvider, tracerErr := newTracerProvider(ctx, res) + if tracerErr != nil { + return shutdown, tracerErr + } + shutdownFuncs = append(shutdownFuncs, tracerProvider.Shutdown) + otel.SetTracerProvider(tracerProvider) + } + + if cfg.OtelMetricsEnabled { + meterProvider, meterErr := newMeterProvider(ctx, res) + if meterErr != nil { + return shutdown, meterErr + } + shutdownFuncs = append(shutdownFuncs, meterProvider.Shutdown) + otel.SetMeterProvider(meterProvider) + } + + if cfg.OtelLogsEnabled { + loggerProvider, loggerErr := newLoggerProvider(ctx, res) + if loggerErr != nil { + return shutdown, loggerErr + } + shutdownFuncs = append(shutdownFuncs, loggerProvider.Shutdown) + global.SetLoggerProvider(loggerProvider) + } + + return shutdown, err +} + +func newResource(cfg *config.Config) (*resource.Resource, error) { + defaultRes := resource.Default() + + customRes := resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName(cfg.OtelServiceName), + semconv.ServiceVersion(cfg.OtelServiceVersion), + ) + + return resource.Merge(defaultRes, customRes) +} + +func newPropagator() propagation.TextMapPropagator { + return propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + ) +} + +func newTracerProvider(ctx context.Context, res *resource.Resource) (*trace.TracerProvider, error) { + var exporter trace.SpanExporter + var err error + + exporter, err = otlptracehttp.New(ctx) + if err != nil { + exporter, err = stdouttrace.New(stdouttrace.WithPrettyPrint()) + if err != nil { + return nil, err + } + } + + tracerProvider := trace.NewTracerProvider( + trace.WithResource(res), + trace.WithBatcher(exporter, + trace.WithBatchTimeout(5*time.Second), + ), + ) + return tracerProvider, nil +} + +func newMeterProvider(ctx context.Context, res *resource.Resource) (*metric.MeterProvider, error) { + var exporter metric.Exporter + var err error + + exporter, err = otlpmetrichttp.New(ctx) + if err != nil { + exporter, err = stdoutmetric.New() + if err != nil { + return nil, err + } + } + + meterProvider := metric.NewMeterProvider( + metric.WithResource(res), + metric.WithReader(metric.NewPeriodicReader(exporter, + metric.WithInterval(60*time.Second))), + ) + + return meterProvider, nil +} + +func newLoggerProvider(ctx context.Context, res *resource.Resource) (*log.LoggerProvider, error) { + var exporter log.Exporter + var err error + + exporter, err = otlploghttp.New(ctx) + if err != nil { + exporter, err = stdoutlog.New() + if err != nil { + return nil, err + } + } + + loggerProvider := log.NewLoggerProvider( + log.WithResource(res), + log.WithProcessor(log.NewBatchProcessor(exporter)), + ) + + return loggerProvider, nil +} diff --git a/foundations/stream/go.mod b/foundations/stream/go.mod new file mode 100644 index 0000000000..4c0266ff8a --- /dev/null +++ b/foundations/stream/go.mod @@ -0,0 +1,76 @@ +module github.com/hcengineering/stream + +go 1.24.1 + +require ( + github.com/aws/aws-sdk-go-v2 v1.36.1 + github.com/aws/aws-sdk-go-v2/config v1.29.6 + github.com/aws/aws-sdk-go-v2/credentials v1.17.59 + github.com/aws/aws-sdk-go-v2/service/s3 v1.77.0 + github.com/golang-jwt/jwt/v5 v5.2.2 + github.com/google/uuid v1.6.0 + github.com/kelseyhightower/envconfig v1.4.0 + github.com/pkg/errors v0.9.1 + github.com/segmentio/kafka-go v0.4.48 + github.com/stretchr/testify v1.11.1 + github.com/tus/tusd/v2 v2.6.0 + github.com/valyala/fasthttp v1.59.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 + go.opentelemetry.io/otel v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 + go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.14.0 + go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 + go.opentelemetry.io/otel/log v0.14.0 + go.opentelemetry.io/otel/sdk v1.38.0 + go.opentelemetry.io/otel/sdk/log v0.14.0 + go.opentelemetry.io/otel/sdk/metric v1.38.0 + go.opentelemetry.io/otel/trace v1.38.0 + go.uber.org/zap v1.27.0 + golang.org/x/exp v0.0.0-20250215185904-eff6e970281f + gopkg.in/vansante/go-ffprobe.v2 v2.2.1 + k8s.io/utils v0.0.0-20241210054802-24370beab758 +) + +require ( + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.9 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect + github.com/aws/smithy-go v1.22.3 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.1 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/grpc v1.75.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/foundations/stream/go.sum b/foundations/stream/go.sum new file mode 100644 index 0000000000..7a61312aa8 --- /dev/null +++ b/foundations/stream/go.sum @@ -0,0 +1,214 @@ +github.com/Acconut/go-httptest-recorder v1.0.0 h1:TAv2dfnqp/l+SUvIaMAUK4GeN4+wqb6KZsFFFTGhoJg= +github.com/Acconut/go-httptest-recorder v1.0.0/go.mod h1:CwQyhTH1kq/gLyWiRieo7c0uokpu3PXeyF/nZjUNtmM= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/aws/aws-sdk-go-v2 v1.36.1 h1:iTDl5U6oAhkNPba0e1t1hrwAo02ZMqbrGq4k5JBWM5E= +github.com/aws/aws-sdk-go-v2 v1.36.1/go.mod h1:5PMILGVKiW32oDzjj6RU52yrNrDPUHcbZQYr1sM7qmM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.9 h1:VZPDrbzdsU1ZxhyWrvROqLY0nxFWgMCAzhn/nYz3X48= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.9/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg= +github.com/aws/aws-sdk-go-v2/config v1.29.6 h1:fqgqEKK5HaZVWLQoLiC9Q+xDlSp+1LYidp6ybGE2OGg= +github.com/aws/aws-sdk-go-v2/config v1.29.6/go.mod h1:Ft+WLODzDQmCTHDvqAH1JfC2xxbZ0MxpZAcJqmE1LTQ= +github.com/aws/aws-sdk-go-v2/credentials v1.17.59 h1:9btwmrt//Q6JcSdgJOLI98sdr5p7tssS9yAsGe8aKP4= +github.com/aws/aws-sdk-go-v2/credentials v1.17.59/go.mod h1:NM8fM6ovI3zak23UISdWidyZuI1ghNe2xjzUZAyT+08= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 h1:KwsodFKVQTlI5EyhRSugALzsV6mG/SGrdjlMXSZSdso= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28/go.mod h1:EY3APf9MzygVhKuPXAc5H+MkGb8k/DOSQjWS0LgkKqI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 h1:BjUcr3X3K0wZPGFg2bxOWW3VPN8rkE3/61zhP+IHviA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32/go.mod h1:80+OGC/bgzzFFTUmcuwD0lb4YutwQeKLFpmt6hoWapU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 h1:m1GeXHVMJsRsUAqG6HjZWx9dj7F5TR+cF1bjyfYyBd4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32/go.mod h1:IitoQxGfaKdVLNg0hD8/DXmAqNy0H4K2H2Sf91ti8sI= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 h1:Pg9URiobXy85kgFev3og2CuOZ8JZUBENF+dcgWBaYNk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 h1:OIHj/nAhVzIXGzbAE+4XmZ8FPvro3THr6NlqErJc3wY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32/go.mod h1:LiBEsDo34OJXqdDlRGsilhlIiXR7DL+6Cx2f4p1EgzI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.0 h1:kT2WeWcFySdYpPgyqJMSUE7781Qucjtn6wBvrgm9P+M= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.0/go.mod h1:WYH1ABybY7JK9TITPnk6ZlP7gQB8psI4c9qDmMsnLSA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 h1:SYVGSFQHlchIcy6e7x12bsrxClCXSP5et8cqVhL8cuw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13/go.mod h1:kizuDaLX37bG5WZaoxGPQR/LNFXpxp0vsUnqfkWXfNE= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 h1:OBsrtam3rk8NfBEq7OLOMm5HtQ9Yyw32X4UQMya/wjw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13/go.mod h1:3U4gFA5pmoCOja7aq4nSaIAGbaOHv2Yl2ug018cmC+Q= +github.com/aws/aws-sdk-go-v2/service/s3 v1.77.0 h1:RCOi1rDmLqOICym/6UeS2cqKED4T4m966w2rl1HfL+g= +github.com/aws/aws-sdk-go-v2/service/s3 v1.77.0/go.mod h1:VC4EKSHqT3nzOcU955VWHMGsQ+w67wfAUBSjC8NOo8U= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 h1:/eE3DogBjYlvlbhd2ssWyeuovWunHLxfgw3s/OJa4GQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.15/go.mod h1:2PCJYpi7EKeA5SkStAmZlF6fi0uUABuhtF8ILHjGc3Y= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 h1:M/zwXiL2iXUrHputuXgmO94TVNmcenPHxgLXLutodKE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14/go.mod h1:RVwIw3y/IqxC2YEXSIkAzRDdEU1iRabDPaYjpGCbCGQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 h1:TzeR06UCMUq+KA3bDkujxK1GVGy+G8qQN/QVYzGLkQE= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.14/go.mod h1:dspXf/oYWGWo6DEvj98wpaTeqt5+DMidZD0A9BYTizc= +github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= +github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/segmentio/kafka-go v0.4.48 h1:9jyu9CWK4W5W+SroCe8EffbrRZVqAOkuaLd/ApID4Vs= +github.com/segmentio/kafka-go v0.4.48/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tus/tusd/v2 v2.6.0 h1:Je243QDKnFTvm/WkLH2bd1oQ+7trolrflRWyuI0PdWI= +github.com/tus/tusd/v2 v2.6.0/go.mod h1:1Eb1lBoSRBfYJ/mQfFVjyw8ZdNMdBqW17vgQKl3Ah9g= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.59.0 h1:Qu0qYHfXvPk1mSLNqcFtEk6DpxgA26hy6bmydotDpRI= +github.com/valyala/fasthttp v1.59.0/go.mod h1:GTxNb9Bc6r2a9D0TWNSPwDz78UxnTGBViY3xZNEqyYU= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 h1:QQqYw3lkrzwVsoEX0w//EhH/TCnpRdEenKBOOEIMjWc= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0/go.mod h1:gSVQcr17jk2ig4jqJ2DX30IdWH251JcNAecvrqTxH1s= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 h1:Oe2z/BCg5q7k4iXC3cqJxKYg0ieRiOqF0cecFYdPTwk= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0/go.mod h1:ZQM5lAJpOsKnYagGg/zV2krVqTtaVdYdDkhMoX6Oalg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.14.0 h1:B/g+qde6Mkzxbry5ZZag0l7QrQBCtVm7lVjaLgmpje8= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.14.0/go.mod h1:mOJK8eMmgW6ocDJn6Bn11CcZ05gi3P8GylBXEkZtbgA= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0 h1:wm/Q0GAAykXv83wzcKzGGqAnnfLFyFe7RslekZuv+VI= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0/go.mod h1:ra3Pa40+oKjvYh+ZD3EdxFZZB0xdMfuileHAm4nNN7w= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 h1:kJxSDN4SgWWTjG/hPp3O7LCGLcHXFlvS2/FFOrwL+SE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0/go.mod h1:mgIOzS7iZeKJdeB8/NYHrJ48fdGc71Llo5bJ1J4DWUE= +go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM= +go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg= +go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= +go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/exp v0.0.0-20250215185904-eff6e970281f h1:oFMYAjX0867ZD2jcNiLBrI9BdpmEkvPyi5YrBGXbamg= +golang.org/x/exp v0.0.0-20250215185904-eff6e970281f/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/vansante/go-ffprobe.v2 v2.2.1 h1:sFV08OT1eZ1yroLCZVClIVd9YySgCh9eGjBWO0oRayI= +gopkg.in/vansante/go-ffprobe.v2 v2.2.1/go.mod h1:qF0AlAjk7Nqzqf3y333Ly+KxN3cKF2JqA3JT5ZheUGE= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0= +k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= diff --git a/foundations/stream/internal/pkg/api/v1/recording/handler.go b/foundations/stream/internal/pkg/api/v1/recording/handler.go new file mode 100644 index 0000000000..e65dc53578 --- /dev/null +++ b/foundations/stream/internal/pkg/api/v1/recording/handler.go @@ -0,0 +1,82 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package recording provides recording http handler. +package recording + +import ( + "context" + + "golang.org/x/exp/slog" + + "net/http" + "sync" + + "github.com/hcengineering/stream/internal/pkg/config" + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/mediaconvert" + "go.uber.org/zap" + + tusd "github.com/tus/tusd/v2/pkg/handler" +) + +type recordingHandler struct { + logger *zap.Logger + once sync.Once + cfg *config.Config + ctx context.Context + tusHandler http.Handler +} + +func (h *recordingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if !h.cfg.Insecure { + r.Header.Set("X-Forwarded-Proto", "https") + } + h.once.Do(h.initialize) + h.tusHandler.ServeHTTP(w, r) +} + +// NewHandler creates a new recording http handler, requires context and config. +func NewHandler(ctx context.Context, cfg *config.Config) http.Handler { + return &recordingHandler{ + logger: log.FromContext(ctx).With(zap.String("handler", "recording")), + cfg: cfg, + ctx: ctx, + } +} + +func (h *recordingHandler) initialize() { + coordinator := mediaconvert.NewStreamCoordinator(h.ctx, h.cfg) + + tusComposer := tusd.NewStoreComposer() + tusComposer.UseCore(coordinator) + tusComposer.UseTerminater(coordinator) + tusComposer.UseConcater(coordinator) + tusComposer.UseLengthDeferrer(coordinator) + + var tusHandler, err = tusd.NewHandler(tusd.Config{ + BasePath: "/recording", + StoreComposer: tusComposer, + RespectForwardedHeaders: true, + DisableDownload: true, + Cors: &tusd.DefaultCorsConfig, + NetworkTimeout: h.cfg.Timeout, + Logger: slog.Default(), + }) + + if err != nil { + panic(err.Error()) + } + + h.tusHandler = tusHandler +} diff --git a/foundations/stream/internal/pkg/api/v1/transcoding/handler.go b/foundations/stream/internal/pkg/api/v1/transcoding/handler.go new file mode 100644 index 0000000000..b9871f7e12 --- /dev/null +++ b/foundations/stream/internal/pkg/api/v1/transcoding/handler.go @@ -0,0 +1,86 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package transcoding provides transcoding http handler. +package transcoding + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/hcengineering/stream/internal/pkg/config" + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/mediaconvert" + "go.uber.org/zap" +) + +type trascodeHandler struct { + scheduler *mediaconvert.Scheduler + logger *zap.Logger +} + +func (t *trascodeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "" { + w.WriteHeader(http.StatusBadRequest) + _, _ = fmt.Fprint(w, "uri is not allowed") + return + } + + if r.Header.Get("Authorization") == "" { + w.WriteHeader(http.StatusUnauthorized) + _, _ = fmt.Fprint(w, "missed Authorization header") + return + } + + var decoder = json.NewDecoder(r.Body) + var task mediaconvert.Task + + if err := decoder.Decode(&task); err != nil { + w.WriteHeader(http.StatusBadRequest) + _, _ = fmt.Fprint(w, "can not decode request body") + return + } + + if !isSupportedFormat(task.Format) { + w.WriteHeader(http.StatusUnsupportedMediaType) + _, _ = fmt.Fprint(w, "output media format is not supported") + return + } + + if err := t.scheduler.Schedule(&task); err != nil { + w.WriteHeader(http.StatusTooManyRequests) + return + } + + w.WriteHeader(http.StatusOK) +} + +// NewHandler creates a new transcoding http handler, requires context and config. +func NewHandler(ctx context.Context, cfg *config.Config) http.Handler { + return &trascodeHandler{ + scheduler: mediaconvert.NewScheduler(ctx, cfg), + logger: log.FromContext(ctx).With(zap.String("handler", "transcoding")), + } +} + +func isSupportedFormat(s string) bool { + switch strings.ToLower(s) { + case "hls": + return true + default: + return false + } +} diff --git a/foundations/stream/internal/pkg/config/config.go b/foundations/stream/internal/pkg/config/config.go new file mode 100644 index 0000000000..61cc586a52 --- /dev/null +++ b/foundations/stream/internal/pkg/config/config.go @@ -0,0 +1,85 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package config provides configuration for the application +package config + +import ( + "net/url" + "time" + + "github.com/pkg/errors" + + "github.com/kelseyhightower/envconfig" +) + +// Config represents configuration for the huly-stream application. +type Config struct { + LogLevel string `split_words:"true" default:"debug" desc:"sets log level for the application"` + ServerSecret string `split_words:"true" default:"" desc:"server secret required to generate and verify tokens"` + PprofEnabled bool `split_words:"true" default:"true" desc:"starts profile server on localhost:6060 if true"` + Insecure bool `split_words:"true" default:"false" desc:"ignores authorization check if true"` + ServeURL string `split_words:"true" desc:"listen on url" default:"0.0.0.0:1080"` + EndpointURL *url.URL `split_words:"true" default:"s3://127.0.0.1:9000" desc:"S3 or Datalake endpoint, example: s3://my-ip-address, datalake://my-ip-address"` + MaxParallelTranscodingCount int `split_words:"true" default:"2" desc:"how much parallel transcodings can be processed"` + MaxThreadCount int `split_words:"true" default:"4" desc:"max number of threads for transcoder"` + + QueueConfig string `split_words:"true" default:"" desc:"queue config"` + Region string `split_words:"true" default:"" desc:"service region"` + + OutputDir string `split_words:"true" default:"/tmp/transcoing/" desc:"path to the directory with transcoding result."` + Timeout time.Duration `default:"5m" desc:"timeout for the upload"` + + // OpenTelemetry configuration + OtelEnabled bool `split_words:"true" default:"true" desc:"enable OpenTelemetry"` + OtelServiceName string `split_words:"true" default:"stream" desc:"service name for OpenTelemetry"` + OtelServiceVersion string `split_words:"true" default:"1.0.0" desc:"service version for OpenTelemetry"` + OtelTracesEnabled bool `split_words:"true" default:"true" desc:"enable OpenTelemetry traces"` + OtelMetricsEnabled bool `split_words:"true" default:"true" desc:"enable OpenTelemetry metrics"` + OtelLogsEnabled bool `split_words:"true" default:"false" desc:"enable OpenTelemetry logs"` +} + +// FromEnv creates new Config from env +func FromEnv() (*Config, error) { + var result Config + + if err := envconfig.Usage("stream", &result); err != nil { + return nil, err + } + + if err := envconfig.Process("stream", &result); err != nil { + return nil, err + } + + if *result.EndpointURL == (url.URL{}) { + result.EndpointURL = nil + } + + if !result.Insecure && result.ServerSecret == "" { + return nil, errors.New("server secret must be provided for secure confgiuration") + } + + return &result, nil +} + +// Endpoint returns upload address +func (c *Config) Endpoint() *url.URL { + var scheme = "https" + if c.Insecure { + scheme = "http" + } + return &url.URL{ + Scheme: scheme, + Host: c.EndpointURL.Host, + } +} diff --git a/foundations/stream/internal/pkg/executor/executor.go b/foundations/stream/internal/pkg/executor/executor.go new file mode 100644 index 0000000000..03050dfc9c --- /dev/null +++ b/foundations/stream/internal/pkg/executor/executor.go @@ -0,0 +1,93 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package executor provides command execution utilities. +package executor + +import ( + "bytes" + "context" + "io" + "os" + "os/exec" + "sync" + + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/tracing" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" +) + +var tracer = otel.Tracer("executor") + +// ExecuteCommands executes multiple commands in parallel +func ExecuteCommands(ctx context.Context, commands []*exec.Cmd) error { + logger := log.FromContext(ctx) + errCh := make(chan error, len(commands)) + + var mu sync.Mutex + var wg sync.WaitGroup + for _, cmd := range commands { + wg.Add(1) + + go func(cmd *exec.Cmd) { + defer wg.Done() + + _, span := tracer.Start(ctx, "cmd", trace.WithAttributes( + attribute.String("command", cmd.String()), + )) + defer span.End() + + var stdoutBuf = &bytes.Buffer{} + var stderrBuf = &bytes.Buffer{} + + if logger.Core().Enabled(zap.DebugLevel) { + cmd.Stdout = io.MultiWriter(os.Stdout, stdoutBuf) + cmd.Stderr = io.MultiWriter(os.Stderr, stderrBuf) + } else { + cmd.Stdout = stdoutBuf + cmd.Stderr = stderrBuf + } + + logger.Info("run command", zap.String("cmd", cmd.String())) + if err := cmd.Run(); err != nil { + tracing.RecordError(span, err) + errCh <- err + + // Lock so only on goroutine can write to stdout/stderr at the same time + mu.Lock() + defer mu.Unlock() + logger.Error("can not wait for command end", zap.Error(err), zap.String("cmd", cmd.String())) + if _, writeErr := os.Stdout.Write(stdoutBuf.Bytes()); writeErr != nil { + logger.Error("can not write stdout ", zap.Error(writeErr)) + } + if _, writeErr := os.Stderr.Write(stderrBuf.Bytes()); writeErr != nil { + logger.Error("can not write stderr", zap.Error(writeErr)) + } + } + }(cmd) + } + + wg.Wait() + close(errCh) + + for err := range errCh { + return err + } + + return nil +} diff --git a/foundations/stream/internal/pkg/executor/executor_test.go b/foundations/stream/internal/pkg/executor/executor_test.go new file mode 100644 index 0000000000..4e8bf3a4d3 --- /dev/null +++ b/foundations/stream/internal/pkg/executor/executor_test.go @@ -0,0 +1,139 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package executor_test + +import ( + "context" + "os/exec" + "testing" + "time" + + "github.com/hcengineering/stream/internal/pkg/executor" + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/stretchr/testify/assert" +) + +func TestCommandExecutor_Execute_Success(t *testing.T) { + tests := []struct { + name string + commands []*exec.Cmd + }{ + { + name: "single command", + commands: []*exec.Cmd{ + exec.Command("echo", "test"), + }, + }, + { + name: "multiple commands", + commands: []*exec.Cmd{ + exec.Command("echo", "test1"), + exec.Command("echo", "test2"), + exec.Command("echo", "test3"), + }, + }, + { + name: "empty commands", + commands: []*exec.Cmd{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + ctx = log.WithFields(ctx) + + err := executor.ExecuteCommands(ctx, tt.commands) + assert.NoError(t, err) + }) + } +} + +func TestCommandExecutor_Execute_Error(t *testing.T) { + tests := []struct { + name string + commands []*exec.Cmd + expectedError bool + }{ + { + name: "single failing command", + commands: []*exec.Cmd{ + exec.Command("false"), + }, + expectedError: true, + }, + { + name: "mixed success and failure", + commands: []*exec.Cmd{ + exec.Command("echo", "success"), + exec.Command("false"), + exec.Command("echo", "another success"), + }, + expectedError: true, + }, + { + name: "non-existent command", + commands: []*exec.Cmd{ + exec.Command("this-command-does-not-exist"), + }, + expectedError: true, + }, + { + name: "multiple failures", + commands: []*exec.Cmd{ + exec.Command("false"), + exec.Command("false"), + exec.Command("false"), + }, + expectedError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + ctx = log.WithFields(ctx) + + err := executor.ExecuteCommands(ctx, tt.commands) + if tt.expectedError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestCommandExecutor_Execute_Parallel(t *testing.T) { + // Create commands that sleep for different durations + commands := []*exec.Cmd{ + exec.Command("sleep", "0.1"), + exec.Command("sleep", "0.1"), + exec.Command("sleep", "0.1"), + exec.Command("sleep", "0.1"), + } + + ctx := context.Background() + ctx = log.WithFields(ctx) + + start := time.Now() + err := executor.ExecuteCommands(ctx, commands) + duration := time.Since(start) + + assert.NoError(t, err) + // If commands run in parallel, total time should be close to 0.1s, not 0.4s + assert.Less(t, duration, 200*time.Millisecond) +} diff --git a/foundations/stream/internal/pkg/log/zap.go b/foundations/stream/internal/pkg/log/zap.go new file mode 100644 index 0000000000..b85fcfbe5b --- /dev/null +++ b/foundations/stream/internal/pkg/log/zap.go @@ -0,0 +1,50 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package log provides simple api for using inherited logging +package log + +import ( + "context" + + "go.uber.org/zap" +) + +type contextKey struct{} + +// WithFields createsa new context with zap.Logger and passed fields +func WithFields(ctx context.Context, fields ...zap.Field) context.Context { + var logger = FromContext(ctx) + if logger == nil { + var err error + logger, err = zap.NewDevelopment() + if err != nil { + panic(err.Error()) + } + logger.Info("zap logger was initialized") + go func() { + <-ctx.Done() + _ = logger.Sync() + }() + } + return context.WithValue(ctx, contextKey{}, logger.With(fields...)) +} + +// FromContext returns zap.Logger from the context +func FromContext(ctx context.Context) *zap.Logger { + var val = ctx.Value(contextKey{}) + if val == nil { + return nil + } + return val.(*zap.Logger) +} diff --git a/foundations/stream/internal/pkg/manifest/hls.go b/foundations/stream/internal/pkg/manifest/hls.go new file mode 100644 index 0000000000..9e8f27248d --- /dev/null +++ b/foundations/stream/internal/pkg/manifest/hls.go @@ -0,0 +1,73 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package manifest provides data types for manifest based media files. +package manifest + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/hcengineering/stream/internal/pkg/profile" +) + +// GenerateHLSPlaylist generates master file for master files for resolution levels +func GenerateHLSPlaylist(profiles []profile.VideoProfile, outputPath, uploadID string) error { + p := filepath.Join(outputPath, uploadID, MasterPlaylistFileName(uploadID)) + d := filepath.Dir(p) + _ = os.MkdirAll(d, os.ModePerm) + // #nosec + file, err := os.Create(p) + if err != nil { + return err + } + defer func() { _ = file.Close() }() + + _, err = file.WriteString("#EXTM3U\n") + if err != nil { + return err + } + + for _, profile := range profiles { + var bandwidth = profile.Bandwidth + var resolution = fmt.Sprintf("%vx%v", profile.Width, profile.Height) + + _, err = fmt.Fprintf(file, "#EXT-X-STREAM-INF:BANDWIDTH=%d,RESOLUTION=%v\n", bandwidth, resolution) + if err != nil { + return err + } + + _, err = file.WriteString(PlaylistFileName(uploadID, profile.Name) + "\n") + if err != nil { + return err + } + } + + return nil +} + +// MasterPlaylistFileName generates master playlist file name +func MasterPlaylistFileName(source string) string { + return fmt.Sprintf("%s_master.m3u8", source) +} + +// PlaylistFileName generates profile playlist file name +func PlaylistFileName(source, profile string) string { + return fmt.Sprintf("%s_%s.m3u8", source, profile) +} + +// ThumbnailFileName generates thumbnail file name +func ThumbnailFileName(source string) string { + return fmt.Sprintf("%s.jpg", source) +} diff --git a/foundations/stream/internal/pkg/manifest/hls_test.go b/foundations/stream/internal/pkg/manifest/hls_test.go new file mode 100644 index 0000000000..b4f8d13e58 --- /dev/null +++ b/foundations/stream/internal/pkg/manifest/hls_test.go @@ -0,0 +1,74 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package manifest_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/hcengineering/stream/internal/pkg/manifest" + "github.com/hcengineering/stream/internal/pkg/profile" + "github.com/stretchr/testify/require" +) + +func TestGenerateHLSPlaylist(t *testing.T) { + profiles := []profile.VideoProfile{ + profile.Profile360p, + profile.Profile480p, + profile.Profile720p, + profile.Profile1080p, + profile.Profile1440p, + } + uploadID := "test123" + defer func() { + _ = os.RemoveAll(uploadID) + }() + + err := manifest.GenerateHLSPlaylist(profiles, "", uploadID) + require.NoError(t, err) + + outputPath := filepath.Join(uploadID, manifest.MasterPlaylistFileName(uploadID)) + + _, err = os.Stat(outputPath) + require.NoError(t, err, "Master playlist file should exist") + + // #nosec + data, err := os.ReadFile(outputPath) + require.NoError(t, err, "Error reading the generated file") + + playlistContent := string(data) + + require.Contains(t, playlistContent, "#EXTM3U", "File must start with #EXTM3U") + + for _, prof := range profiles { + expectedLine := manifest.PlaylistFileName(uploadID, prof.Name) + require.Contains(t, playlistContent, expectedLine, "Missing expected reference: "+expectedLine) + } +} + +func TestMasterPlaylistFileName(t *testing.T) { + filename := manifest.MasterPlaylistFileName("example") + require.Equal(t, "example_master.m3u8", filename) +} + +func TestPlaylistFileName(t *testing.T) { + filename := manifest.PlaylistFileName("example", "720p") + require.Equal(t, "example_720p.m3u8", filename) +} + +func TestThumbnailFileName(t *testing.T) { + filename := manifest.ThumbnailFileName("example") + require.Equal(t, "example.jpg", filename) +} diff --git a/foundations/stream/internal/pkg/mediaconvert/command.go b/foundations/stream/internal/pkg/mediaconvert/command.go new file mode 100644 index 0000000000..7017359282 --- /dev/null +++ b/foundations/stream/internal/pkg/mediaconvert/command.go @@ -0,0 +1,182 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package mediaconvert + +import ( + "context" + "fmt" + "io" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "github.com/pkg/errors" + + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/manifest" + "github.com/hcengineering/stream/internal/pkg/profile" + "go.uber.org/zap" +) + +// LogLevel is ffmpeg log level +type LogLevel string + +const ( + // LogLevelQuiet is quiet log level + LogLevelQuiet LogLevel = "quiet" + // LogLevelPanic is panic log level + LogLevelPanic LogLevel = "panic" + // LogLevelFatal is fatal log level + LogLevelFatal LogLevel = "fatal" + // LogLevelError is error log level + LogLevelError LogLevel = "error" + // LogLevelWarning is warning log level + LogLevelWarning LogLevel = "warning" + // LogLevelInfo is info log level + LogLevelInfo LogLevel = "info" + // LogLevelVerbose is verbose log level + LogLevelVerbose LogLevel = "verbose" + // LogLevelDebug is debug log level + LogLevelDebug LogLevel = "debug" + // LogLevelTrace is trace log level + LogLevelTrace LogLevel = "trace" +) + +// Options represents configuration for the ffmpeg command +type Options struct { + Input string + OutputDir string + LogLevel LogLevel + Threads int + UploadID string + Profiles []profile.VideoProfile +} + +func newFfmpegCommand(ctx context.Context, in io.Reader, args []string) (*exec.Cmd, error) { + if ctx == nil { + return nil, errors.New("ctx should not be nil") + } + + var cmd = exec.CommandContext(ctx, "ffmpeg", args...) + cmd.Stdin = in + + var logger = log.FromContext(ctx).With(zap.String("func", "newFFMpegCommand")) + logger.Debug("prepared command: ", zap.String("cmd", cmd.String())) + + return cmd, nil +} + +func buildCommonCommand(opts *Options) []string { + var result = []string{ + "-y", // Overwrite output files without asking. + "-v", string(opts.LogLevel), + "-err_detect", "ignore_err", + "-fflags", "+discardcorrupt", + "-threads", fmt.Sprint(opts.Threads), + "-i", opts.Input, + } + + // If input is a URL, add HTTP specific parameters + if strings.HasPrefix(opts.Input, "http://") || strings.HasPrefix(opts.Input, "https://") { + result = append(result, + "-reconnect", "1", + "-reconnect_streamed", "1", + "-reconnect_delay_max", "5", + ) + } + + return result +} + +func buildHLSCommand(profile profile.VideoProfile, opts *Options) []string { + return []string{ + "-f", "hls", + "-hls_time", "5", + // Use HLS flags + // - split_by_time + // Allow segments to start on frames other than key frames. + // This improves behavior on some players when the time between key frames is inconsistent, + // but may make things worse on others, and can cause some oddities during seeking. + // This flag should be used with the hls_time option. + // - temp_file + // Write segment data to filename.tmp and rename to filename only once the segment is complete. + "-hls_flags", "split_by_time+temp_file", + // Do not limit number of HLS segments + "-hls_list_size", "0", + "-hls_segment_filename", filepath.Join(opts.OutputDir, opts.UploadID, fmt.Sprintf("%s_%s_%s.ts", opts.UploadID, "%03d", profile.Name)), + } +} + +func buildVideoCommand(profile profile.VideoProfile) []string { + crf := profile.CRF + if crf == 0 { + crf = 23 + } + command := []string{ + // Transcode only first video and optionally audio stream + "-map", "0:v:0", + "-map", "0:a?", + // Set up codecs + "-c:a", profile.AudioCodec, + "-c:v", profile.VideoCodec, + "-preset", "veryfast", + "-crf", strconv.Itoa(crf), + "-g", "60", + } + + if profile.VideoCodec != "copy" && profile.Scale { + command = append(command, "-vf", "scale=-2:"+strconv.Itoa(profile.Height)) + } + + return command +} + +// BuildAudioCommand returns flags for getting the audio from the input +func BuildAudioCommand(opts *Options) []string { + var commonPart = buildCommonCommand(opts) + + return append(commonPart, + "-vn", "-acodec", + "copy", filepath.Join(opts.OutputDir, opts.UploadID), + ) +} + +// BuildVideoCommand returns ffmpeg command for converting video. +func BuildVideoCommand(opts *Options) []string { + if len(opts.Profiles) == 0 { + return []string{} + } + + var command = buildCommonCommand(opts) + for _, profile := range opts.Profiles { + command = append(command, buildVideoCommand(profile)...) + command = append(command, buildHLSCommand(profile, opts)...) + command = append(command, filepath.Join(opts.OutputDir, opts.UploadID, manifest.PlaylistFileName(opts.UploadID, profile.Name))) + } + return command +} + +// BuildThumbnailCommand creates a command that creates a thumbnail for the input video +func BuildThumbnailCommand(opts *Options) []string { + return append([]string{}, + "-y", // Overwrite output files without asking. + "-i", opts.Input, + "-vframes", "1", + "-update", "1", + filepath.Join(opts.OutputDir, opts.UploadID, opts.UploadID+".jpg"), + ) +} diff --git a/foundations/stream/internal/pkg/mediaconvert/command_test.go b/foundations/stream/internal/pkg/mediaconvert/command_test.go new file mode 100644 index 0000000000..c516f89e74 --- /dev/null +++ b/foundations/stream/internal/pkg/mediaconvert/command_test.go @@ -0,0 +1,96 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package mediaconvert_test + +import ( + "strings" + "testing" + + "github.com/hcengineering/stream/internal/pkg/mediaconvert" + "github.com/hcengineering/stream/internal/pkg/profile" + "github.com/stretchr/testify/require" +) + +func Test_BuildVideoCommand_Empty(t *testing.T) { + var profiles []profile.VideoProfile + + var rawCommand = mediaconvert.BuildVideoCommand(&mediaconvert.Options{ + OutputDir: "test", + Input: "pipe:0", + UploadID: "1", + Threads: 4, + LogLevel: mediaconvert.LogLevelDebug, + Profiles: profiles, + }) + + require.Empty(t, rawCommand) +} + +func Test_BuildVideoCommand_Scaling(t *testing.T) { + var profiles = []profile.VideoProfile{ + profile.Profile720p, + profile.Profile480p, + } + + var scaleCommand = mediaconvert.BuildVideoCommand(&mediaconvert.Options{ + OutputDir: "test", + Input: "pipe:0", + UploadID: "1", + Threads: 4, + LogLevel: mediaconvert.LogLevelDebug, + Profiles: profiles, + }) + + const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -c:a aac -c:v libx264 -preset veryfast -crf 25 -g 60 -vf scale=-2:720 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_720p.ts test/1/1_720p.m3u8 -map 0:v:0 -map 0:a? -c:a aac -c:v libx264 -preset veryfast -crf 27 -g 60 -vf scale=-2:480 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_480p.ts test/1/1_480p.m3u8` + + require.Contains(t, expected, strings.Join(scaleCommand, " ")) +} + +func Test_BuildVideoCommand_Original(t *testing.T) { + var profiles = []profile.VideoProfile{ + profile.MakeProfileOriginal(640, 480), + } + + var rawCommand = mediaconvert.BuildVideoCommand(&mediaconvert.Options{ + OutputDir: "test", + Input: "pipe:0", + UploadID: "1", + Threads: 4, + LogLevel: mediaconvert.LogLevelDebug, + Profiles: profiles, + }) + + const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -c:a copy -c:v copy -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_orig.ts test/1/1_orig.m3u8` + + require.Contains(t, expected, strings.Join(rawCommand, " ")) +} + +func Test_BuildVideoCommand_OriginalT(t *testing.T) { + var profiles = []profile.VideoProfile{ + profile.MakeProfileOriginalT(640, 480), + } + + var rawCommand = mediaconvert.BuildVideoCommand(&mediaconvert.Options{ + OutputDir: "test", + Input: "pipe:0", + UploadID: "1", + Threads: 4, + LogLevel: mediaconvert.LogLevelDebug, + Profiles: profiles, + }) + + const expected = `-y -v debug -err_detect ignore_err -fflags +discardcorrupt -threads 4 -i pipe:0 -map 0:v:0 -map 0:a? -c:a aac -c:v libx264 -preset veryfast -crf 23 -g 60 -f hls -hls_time 5 -hls_flags split_by_time+temp_file -hls_list_size 0 -hls_segment_filename test/1/1_%03d_orig.ts test/1/1_orig.m3u8` + + require.Contains(t, expected, strings.Join(rawCommand, " ")) +} diff --git a/foundations/stream/internal/pkg/mediaconvert/coordinator.go b/foundations/stream/internal/pkg/mediaconvert/coordinator.go new file mode 100644 index 0000000000..48c20b78f2 --- /dev/null +++ b/foundations/stream/internal/pkg/mediaconvert/coordinator.go @@ -0,0 +1,221 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package mediaconvert + +import ( + "context" + "fmt" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/pkg/errors" + + "github.com/google/uuid" + "github.com/hcengineering/stream/internal/pkg/config" + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/manifest" + "github.com/hcengineering/stream/internal/pkg/sharedpipe" + "github.com/hcengineering/stream/internal/pkg/storage" + "github.com/hcengineering/stream/internal/pkg/uploader" + "github.com/tus/tusd/v2/pkg/handler" + "go.uber.org/zap" +) + +var recordingDir = "r" + +// StreamCoordinator represents manager for streams. It creates a new stream for a client and manages it's life cycle. +type StreamCoordinator struct { + conf *config.Config + outputDir string + uploadOptions uploader.Options + + mainContext context.Context + logger *zap.Logger + + streams sync.Map + cancels sync.Map +} + +var _ handler.DataStore = (*StreamCoordinator)(nil) +var _ handler.ConcaterDataStore = (*StreamCoordinator)(nil) +var _ handler.TerminaterDataStore = (*StreamCoordinator)(nil) +var _ handler.LengthDeferrerDataStore = (*StreamCoordinator)(nil) + +// NewStreamCoordinator creates a new scheduler for transcode operations. +func NewStreamCoordinator(ctx context.Context, c *config.Config) *StreamCoordinator { + outputDir := filepath.Join(c.OutputDir, recordingDir) + + return &StreamCoordinator{ + conf: c, + outputDir: outputDir, + uploadOptions: uploader.Options{ + RetryDelay: time.Millisecond * 100, + Timeout: c.Timeout, + WorkerCount: uint32(c.MaxThreadCount), + RetryCount: 10, + BufferSize: 128, + Dir: outputDir, + }, + mainContext: ctx, + logger: log.FromContext(ctx).With(zap.String("Scheduler", outputDir)), + } +} + +// NewUpload creates a new worker with passed parameters +func (s *StreamCoordinator) NewUpload(ctx context.Context, info handler.FileInfo) (handler.Upload, error) { + if info.ID == "" { + info.ID = uuid.NewString() + } + s.logger.Sugar().Debugf("stream: %v", info) + s.logger.Debug("NewUpload", zap.String("ID", info.ID)) + + var stream = &Stream{ + writer: sharedpipe.NewWriter(), + info: info, + logger: log.FromContext(s.mainContext).With(zap.String("stream", info.ID)), + done: make(chan struct{}), + } + + width, err := strconv.Atoi(info.MetaData["width"]) + if err != nil { + return nil, errors.Wrapf(err, "can not parse video width: %v", info.MetaData["width"]) + } + + height, err := strconv.Atoi(info.MetaData["height"]) + if err != nil { + return nil, errors.Wrapf(err, "can not parse video height: %v", info.MetaData["height"]) + } + + meta := VideoMeta{ + Width: width, + Height: height, + Codec: extractCodec(info.MetaData["contentType"]), + ContentType: extractContentType(info.MetaData["contentType"]), + } + + if s.conf.EndpointURL != nil { + s.logger.Sugar().Debugf("initializing uploader for %v", info) + + // setup content uploader for transcoded outputs + var opts = s.uploadOptions + opts.Dir = filepath.Join(opts.Dir, info.ID) + opts.Source = manifest.MasterPlaylistFileName(info.ID) + + // create storage backend + var stg, err = storage.NewStorageByURL(s.mainContext, s.conf.Endpoint(), s.conf.EndpointURL.Scheme, info.MetaData["token"], info.MetaData["workspace"]) + if err != nil { + s.logger.Error("can not create storage by url", zap.Error(err)) + return nil, errors.Wrapf(err, "can not create storage") + } + stream.storage = stg + + // if storage supports multipart, initialize raw upload + if ms, ok := stg.(storage.MultipartStorage); ok { + multipart, err := NewMultipartUpload(s.mainContext, ms, info, meta.ContentType) + if err != nil { + s.logger.Error("multipart upload failed", zap.Error(err)) + return nil, errors.Wrapf(err, "multipart upload failed") + } + stream.multipart = multipart + } + } + + s.streams.Store(stream.info.ID, stream) + if err := stream.start(); err != nil { + return nil, err + } + + s.manageTimeout(stream) + + s.logger.Debug("NewUpload", zap.String("done", info.ID)) + return stream, nil +} + +// GetUpload returns current a worker based on upload id +func (s *StreamCoordinator) GetUpload(ctx context.Context, id string) (upload handler.Upload, err error) { + logger := s.logger.With(zap.String("func", "GetUpload")).With(zap.String("id", id)) + + if v, ok := s.streams.Load(id); ok { + logger.Debug("found stream") + var w = v.(*Stream) + s.manageTimeout(w) + return w, nil + } + + logger.Warn("stream not found") + return nil, fmt.Errorf("stream not found: %v", id) +} + +// AsTerminatableUpload returns tusd handler.TerminatableUpload +func (s *StreamCoordinator) AsTerminatableUpload(upload handler.Upload) handler.TerminatableUpload { + return upload.(*Stream) +} + +// AsLengthDeclarableUpload returns tusd handler.LengthDeclarableUpload +func (s *StreamCoordinator) AsLengthDeclarableUpload(upload handler.Upload) handler.LengthDeclarableUpload { + return upload.(*Stream) +} + +func (s *StreamCoordinator) manageTimeout(w *Stream) { + var cancelCtx, cancel = context.WithCancel(context.Background()) + if v, ok := s.cancels.Load(w.info.ID); ok { + v.(context.CancelFunc)() + } + s.cancels.Store(w.info.ID, cancel) + go func() { + select { + case <-w.done: + w.logger.Debug("stream has finished") + s.cancels.Delete(w.info.ID) + s.streams.Delete(w.info.ID) + return + case <-cancelCtx.Done(): + w.logger.Debug("stream timeout has refreshed") + return + case <-time.After(s.conf.Timeout): + w.logger.Error("stream timeout") + s.cancels.Delete(w.info.ID) + var terminateCtx, terminateCancel = context.WithTimeout(context.Background(), s.conf.Timeout) + defer terminateCancel() + _ = w.Terminate(terminateCtx) + s.streams.Delete(w.info.ID) + } + }() +} + +func extractCodec(mimeType string) string { + codecRegex := regexp.MustCompile(`codecs["\s=]+([^",\s]+)`) + matches := codecRegex.FindStringSubmatch(mimeType) + codec := "unknown" + if len(matches) > 1 { + codec = matches[1] + } + + return codec +} + +func extractContentType(mimeType string) string { + contentType := "video/mp4" + parts := strings.Split(mimeType, ";") + if parts[0] != "" { + contentType = strings.TrimSpace(parts[0]) + } + return contentType +} diff --git a/foundations/stream/internal/pkg/mediaconvert/multipart.go b/foundations/stream/internal/pkg/mediaconvert/multipart.go new file mode 100644 index 0000000000..b153f62fcb --- /dev/null +++ b/foundations/stream/internal/pkg/mediaconvert/multipart.go @@ -0,0 +1,190 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package mediaconvert provides types and functions for video transcoding +package mediaconvert + +import ( + "bytes" + "context" + "time" + + "github.com/pkg/errors" + + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/storage" + "github.com/tus/tusd/v2/pkg/handler" + "go.uber.org/zap" +) + +// minimum size for multipart parts (backend requires >=5MiB for all but last part) +const minPartSize = 5 * 1024 * 1024 + +// MultipartUpload uploads chunks via multipart upload +type MultipartUpload struct { + logger *zap.Logger + buffer *bytes.Buffer + info handler.FileInfo + + storage storage.MultipartStorage + objectName string + uploadID string + parts []storage.MultipartPart + nextPartNum int + + terminated bool + completed bool + + bytesWritten int64 + bytesUploaded int64 +} + +// NewMultipartUpload creates a new multipart upload +func NewMultipartUpload( + ctx context.Context, + multipartStorage storage.MultipartStorage, + info handler.FileInfo, + contentType string, +) (*MultipartUpload, error) { + objectName := info.ID + uploadID, err := multipartStorage.MultipartUploadStart(ctx, objectName, contentType) + if err != nil { + return nil, errors.Wrap(err, "failed to initialize multipart upload") + } + + logger := log.FromContext(ctx).With(zap.String("multipart", "upload"), zap.String("uploadID", uploadID)) + + return &MultipartUpload{ + logger: logger, + buffer: bytes.NewBuffer(nil), + info: info, + storage: multipartStorage, + objectName: objectName, + uploadID: uploadID, + parts: make([]storage.MultipartPart, 0), + nextPartNum: 1, + }, nil +} + +// Write writes chunk of data to the storage +func (w *MultipartUpload) Write(ctx context.Context, data []byte) error { + if w.terminated || w.completed { + return errors.New("upload already terminated or completed") + } + + if err := ctx.Err(); err != nil { + return err + } + + _, err := w.buffer.Write(data) + if err != nil { + return errors.Wrap(err, "failed to write to buffer") + } + w.bytesWritten += int64(len(data)) + + // flush parts of at least minPartSize + for w.buffer.Len() >= minPartSize { + partNum := w.nextPartNum + partData := w.buffer.Next(minPartSize) + + if err := ctx.Err(); err != nil { + return err + } + + part, err := w.storage.MultipartUploadPart(ctx, w.objectName, w.uploadID, partNum, partData) + if err != nil { + w.logger.Error("multipart upload part failed", zap.Error(err), zap.Int("partNumber", partNum)) + return errors.Wrap(err, "failed to upload part") + } + + w.bytesUploaded += int64(len(partData)) + w.parts = append(w.parts, *part) + w.nextPartNum++ + } + + return nil +} + +// Terminate cancels the upload +func (w *MultipartUpload) Terminate(ctx context.Context) error { + if w.terminated || w.completed { + return nil + } + w.terminated = true + + w.logger.Debug("terminating multipart upload", zap.Int("parts", len(w.parts))) + + // create new context in case the main context is canceled + cancelCtx := ctx + if ctx.Err() != nil { + var cancel context.CancelFunc + cancelCtx, cancel = context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + } + + if err := w.storage.MultipartUploadCancel(cancelCtx, w.objectName, w.uploadID); err != nil { + w.logger.Error("multipart upload cancel failed", zap.Error(err)) + return errors.Wrap(err, "failed to cancel multipart upload") + } + + w.logger.Debug("multipart upload terminated", zap.Int("parts", len(w.parts))) + + return nil +} + +// Complete uploads last bytes and completes the upload +func (w *MultipartUpload) Complete(ctx context.Context) error { + if w.completed { + return nil + } + + if w.terminated { + return errors.New("cannot complete terminated upload") + } + + w.logger.Debug("finishing multipart upload", zap.Int("parts", len(w.parts))) + + // flush any remaining data as last part + if w.buffer.Len() > 0 { + partNum := w.nextPartNum + lastData := w.buffer.Bytes() + + part, err := w.storage.MultipartUploadPart(ctx, w.objectName, w.uploadID, partNum, lastData) + if err != nil { + w.logger.Error("multipart upload last part failed", zap.Error(err), zap.Int("partNumber", partNum)) + return errors.Wrap(err, "failed to upload last part") + } + + w.bytesUploaded += int64(len(lastData)) + w.parts = append(w.parts, *part) + } + + if len(w.parts) == 0 { + w.logger.Warn("cannot complete upload with no parts") + return errors.New("cannot complete upload with no parts") + } + + if err := w.storage.MultipartUploadComplete(ctx, w.objectName, w.uploadID, w.parts); err != nil { + w.logger.Error("multipart upload complete failed", zap.Error(err)) + return errors.Wrap(err, "failed to complete multipart upload") + } + + w.completed = true + w.logger.Info( + "multipart upload completed", + zap.Int64("bytesUploaded", w.bytesUploaded), + zap.Int64("bytesWritten", w.bytesWritten), + ) + + return nil +} diff --git a/foundations/stream/internal/pkg/mediaconvert/scheduler.go b/foundations/stream/internal/pkg/mediaconvert/scheduler.go new file mode 100644 index 0000000000..d41998f75d --- /dev/null +++ b/foundations/stream/internal/pkg/mediaconvert/scheduler.go @@ -0,0 +1,338 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package mediaconvert + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/google/uuid" + "github.com/hcengineering/stream/internal/pkg/config" + "github.com/hcengineering/stream/internal/pkg/executor" + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/manifest" + "github.com/hcengineering/stream/internal/pkg/storage" + "github.com/hcengineering/stream/internal/pkg/token" + "github.com/hcengineering/stream/internal/pkg/tracing" + "github.com/hcengineering/stream/internal/pkg/uploader" + "github.com/pkg/errors" + "go.opentelemetry.io/otel" + "go.uber.org/zap" + "gopkg.in/vansante/go-ffprobe.v2" +) + +var tracer = otel.Tracer("mediaconvert") + +// Task represents transcoding task +type Task struct { + ID string + Status string + Source string + Format string + Workspace string + Metadata map[string]string +} + +// TaskResult represents transcoding task result +type TaskResult struct { + Playlist string `json:"playlist"` + Thumbnail string `json:"thumbnail"` + Width int `json:"width"` + Height int `json:"height"` +} + +// Scheduler manages transcoding tasks by passed config +type Scheduler struct { + logger *zap.Logger + taskCh chan *Task + cfg *config.Config + ctx context.Context +} + +// Schedule schedules a task to transcode +func (p *Scheduler) Schedule(t *Task) error { + t.ID = uuid.NewString() + t.Status = "planned" + + select { + case p.taskCh <- t: + return nil + default: + return fmt.Errorf("task queue is full") + } +} + +// NewScheduler creates a new instance of transcoding task scheduler +func NewScheduler(ctx context.Context, cfg *config.Config) *Scheduler { + var p = &Scheduler{ + taskCh: make(chan *Task, 128), + cfg: cfg, + ctx: ctx, + logger: log.FromContext(ctx).With(zap.String("transcoding", "scheduler")), + } + + go p.start() + + return p +} + +func (p *Scheduler) start() { + go func() { + <-p.ctx.Done() + close(p.taskCh) + }() + + for range p.cfg.MaxParallelTranscodingCount { + go func() { + for task := range p.taskCh { + err := tracing.WithSpan(p.ctx, tracer, "transcode", func(ctx context.Context) error { + return p.processTask(ctx, task) + }) + if err != nil { + p.logger.Error("failed to process task", zap.Error(err)) + } + } + }() + } +} + +// TODO: add a factory pattern to process tasks by different media type +func (p *Scheduler) processTask(ctx context.Context, task *Task) error { + var logger = p.logger.With(zap.String("task-id", task.ID)) + + logger.Debug("start") + defer logger.Debug("finished") + + logger.Debug("phase 1: get a token") + var tokenString, err = token.NewToken(p.cfg.ServerSecret, task.Workspace, "stream") + if err != nil { + logger.Error("can not create token", zap.Error(err)) + return errors.Wrapf(err, "can not create token") + } + + logger.Debug("phase 2: preparing fs") + var destinationFolder = filepath.Join(p.cfg.OutputDir, task.ID) + var _, filename = filepath.Split(task.Source) + var sourceFilePath = filepath.Join(destinationFolder, filename) + err = os.MkdirAll(destinationFolder, os.ModePerm) + if err != nil { + logger.Error("can not create temporary folder", zap.Error(err)) + return errors.Wrapf(err, "can not create temporary folder") + } + + defer func() { + if err = os.RemoveAll(destinationFolder); err != nil { + logger.Error("failed to cleanup temporary folder", zap.Error(err)) + } + }() + + logger.Debug("phase 3: get the remote file") + + remoteStorage, err := storage.NewStorageByURL(ctx, p.cfg.Endpoint(), p.cfg.EndpointURL.Scheme, tokenString, task.Workspace) + + if err != nil { + logger.Error("can not create storage by url", zap.Error(err), zap.String("url", p.cfg.EndpointURL.String())) + _ = os.RemoveAll(destinationFolder) + return err + } + + stat, err := remoteStorage.StatFile(ctx, task.Source) + if err != nil { + logger.Error("can not stat a file", zap.Error(err), zap.String("filepath", task.Source)) + _ = os.RemoveAll(destinationFolder) + return errors.Wrapf(err, "can not stat a file") + } + + if !IsSupportedMediaType(stat.Type) { + logger.Info("unsupported media type", zap.String("type", stat.Type)) + _ = os.RemoveAll(destinationFolder) + return fmt.Errorf("unsupported media type: %s", stat.Type) + } + + if err = remoteStorage.GetFile(ctx, task.Source, sourceFilePath); err != nil { + logger.Error("can not download a file", zap.Error(err), zap.String("filepath", task.Source)) + _ = os.RemoveAll(destinationFolder) + // TODO: reschedule + return errors.Wrapf(err, "can not download a file") + } + + logger.Debug("phase 4: prepare to transcode") + probe, err := tracing.WithSpanResult(ctx, tracer, "ffprobe", func(ctx context.Context) (*ffprobe.ProbeData, error) { + return ffprobe.ProbeURL(ctx, sourceFilePath) + }) + if err != nil { + logger.Error("can not get probe for a file", zap.Error(err), zap.String("filepath", sourceFilePath)) + _ = os.RemoveAll(destinationFolder) + return errors.Wrapf(err, "can not get ffprobe") + } + + audioStream := probe.FirstAudioStream() + if audioStream == nil { + logger.Info("no audio stream found in the file", zap.String("filepath", sourceFilePath)) + } + + videoStream := probe.FirstVideoStream() + if videoStream == nil { + logger.Error("no video stream found in the file", zap.String("filepath", sourceFilePath)) + _ = os.RemoveAll(destinationFolder) + return fmt.Errorf("no video stream found") + } + + logger.Debug("video stream found", zap.String("codec", videoStream.CodecName), zap.Int("width", videoStream.Width), zap.Int("height", videoStream.Height)) + + meta := VideoMeta{ + Width: videoStream.Width, + Height: videoStream.Height, + Codec: videoStream.CodecName, + ContentType: stat.Type, + } + + var profiles = DefaultTranscodingProfiles(meta) + + var opts = Options{ + Input: sourceFilePath, + OutputDir: p.cfg.OutputDir, + UploadID: task.ID, + Threads: p.cfg.MaxThreadCount, + Profiles: profiles, + // Level: level, + // Transcode: !IsHLSSupportedVideoCodec(codec), + // ScalingLevels: append(resconv.SubLevels(res), level), + } + + logger.Debug("phase 5: start async upload process") + var uploader = uploader.New(ctx, remoteStorage, uploader.Options{ + Dir: destinationFolder, + WorkerCount: uint32(opts.Threads), + BufferSize: 128, + RetryCount: 10, + RetryDelay: time.Millisecond * 100, + Timeout: p.cfg.Timeout, + Source: task.Source, + SourceFile: sourceFilePath, + }) + + err = manifest.GenerateHLSPlaylist(profiles, p.cfg.OutputDir, opts.UploadID) + if err != nil { + logger.Error("can not generate hls playlist", zap.String("out", p.cfg.OutputDir), zap.String("uploadID", opts.UploadID)) + _ = os.RemoveAll(destinationFolder) + return errors.Wrapf(err, "can not generate hls playlist") + } + + go uploader.Start() + + logger.Debug("phase 6: start async transcode processes") + + var argsSlice = [][]string{ + BuildThumbnailCommand(&opts), + BuildVideoCommand(&opts), + } + var cmds []*exec.Cmd + + for _, args := range argsSlice { + cmd, cmdErr := newFfmpegCommand(ctx, nil, args) + if cmdErr != nil { + logger.Error("can not create a new command", zap.Error(cmdErr), zap.Strings("args", args)) + go uploader.Cancel() + return errors.Wrapf(cmdErr, "can not create a new command") + } + cmds = append(cmds, cmd) + } + + logger.Debug("phase 7: wait for the result") + execErr := tracing.WithSpan(ctx, tracer, "ffmpeg", func(ctx context.Context) error { + return executor.ExecuteCommands(ctx, cmds) + }) + if execErr != nil { + logger.Error("can not wait for command end ", zap.Error(execErr)) + go uploader.Cancel() + return errors.Wrapf(execErr, "can not execute command") + } + + logger.Debug("phase 8: schedule cleanup") + go uploader.Stop() + + logger.Debug("phase 9: try to set metadata") + + if metaProvider, ok := remoteStorage.(storage.MetaProvider); ok { + var result = TaskResult{ + Width: videoStream.Width, + Height: videoStream.Height, + Playlist: manifest.MasterPlaylistFileName(task.ID), + Thumbnail: manifest.ThumbnailFileName(task.ID), + } + + logger.Debug( + "applying metadata", + zap.String("url", result.Playlist), + zap.String("thumbnail", result.Thumbnail), + zap.String("source", task.Source), + ) + err = metaProvider.PatchMeta( + ctx, + task.Source, + &storage.Metadata{ + "hls": map[string]any{ + "source": result.Playlist, + "thumbnail": result.Thumbnail, + }, + "width": result.Width, + "height": result.Height, + }, + ) + if err != nil { + logger.Error("can not patch the source file", zap.Error(err)) + } + } + return nil +} + +// IsHLSSupportedVideoCodec checks whether the codec is supported by HLS +func IsHLSSupportedVideoCodec(codec string) bool { + switch codec { + case "h264", "h265": + return true + default: + if strings.HasPrefix(codec, "avc1") { + return true + } else if strings.HasPrefix(codec, "av1") { + return true + } + return false + } +} + +// IsSupportedMediaType checks whether transcoding is supported for given media type +func IsSupportedMediaType(mediaType string) bool { + // Explicitly disable conversion for video/mp2t and video/x-mpegurl + switch mediaType { + case "video/mp2t", "video/x-mpegurl": + return false + case "video/mp4": + return true + case "video/webm": + return true + case "video/quicktime": + return true + default: + return false + } +} diff --git a/foundations/stream/internal/pkg/mediaconvert/strategy.go b/foundations/stream/internal/pkg/mediaconvert/strategy.go new file mode 100644 index 0000000000..3fff4a80aa --- /dev/null +++ b/foundations/stream/internal/pkg/mediaconvert/strategy.go @@ -0,0 +1,68 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package mediaconvert + +import ( + "fmt" + + "github.com/hcengineering/stream/internal/pkg/profile" + "github.com/hcengineering/stream/internal/pkg/resconv" +) + +// VideoMeta contains information about the video +type VideoMeta struct { + Width int + Height int + Codec string + ContentType string +} + +// DefaultTranscodingProfiles uses original resolution and two more resolutions +func DefaultTranscodingProfiles(meta VideoMeta) []profile.VideoProfile { + var profiles = make([]profile.VideoProfile, 0) + + var res = fmt.Sprintf("%v:%v", meta.Width, meta.Height) + var sublevels = resconv.SubLevels(res) + + if IsHLSSupportedVideoCodec(meta.Codec) { + profile := profile.MakeProfileOriginal(meta.Width, meta.Height) + profiles = append(profiles, profile) + } else { + profile := profile.MakeProfileOriginalT(meta.Width, meta.Height) + profiles = append(profiles, profile) + } + + for _, level := range sublevels { + if profile, ok := profile.GetProfileByName(level); ok { + profiles = append(profiles, profile) + } + } + + return profiles +} + +// FastTranscodingProfiles uses fastest possible video profile +func FastTranscodingProfiles(meta VideoMeta) []profile.VideoProfile { + if IsHLSSupportedVideoCodec(meta.Codec) { + return []profile.VideoProfile{ + profile.MakeProfileOriginal(meta.Width, meta.Height), + } + } + + return []profile.VideoProfile{ + profile.Profile360p, + } +} diff --git a/foundations/stream/internal/pkg/mediaconvert/stream.go b/foundations/stream/internal/pkg/mediaconvert/stream.go new file mode 100644 index 0000000000..a4201532ad --- /dev/null +++ b/foundations/stream/internal/pkg/mediaconvert/stream.go @@ -0,0 +1,202 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package mediaconvert provides types and functions for video transcoding +package mediaconvert + +import ( + "context" + "io" + "sync" + + "github.com/pkg/errors" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + "github.com/hcengineering/stream/internal/pkg/sharedpipe" + "github.com/hcengineering/stream/internal/pkg/storage" + "github.com/hcengineering/stream/internal/pkg/tracing" + "github.com/tus/tusd/v2/pkg/handler" + "go.uber.org/zap" +) + +// Stream manages client's input and transcodes it based on the passed configuration +type Stream struct { + logger *zap.Logger + info handler.FileInfo + writer *sharedpipe.Writer + reader *sharedpipe.Reader + storage storage.Storage + multipart *MultipartUpload + + done chan struct{} +} + +var _ handler.Upload = (*Stream)(nil) +var _ handler.ConcatableUpload = (*Stream)(nil) +var _ handler.TerminatableUpload = (*Stream)(nil) +var _ handler.LengthDeclarableUpload = (*Stream)(nil) + +// WriteChunk is called when client sends a chunk of raw data +func (w *Stream) WriteChunk(ctx context.Context, _ int64, src io.Reader) (int64, error) { + ctx, span := tracer.Start(ctx, "WriteChunk", trace.WithAttributes( + attribute.String("workspace", w.info.MetaData["workspace"]), + attribute.String("upload_id", w.info.ID), + )) + defer span.End() + + w.logger.Debug("Write Chunk start", zap.Int64("offset", w.info.Offset)) + data, err := io.ReadAll(src) + if err != nil { + return 0, err + } + // write into pipeline for transcoding + written, err := w.writer.Write(data) + if err != nil { + return int64(written), err + } + + n := int64(len(data)) + w.info.Offset += n + + if w.multipart != nil { + if writeErr := w.multipart.Write(ctx, data); writeErr != nil { + w.logger.Error("multipart upload part failed", zap.Error(writeErr)) + return n, writeErr + } + } + + w.logger.Debug("write chunk end", zap.Int64("offset", w.info.Offset), zap.Error(err)) + return n, nil +} + +// DeclareLength sets length of the video input +func (w *Stream) DeclareLength(ctx context.Context, length int64) error { + w.info.Size = length + w.info.SizeIsDeferred = false + w.logger.Debug("DeclareLength", zap.Int64("size", length), zap.Bool("SizeIsDeferred", w.info.SizeIsDeferred)) + return nil +} + +// GetInfo returns info about transcoing status +func (w *Stream) GetInfo(ctx context.Context) (handler.FileInfo, error) { + w.logger.Debug("GetInfo is executed") + return w.info, nil +} + +// GetReader returns Stream's bytes stream +func (w *Stream) GetReader(ctx context.Context) (io.ReadCloser, error) { + w.logger.Debug("GetReader is executed, creating current reader...") + return w.reader, nil +} + +// Terminate calls when upload has failed +func (w *Stream) Terminate(ctx context.Context) error { + ctx, span := tracer.Start(ctx, "Terminate", trace.WithAttributes( + attribute.String("workspace", w.info.MetaData["workspace"]), + attribute.String("upload_id", w.info.ID), + )) + defer span.End() + + w.logger.Debug("terminate upload") + + // Close the writer first to signal EOF to all readers + if err := w.writer.Close(); err != nil { + tracing.RecordError(span, err) + w.logger.Error("failed to close writer", zap.Error(err)) + return err + } + + var wg sync.WaitGroup + + // cancel multipart upload if in progress + if w.multipart != nil { + wg.Add(1) + go func() { + defer wg.Done() + if err := w.multipart.Terminate(ctx); err != nil { + w.logger.Error("multipart upload cancel failed", zap.Error(err)) + } + }() + } + + wg.Wait() + + // Signal that the stream is done + close(w.done) + + return nil +} + +// ConcatUploads calls when upload resumed after fail +func (w *Stream) ConcatUploads(ctx context.Context, partialUploads []handler.Upload) error { + w.logger.Debug("ConcatUploads was executed, it's not implemented") + // + // TODO: load raw source from the Buckup bucket, terminate all Streams with same ID and start process again. + // + return errors.New("not implemented") +} + +// FinishUpload calls when upload finished without errors on the client side +func (w *Stream) FinishUpload(ctx context.Context) error { + ctx, span := tracer.Start(ctx, "FinishUpload", trace.WithAttributes( + attribute.String("workspace", w.info.MetaData["workspace"]), + attribute.String("upload_id", w.info.ID), + )) + defer span.End() + + w.logger.Debug("finish upload") + + // Close the writer first to signal EOF to all readers + if err := w.writer.Close(); err != nil { + tracing.RecordError(span, err) + w.logger.Error("failed to close writer", zap.Error(err)) + return err + } + + var wg sync.WaitGroup + var completeErr error + + // finalize raw multipart stream if supported + if w.multipart != nil { + wg.Add(1) + go func() { + defer wg.Done() + if err := w.multipart.Complete(ctx); err != nil { + w.logger.Error("multipart upload complete failed", zap.Error(err)) + tracing.RecordError(span, err) + completeErr = err + return + } + }() + } + + wg.Wait() + + // Signal that the stream is done + close(w.done) + + return completeErr +} + +// AsConcatableUpload returns tusd handler.ConcatableUpload +func (s *StreamCoordinator) AsConcatableUpload(upload handler.Upload) handler.ConcatableUpload { + s.logger.Debug("AsConcatableUpload is executed") + return upload.(*Stream) +} + +func (w *Stream) start() error { + defer w.logger.Debug("start done") + w.reader = w.writer.Transpile() + return nil +} diff --git a/foundations/stream/internal/pkg/mediaconvert/transcoder.go b/foundations/stream/internal/pkg/mediaconvert/transcoder.go new file mode 100644 index 0000000000..c0b4f9bc0a --- /dev/null +++ b/foundations/stream/internal/pkg/mediaconvert/transcoder.go @@ -0,0 +1,248 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package mediaconvert + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/hcengineering/stream/internal/pkg/config" + "github.com/hcengineering/stream/internal/pkg/executor" + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/manifest" + "github.com/hcengineering/stream/internal/pkg/storage" + "github.com/hcengineering/stream/internal/pkg/token" + "github.com/hcengineering/stream/internal/pkg/tracing" + "github.com/hcengineering/stream/internal/pkg/uploader" + "github.com/pkg/errors" + "go.uber.org/zap" + "gopkg.in/vansante/go-ffprobe.v2" +) + +var transcodingDir = "t" + +// Transcoder process one transcoding task +type Transcoder struct { + ctx context.Context + cfg *config.Config + outputDir string + logger *zap.Logger +} + +// NewTranscoder creates a new instance of task transcoder +func NewTranscoder(ctx context.Context, cfg *config.Config) *Transcoder { + outputDir := filepath.Join(cfg.OutputDir, transcodingDir) + + var p = &Transcoder{ + cfg: cfg, + ctx: ctx, + outputDir: outputDir, + logger: log.FromContext(ctx).With(zap.String("transcoding", "transcoder")), + } + + return p +} + +// Transcode handles one transcoding task +func (p *Transcoder) Transcode(ctx context.Context, task *Task) (*TaskResult, error) { + var logger = p.logger.With(zap.String("task-id", task.ID)) + + logger.Debug("start") + defer logger.Debug("finished") + + logger.Debug("phase 1: get a token") + var tokenString, err = token.NewToken(p.cfg.ServerSecret, task.Workspace, "stream") + if err != nil { + logger.Error("can not create token", zap.Error(err)) + return nil, errors.Wrapf(err, "can not create token") + } + + logger.Debug("phase 2: preparing fs") + var destinationFolder = filepath.Join(p.outputDir, task.ID) + var _, filename = filepath.Split(task.Source) + err = os.MkdirAll(destinationFolder, os.ModePerm) + if err != nil { + logger.Error("can not create temporary folder", zap.Error(err)) + return nil, errors.Wrapf(err, "can not create temporary folder") + } + + defer func() { + logger.Debug("remove temporary folder") + if err = os.RemoveAll(destinationFolder); err != nil { + logger.Error("failed to cleanup temporary folder", zap.Error(err)) + } + }() + + logger.Debug("phase 3: get the remote file") + + remoteStorage, err := storage.NewStorageByURL(ctx, p.cfg.Endpoint(), p.cfg.EndpointURL.Scheme, tokenString, task.Workspace) + if err != nil { + logger.Error("can not create storage by url", zap.Error(err), zap.String("url", p.cfg.EndpointURL.String())) + return nil, errors.Wrapf(err, "can not create storage by url") + } + + stat, err := remoteStorage.StatFile(ctx, task.Source) + if err != nil { + logger.Error("can not stat file", zap.Error(err), zap.String("filepath", task.Source)) + return nil, errors.Wrapf(err, "can not stat file") + } + + if !IsSupportedMediaType(stat.Type) { + logger.Info("unsupported media type", zap.String("type", stat.Type)) + return nil, fmt.Errorf("unsupported media type: %s", stat.Type) + } + + sourceFilePath := filepath.Join(destinationFolder, filename) + if err = remoteStorage.GetFile(ctx, task.Source, sourceFilePath); err != nil { + logger.Error("can not download source file", zap.Error(err), zap.String("filepath", task.Source)) + // TODO: reschedule + return nil, errors.Wrapf(err, "can not download source file") + } + + logger.Debug("phase 4: prepare to transcode") + probe, err := tracing.WithSpanResult(ctx, tracer, "ffprobe", func(spanCtx context.Context) (*ffprobe.ProbeData, error) { + return ffprobe.ProbeURL(spanCtx, sourceFilePath) + }) + if err != nil { + logger.Error("can not get ffprobe", zap.Error(err), zap.String("filepath", sourceFilePath)) + return nil, errors.Wrapf(err, "can not get ffprobe") + } + + videoStream := probe.FirstVideoStream() + if videoStream == nil { + logger.Error("no video stream found", zap.String("filepath", sourceFilePath)) + return nil, fmt.Errorf("no video stream found") + } + + logger.Debug("video stream found", zap.String("codec", videoStream.CodecName), zap.Int("width", videoStream.Width), zap.Int("height", videoStream.Height)) + + audioStream := probe.FirstAudioStream() + if audioStream == nil { + logger.Info("no audio stream found", zap.String("filepath", sourceFilePath)) + } + + meta := VideoMeta{ + Width: videoStream.Width, + Height: videoStream.Height, + Codec: videoStream.CodecName, + ContentType: stat.Type, + } + + var profiles = DefaultTranscodingProfiles(meta) + + var opts = Options{ + Input: sourceFilePath, + OutputDir: p.outputDir, + LogLevel: LogLevel(p.cfg.LogLevel), + Profiles: profiles, + UploadID: task.ID, + Threads: p.cfg.MaxThreadCount, + } + + logger.Debug("phase 5: start async upload process") + var uploader = uploader.New(ctx, remoteStorage, uploader.Options{ + Dir: destinationFolder, + WorkerCount: uint32(opts.Threads), + BufferSize: 128, + RetryCount: 10, + RetryDelay: time.Millisecond * 100, + Timeout: p.cfg.Timeout, + Source: task.Source, + SourceFile: sourceFilePath, + }) + + err = manifest.GenerateHLSPlaylist(profiles, p.outputDir, opts.UploadID) + if err != nil { + logger.Error("can not generate hls playlist", zap.String("out", p.outputDir), zap.String("uploadID", opts.UploadID)) + return nil, errors.Wrapf(err, "can not generate hls playlist") + } + + go uploader.Start() + + logger.Debug("phase 6: start async transcode processes") + + var argsSlice = [][]string{ + BuildThumbnailCommand(&opts), + BuildVideoCommand(&opts), + } + + var cmds []*exec.Cmd + for _, args := range argsSlice { + if len(args) == 0 { + logger.Debug("skip empty command") + continue + } + + cmd, cmdErr := newFfmpegCommand(ctx, nil, args) + if cmdErr != nil { + logger.Error("can not create a new command", zap.Error(cmdErr), zap.Strings("args", args)) + go uploader.Cancel() + return nil, errors.Wrapf(cmdErr, "can not create a new command") + } + + cmds = append(cmds, cmd) + } + + execErr := tracing.WithSpan(ctx, tracer, "ffmpeg", func(ctx context.Context) error { + return executor.ExecuteCommands(ctx, cmds) + }) + if execErr != nil { + uploader.Cancel() + return nil, errors.Wrapf(execErr, "can not execute command") + } + + logger.Debug("phase 7: schedule cleanup") + uploader.Stop() + + logger.Debug("phase 8: try to set metadata") + + var result = TaskResult{ + Width: videoStream.Width, + Height: videoStream.Height, + Playlist: manifest.MasterPlaylistFileName(task.ID), + Thumbnail: manifest.ThumbnailFileName(task.ID), + } + + if metaProvider, ok := remoteStorage.(storage.MetaProvider); ok { + logger.Debug( + "applying metadata", + zap.String("url", result.Playlist), + zap.String("thumbnail", result.Thumbnail), + zap.String("source", task.Source), + ) + metaErr := metaProvider.PatchMeta( + ctx, + task.Source, + &storage.Metadata{ + "hls": map[string]any{ + "source": result.Playlist, + "thumbnail": result.Thumbnail, + }, + "width": result.Width, + "height": result.Height, + }, + ) + if metaErr != nil { + logger.Error("can not patch the source file", zap.Error(metaErr)) + } + } + + return &result, nil +} diff --git a/foundations/stream/internal/pkg/pprof/pprof.go b/foundations/stream/internal/pkg/pprof/pprof.go new file mode 100644 index 0000000000..081e9603df --- /dev/null +++ b/foundations/stream/internal/pkg/pprof/pprof.go @@ -0,0 +1,53 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package pprof provides all-in configured pprof server for debugging purposes +package pprof + +import ( + "context" + "net/http" + "net/http/pprof" + "time" + + "github.com/hcengineering/stream/internal/pkg/log" + "go.uber.org/zap" +) + +// ListenAndServe - configures pprof http handlers +func ListenAndServe(ctx context.Context, listenOn string) { + log.FromContext(ctx).Debug("Profiler is enabled", zap.String("listening on", listenOn)) + mux := http.NewServeMux() + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + mux.Handle("/debug/pprof/allocs", pprof.Handler("allocs")) + mux.Handle("/debug/pprof/block", pprof.Handler("block")) + mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine")) + mux.Handle("/debug/pprof/heap", pprof.Handler("heap")) + mux.Handle("/debug/pprof/mutex", pprof.Handler("mutex")) + mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate")) + server := &http.Server{ + Addr: listenOn, + Handler: mux, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + if err := server.ListenAndServe(); err != nil { + log.FromContext(ctx).Debug("Failed to start profiler", zap.Error(err)) + } + <-ctx.Done() + _ = server.Close() +} diff --git a/foundations/stream/internal/pkg/profile/profile.go b/foundations/stream/internal/pkg/profile/profile.go new file mode 100644 index 0000000000..c84a3437ef --- /dev/null +++ b/foundations/stream/internal/pkg/profile/profile.go @@ -0,0 +1,186 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package profile provides video profiles +package profile + +import ( + "fmt" + + "github.com/hcengineering/stream/internal/pkg/resconv" +) + +// VideoProfile represents a video profile +type VideoProfile struct { + Name string + Width int // 0 means keep original + Height int // 0 means keep original + Bandwidth int + Scale bool + + // Codec settings + VideoCodec string + AudioCodec string + + // Advanced encoding settings + CRF int // Constant Rate Factor (0-51 for x264) +} + +// profileOriginal is a profile for original video without transcoding +var profileOriginal = VideoProfile{ + Name: "orig", + Scale: false, + VideoCodec: "copy", + AudioCodec: "copy", + CRF: 23, +} + +// profileOriginalT is a profile for transcoding video in the original resolution +var profileOriginalT = VideoProfile{ + Name: "orig", + Scale: false, + VideoCodec: "libx264", + AudioCodec: "aac", + CRF: 23, +} + +// Profile360p is a profile for transcoding video in 360p +var Profile360p = VideoProfile{ + Name: "360p", + Scale: true, + Width: 640, + Height: 360, + Bandwidth: 500000, + VideoCodec: "libx264", + AudioCodec: "aac", + CRF: 28, +} + +// Profile480p is a profile for transcoding video in 480p +var Profile480p = VideoProfile{ + Name: "480p", + Scale: true, + Width: 854, + Height: 480, + Bandwidth: 2000000, + VideoCodec: "libx264", + AudioCodec: "aac", + CRF: 27, +} + +// Profile720p is a profile for transcoding video in 720p +var Profile720p = VideoProfile{ + Name: "720p", + Scale: true, + Width: 1280, + Height: 720, + Bandwidth: 5000000, + VideoCodec: "libx264", + AudioCodec: "aac", + CRF: 25, +} + +// Profile1080p is a profile for transcoding video in 1080p +var Profile1080p = VideoProfile{ + Name: "1080p", + Scale: true, + Width: 1920, + Height: 1080, + Bandwidth: 8000000, + VideoCodec: "libx264", + AudioCodec: "aac", + CRF: 23, +} + +// Profile1440p is a profile for transcoding video in 1440p +var Profile1440p = VideoProfile{ + Name: "1440p", + Scale: true, + Width: 2560, + Height: 1440, + Bandwidth: 12000000, + VideoCodec: "libx264", + AudioCodec: "aac", + CRF: 23, +} + +// Profile2160p is a profile for transcoding video in 2160p +var Profile2160p = VideoProfile{ + Name: "2160p", + Scale: true, + Width: 3840, + Height: 2160, + Bandwidth: 25000000, + VideoCodec: "libx264", // Consider libx265 + AudioCodec: "aac", + CRF: 22, +} + +// Profile4320p is a profile for transcoding video in 360p +var Profile4320p = VideoProfile{ + Name: "4320p", + Scale: true, + Width: 7680, + Height: 4320, + Bandwidth: 50000000, + VideoCodec: "libx264", // Consider libx265 + AudioCodec: "aac", + CRF: 22, +} + +// Profiles is a map of all supported profiles +var Profiles = map[string]VideoProfile{ + "360p": Profile360p, + "480p": Profile480p, + "720p": Profile720p, + "1080p": Profile1080p, + "1440p": Profile1440p, + "2160p": Profile2160p, + "4320p": Profile4320p, +} + +// MakeProfileOriginal creates a profile for original video without transcoding +func MakeProfileOriginal(width, height int) VideoProfile { + resolution := fmt.Sprintf("%v:%v", width, height) + level := resconv.Level(resolution) + bandwidth := resconv.Bandwidth(level) + + profile := profileOriginal + profile.Width = width + profile.Height = height + profile.Bandwidth = bandwidth + + return profile +} + +// MakeProfileOriginalT creates a profile for transcoding video in the original resolution +func MakeProfileOriginalT(width, height int) VideoProfile { + resolution := fmt.Sprintf("%v:%v", width, height) + level := resconv.Level(resolution) + bandwidth := resconv.Bandwidth(level) + + profile := profileOriginalT + profile.Width = width + profile.Height = height + profile.Bandwidth = bandwidth + + return profile +} + +// GetProfileByName returns a VideoProfile by name +func GetProfileByName(name string) (VideoProfile, bool) { + profile, ok := Profiles[name] + return profile, ok +} diff --git a/foundations/stream/internal/pkg/profile/profile_test.go b/foundations/stream/internal/pkg/profile/profile_test.go new file mode 100644 index 0000000000..1ad67d7959 --- /dev/null +++ b/foundations/stream/internal/pkg/profile/profile_test.go @@ -0,0 +1,177 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package profile_test + +import ( + "testing" + + "github.com/hcengineering/stream/internal/pkg/profile" + "github.com/stretchr/testify/assert" +) + +func TestGetProfileByName(t *testing.T) { + tests := []struct { + name string + expected profile.VideoProfile + }{ + { + name: "360p", + expected: profile.Profile360p, + }, + { + name: "480p", + expected: profile.Profile480p, + }, + { + name: "720p", + expected: profile.Profile720p, + }, + { + name: "1080p", + expected: profile.Profile1080p, + }, + { + name: "1440p", + expected: profile.Profile1440p, + }, + { + name: "2160p", + expected: profile.Profile2160p, + }, + { + name: "4320p", + expected: profile.Profile4320p, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + profile, ok := profile.GetProfileByName(tt.name) + assert.True(t, ok) + assert.Equal(t, tt.expected, profile) + }) + } +} + +func TestGetProfileByName_Failure(t *testing.T) { + tests := []struct { + name string + }{ + { + name: "foo", + }, + { + name: "original", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, ok := profile.GetProfileByName(tt.name) + assert.False(t, ok) + }) + } +} + +func TestMakeProfileOriginal(t *testing.T) { + tests := []struct { + name string + width int + height int + expected profile.VideoProfile + }{ + { + name: "480p", + width: 640, + height: 480, + expected: profile.VideoProfile{ + Name: "orig", + VideoCodec: "copy", + AudioCodec: "copy", + Width: 640, + Height: 480, + Bandwidth: 2000000, + CRF: 23, + }, + }, + { + name: "1440p", + width: 2000, + height: 1200, + expected: profile.VideoProfile{ + Name: "orig", + VideoCodec: "copy", + AudioCodec: "copy", + Width: 2000, + Height: 1200, + Bandwidth: 8000000, + CRF: 23, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + profile := profile.MakeProfileOriginal(tt.width, tt.height) + assert.Equal(t, tt.expected, profile) + }) + } +} + +func TestMakeProfileOriginalT(t *testing.T) { + tests := []struct { + name string + width int + height int + expected profile.VideoProfile + }{ + { + name: "720p", + width: 1280, + height: 720, + expected: profile.VideoProfile{ + Name: "orig", + VideoCodec: "libx264", + AudioCodec: "aac", + Width: 1280, + Height: 720, + Bandwidth: 5000000, + CRF: 23, + }, + }, + { + name: "2160p", + width: 3840, + height: 2160, + expected: profile.VideoProfile{ + Name: "orig", + VideoCodec: "libx264", + AudioCodec: "aac", + Width: 3840, + Height: 2160, + Bandwidth: 25000000, + CRF: 23, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + profile := profile.MakeProfileOriginalT(tt.width, tt.height) + assert.Equal(t, tt.expected, profile) + }) + } +} diff --git a/foundations/stream/internal/pkg/queue/config.go b/foundations/stream/internal/pkg/queue/config.go new file mode 100644 index 0000000000..193f9eb140 --- /dev/null +++ b/foundations/stream/internal/pkg/queue/config.go @@ -0,0 +1,44 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package queue + +import "strings" + +// Config reprents queue config +type Config struct { + Postfix string + Brokers []string + ClientID string + Region string +} + +// ParseConfig parses the queue config string +func ParseConfig(config, clientID, region string) Config { + var parts = strings.Split(config, ";") + var brokers = strings.Split(parts[0], ",") + var postfix = "" + + if len(parts) > 1 { + postfix = parts[1] + } + + var cfg = Config{ + Postfix: postfix, + Brokers: brokers, + ClientID: clientID, + Region: region, + } + + return cfg +} diff --git a/foundations/stream/internal/pkg/queue/config_test.go b/foundations/stream/internal/pkg/queue/config_test.go new file mode 100644 index 0000000000..b4772780c5 --- /dev/null +++ b/foundations/stream/internal/pkg/queue/config_test.go @@ -0,0 +1,90 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package queue_test + +import ( + "reflect" + "testing" + + "github.com/hcengineering/stream/internal/pkg/queue" +) + +func Test_ParseConfig(t *testing.T) { + tests := []struct { + name string + config string + clientID string + region string + want queue.Config + }{ + { + name: "Basic broker configuration", + config: "broker1:9092,broker2:9092", + clientID: "test-client", + region: "us-west", + want: queue.Config{ + Postfix: "", + Brokers: []string{"broker1:9092", "broker2:9092"}, + ClientID: "test-client", + Region: "us-west", + }, + }, + { + name: "Broker configuration with postfix", + config: "broker1:9092,broker2:9092;custom-postfix", + clientID: "test-client", + region: "eu-central", + want: queue.Config{ + Postfix: "custom-postfix", + Brokers: []string{"broker1:9092", "broker2:9092"}, + ClientID: "test-client", + Region: "eu-central", + }, + }, + { + name: "Single broker configuration", + config: "broker1:9092", + clientID: "single-broker-client", + region: "asia-east", + want: queue.Config{ + Postfix: "", + Brokers: []string{"broker1:9092"}, + ClientID: "single-broker-client", + Region: "asia-east", + }, + }, + { + name: "Multiple semicolons in config", + config: "broker1:9092,broker2:9092;postfix;extra", + clientID: "test-client", + region: "us-east", + want: queue.Config{ + Postfix: "postfix", + Brokers: []string{"broker1:9092", "broker2:9092"}, + ClientID: "test-client", + Region: "us-east", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := queue.ParseConfig(tt.config, tt.clientID, tt.region) + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseConfig() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/foundations/stream/internal/pkg/queue/queue.go b/foundations/stream/internal/pkg/queue/queue.go new file mode 100644 index 0000000000..b79786f67d --- /dev/null +++ b/foundations/stream/internal/pkg/queue/queue.go @@ -0,0 +1,214 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package queue provides access to kafka queue +package queue + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/segmentio/kafka-go" + "go.uber.org/zap" +) + +// TopicTranscodeRequest is the name of the transcode requests topic. +const TopicTranscodeRequest = "stream.transcode.request" + +// TopicTranscodeResult is the name of the transcode results topic. +const TopicTranscodeResult = "stream.transcode.result" + +// TranscodeRequest represents transcode request. +type TranscodeRequest struct { + BlobID string + WorkspaceUUID string + ContentType string + Source map[string]any +} + +// TranscodeResult represents transcode result. +type TranscodeResult struct { + BlobID string `json:"blobId"` + WorkspaceUUID string `json:"workspaceUuid"` + Source map[string]any `json:"source"` + Playlist string `json:"playlist"` + Thumbnail string `json:"thumbnail"` +} + +// ConsumerOptions represents options for the consumer +type ConsumerOptions struct { + Topic string + Group string + Config Config +} + +// Consumer provides a consumer interface to a Kafka queue +type Consumer interface { + Read(ctx context.Context) (kafka.Message, error) + Fetch(ctx context.Context) (kafka.Message, error) + Commit(ctx context.Context, msg kafka.Message) error + Close() error +} + +// ProducerOptions represents options for the producer +type ProducerOptions struct { + Topic string + Group string + Config Config + RetryCount int +} + +// Producer provides a producer interface to a Kafka queue +type Producer interface { + Send(ctx context.Context, workspaceID string, data any) error + Close() error +} + +// TConsumer implements Consumer interface +type TConsumer struct { + topic string + reader *kafka.Reader + options ConsumerOptions +} + +// TProducer implements Producer interface +type TProducer struct { + topic string + writer *kafka.Writer + options ProducerOptions +} + +// Logger is kafka.Logger implementation +type Logger struct { + logger *zap.Logger +} + +var _ Producer = (*TProducer)(nil) +var _ Consumer = (*TConsumer)(nil) +var _ kafka.Logger = (*Logger)(nil) + +// NewLogger creates new kafka.Logger +func NewLogger(ctx context.Context) kafka.Logger { + return &Logger{ + logger: log.FromContext(ctx), + } +} + +// Printf logs a message +func (l *Logger) Printf(msg string, args ...any) { + l.logger.Sugar().Debugf(msg, args...) +} + +// NewConsumer creates a new transcoding request consumer +func NewConsumer(ctx context.Context, options ConsumerOptions) Consumer { + platformTopic := makeTopicID(options.Topic, options.Config) + groupID := makeGroupID(options.Group, options.Topic, options.Config) + + reader := kafka.NewReader(kafka.ReaderConfig{ + Brokers: options.Config.Brokers, + GroupID: groupID, + Topic: platformTopic, + Logger: NewLogger(ctx), + HeartbeatInterval: 1 * time.Second, + SessionTimeout: 10 * time.Second, + }) + + return &TConsumer{ + options: options, + topic: platformTopic, + reader: reader, + } +} + +// Read reads (fetch and commit) a queue message. +func (c *TConsumer) Read(ctx context.Context) (kafka.Message, error) { + return c.reader.ReadMessage(ctx) +} + +// Fetch fetches a queue message. +func (c *TConsumer) Fetch(ctx context.Context) (kafka.Message, error) { + return c.reader.FetchMessage(ctx) +} + +// Commit commits the queue message. +func (c *TConsumer) Commit(ctx context.Context, msg kafka.Message) error { + return c.reader.CommitMessages(ctx, msg) +} + +// Close closes the consumer. +func (c *TConsumer) Close() error { + return c.reader.Close() +} + +// NewProducer creates a new transcoding result producer +func NewProducer(ctx context.Context, options ProducerOptions) Producer { + platformTopic := makeTopicID(options.Topic, options.Config) + writer := &kafka.Writer{ + Addr: kafka.TCP(options.Config.Brokers...), + Topic: platformTopic, + AllowAutoTopicCreation: true, + Logger: NewLogger(ctx), + } + return &TProducer{ + options: options, + topic: platformTopic, + writer: writer, + } +} + +// Send sends a message to the queue topic +func (p *TProducer) Send(ctx context.Context, workspaceID string, data any) error { + if data == nil { + return fmt.Errorf("event data is empty") + } + + value, err := json.Marshal(data) + + if err != nil { + return err + } + + for range p.options.RetryCount { + err = p.writer.WriteMessages( + ctx, + kafka.Message{ + Key: []byte(workspaceID), + Value: value, + }, + ) + if err == nil { + break + } + } + + return err +} + +// Close closes the producer. +func (p *TProducer) Close() error { + return p.writer.Close() +} + +func makeGroupID(groupID, topic string, config Config) string { + return makeTopicID(topic, config) + "-" + groupID +} + +func makeTopicID(topic string, config Config) string { + if config.Region != "" { + return config.Region + "." + topic + config.Postfix + } + return topic + config.Postfix +} diff --git a/foundations/stream/internal/pkg/queue/worker.go b/foundations/stream/internal/pkg/queue/worker.go new file mode 100644 index 0000000000..4fc0802f0b --- /dev/null +++ b/foundations/stream/internal/pkg/queue/worker.go @@ -0,0 +1,134 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package queue + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/hcengineering/stream/internal/pkg/config" + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/mediaconvert" + "github.com/hcengineering/stream/internal/pkg/tracing" + "github.com/pkg/errors" + "github.com/segmentio/kafka-go" + "go.opentelemetry.io/otel" + "go.uber.org/zap" +) + +var tracer = otel.Tracer("worker") + +// Worker is a queue processing worker +type Worker struct { + logger *zap.Logger + cfg *config.Config + consumer Consumer + producer Producer +} + +// NewWorker creates new worker +func NewWorker(ctx context.Context, consumer Consumer, producer Producer, cfg *config.Config) *Worker { + logger := log.FromContext(ctx).With(zap.String("queue", "worker")) + + return &Worker{ + logger: logger, + cfg: cfg, + consumer: consumer, + producer: producer, + } +} + +// Start queue processing +func (w *Worker) Start(ctx context.Context) error { + w.logger.Info("starting worker") + defer w.logger.Info("worker stopped") + + for { + select { + case <-ctx.Done(): + w.logger.Info("worker shutdown requested") + return ctx.Err() + default: + if err := w.fetchAndProcessMessage(ctx); err != nil { + if errors.Is(err, context.Canceled) { + return err + } + w.logger.Error("failed to process message", zap.Error(err)) + } + } + } +} + +func (w *Worker) fetchAndProcessMessage(ctx context.Context) error { + // TODO We should use Fetch and Commit here but as long processing + // time is longer than the max poll interval, we have to use Read. + msg, err := w.consumer.Read(ctx) + if err != nil { + return fmt.Errorf("read message: %w", err) + } + + logger := w.logger.With( + zap.String("message-key", string(msg.Key)), + zap.Int64("message-offset", msg.Offset), + zap.Int("partition", msg.Partition), + ) + logger.Debug("message", zap.ByteString("message", msg.Value)) + + err = w.processMessage(ctx, msg, logger) + if err != nil { + w.logger.Error("failed to process message", zap.Error(err)) + return fmt.Errorf("process message: %w", err) + } + + return nil +} + +func (w *Worker) processMessage(ctx context.Context, msg kafka.Message, logger *zap.Logger) error { + var req TranscodeRequest + if err := json.Unmarshal(msg.Value, &req); err != nil { + return fmt.Errorf("parse failed: %w", err) + } + + if !mediaconvert.IsSupportedMediaType(req.ContentType) { + logger.Debug("unsupported content type", zap.String("contentType", req.ContentType)) + return nil + } + + task := mediaconvert.Task{ + ID: req.BlobID, + Workspace: req.WorkspaceUUID, + Source: req.BlobID, + } + + transcoder := mediaconvert.NewTranscoder(ctx, w.cfg) + res, err := tracing.WithSpanResult(ctx, tracer, "transcode", func(spanCtx context.Context) (*mediaconvert.TaskResult, error) { + return transcoder.Transcode(spanCtx, &task) + }) + + if res != nil { + result := TranscodeResult{ + BlobID: req.BlobID, + WorkspaceUUID: req.WorkspaceUUID, + Source: req.Source, + Playlist: res.Playlist, + Thumbnail: res.Thumbnail, + } + if err = w.producer.Send(ctx, req.WorkspaceUUID, result); err != nil { + logger.Error("failed to send transcode result", zap.Error(err)) + } + } + + return err +} diff --git a/foundations/stream/internal/pkg/resconv/resconv.go b/foundations/stream/internal/pkg/resconv/resconv.go new file mode 100644 index 0000000000..da9b855538 --- /dev/null +++ b/foundations/stream/internal/pkg/resconv/resconv.go @@ -0,0 +1,139 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package resconv implements conversions to and from string representations of video resolutions. +package resconv + +import ( + "sort" + "strconv" + "strings" +) + +const defaultLevel = "360p" + +var prefixes = []struct { + pixels int + width int + height int + label string +}{ + {pixels: 640 * 360, width: 640, height: 360, label: "360p"}, + {pixels: 1280 * 720, width: 720, height: 720, label: "480p"}, + {pixels: 1920 * 1080, width: 1920, height: 1080, label: "720p"}, + {pixels: 2560 * 1440, width: 2560, height: 1440, label: "1080p"}, + {pixels: 3840 * 2160, width: 3840, height: 2160, label: "1440p"}, + {pixels: 7680 * 4320, width: 7680, height: 4320, label: "2160p"}, +} + +var bandwidthMap = map[string]int{ + "360p": 500000, + "480p": 2000000, + "720p": 5000000, + "1080p": 8000000, + "1440p": 12000000, + "2160p": 25000000, + "4320p": 50000000, +} + +var resolutions = map[string]string{ + "360p": "640:360", + "480p": "640:480", + "720p": "1280:720", + "1080p": "1920:1080", + "1440p": "2560:1440", + "2160p": "3840:2160", + "4320p": "7680:4320", +} + +// SubLevels returns sublevels for the resolution +func SubLevels(resolution string) (res []string) { + var height = Height(resolution) + var idx = sort.Search(len(prefixes), func(i int) bool { + return height < prefixes[i].height + }) + if idx < 2 { + return res + } + + idx-- + idx = min(idx, 3) + + for idx >= 1 { + res = append(res, prefixes[idx].label) + idx-- + if len(res) == 2 { + break + } + } + + return res +} + +// Resolution returns default resolution based on the level +func Resolution(level string) string { + if res, ok := resolutions[level]; ok { + return res + } + return Resolution(defaultLevel) +} + +// Level converts the resolution to short prefix +func Level(resolution string) string { + var height = Height(resolution) + idx := sort.Search(len(prefixes), func(i int) bool { + return height < prefixes[i].height + }) + if idx == len(prefixes) { + return "4320p" + } + + return prefixes[idx].label +} + +// Height returns height for the resolution +func Height(resolution string) int { + var parts = strings.Split(resolution, ":") + var h = 240 + if len(parts) > 1 { + var _h, _ = strconv.Atoi(parts[1]) + h = max(h, _h) + } + return h +} + +// Pixels returns amount of pixels for the resolution +func Pixels(resolution string) int { + var parts = strings.Split(resolution, ":") + var w, h = 320, 240 + + if len(parts) > 1 { + var _w, _ = strconv.Atoi(parts[0]) + var _h, _ = strconv.Atoi(parts[1]) + w = max(w, _w) + h = max(h, _h) + } + + return w * h +} + +// Bandwidth returns default bandwidth for the resolution +func Bandwidth(resolution string) int { + if v, ok := bandwidthMap[resolution]; ok { + return v + } + + return bandwidthMap[defaultLevel] +} diff --git a/foundations/stream/internal/pkg/resconv/resconv_test.go b/foundations/stream/internal/pkg/resconv/resconv_test.go new file mode 100644 index 0000000000..e2b187ef01 --- /dev/null +++ b/foundations/stream/internal/pkg/resconv/resconv_test.go @@ -0,0 +1,170 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package resconv_test + +import ( + "testing" + + "github.com/hcengineering/stream/internal/pkg/resconv" + "github.com/stretchr/testify/require" +) + +func Test_Resconv_Level(t *testing.T) { + tests := []struct { + res string + expected string + }{ + {res: "320:240", expected: "360p"}, + {res: "320:360", expected: "480p"}, + {res: "640:480", expected: "480p"}, + {res: "1280:720", expected: "720p"}, + {res: "1920:1080", expected: "1080p"}, + {res: "1600:1080", expected: "1080p"}, + {res: "2160:1080", expected: "1080p"}, + {res: "2880:1800", expected: "1440p"}, + {res: "2560:1440", expected: "1440p"}, + {res: "3840:2160", expected: "2160p"}, + {res: "5120:2880", expected: "2160p"}, + {res: "480:720", expected: "720p"}, + {res: "720:1080", expected: "1080p"}, + {res: "1440:2560", expected: "2160p"}, + {res: "9000:4000", expected: "2160p"}, + } + + for _, tt := range tests { + t.Run(tt.expected, func(t *testing.T) { + result := resconv.Level(tt.res) + require.Equal(t, tt.expected, result, "ResolutionFromPixels(%d)", tt.res) + }) + } +} + +func Test_Resconv_SubLevels(t *testing.T) { + tests := []struct { + name string + res string + expected []string + }{ + { + name: "pixels below smallest resolution", + res: "640:479", + expected: nil, + }, + { + name: "pixels equal to smallest resolution", + res: "640:480", + expected: nil, + }, + { + name: "pixels just above smallest resolution", + res: "641:480", + expected: nil, + }, + { + name: "pixels equal to 720p", + res: "1280:720", + expected: []string{"480p"}, + }, + { + name: "pixels just above 480p", + res: "1280:721", + expected: []string{"480p"}, + }, + { + name: "pixels equal to 1k", + res: "1920:1080", + expected: []string{"720p", "480p"}, + }, + { + name: "pixels just above 1k", + res: "1920:1081", + expected: []string{"720p", "480p"}, + }, + { + name: "pixels equal to 1k", + res: "2560:1440", + expected: []string{"1080p", "720p"}, + }, + { + name: "pixels between 1k and 2k", + res: "2880:1800", + expected: []string{"1080p", "720p"}, + }, + { + name: "pixels equal to 2k", + res: "3840:2160", + expected: []string{"1080p", "720p"}, + }, + { + name: "pixels equal to 4k", + res: "5120:2160", + expected: []string{"1080p", "720p"}, + }, + { + name: "pixels equal to 5k", + res: "7680:4320", + expected: []string{"1080p", "720p"}, + }, + { + name: "pixels above largest resolution", + res: "7681:4320", + expected: []string{"1080p", "720p"}, + }, + { + name: "vertical video", + res: "1440:2560", + expected: []string{"1080p", "720p"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := resconv.SubLevels(tt.res) + require.Equal(t, tt.expected, result, "SubResolutionsFromPixels(%d) returned unexpected result", tt.res) + }) + } +} + +func Test_Resconv_Height(t *testing.T) { + tests := []struct { + res string + expected int + }{ + // 3x2 + {res: "320:240", expected: 240}, + {res: "640:480", expected: 480}, + {res: "1280:720", expected: 720}, + // 16x9 + {res: "1920:1080", expected: 1080}, + {res: "2560:1440", expected: 1440}, + {res: "3840:2160", expected: 2160}, + // Vertical + {res: "480:720", expected: 720}, + {res: "720:1080", expected: 1080}, + {res: "1440:2560", expected: 2560}, + // Custom + {res: "1600:1080", expected: 1080}, + {res: "1930:1090", expected: 1090}, + {res: "2160:1080", expected: 1080}, + {res: "5120:2880", expected: 2880}, + {res: "9000:4000", expected: 4000}, + } + + for _, tt := range tests { + t.Run(tt.res, func(t *testing.T) { + height := resconv.Height(tt.res) + require.Equal(t, tt.expected, height, "Height(%s)", tt.res) + }) + } +} diff --git a/foundations/stream/internal/pkg/sharedpipe/shared_pipe.go b/foundations/stream/internal/pkg/sharedpipe/shared_pipe.go new file mode 100644 index 0000000000..0a698b46c8 --- /dev/null +++ b/foundations/stream/internal/pkg/sharedpipe/shared_pipe.go @@ -0,0 +1,116 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sharedpipe provided a shared pipe that can be used when one writer can be shared between a couple of readers. +package sharedpipe + +import ( + "io" + "sync" + "sync/atomic" +) + +// Chunk represents a chunk of raw data for the readers +type Chunk struct { + content []byte + Next atomic.Pointer[Chunk] + ready chan struct{} + done chan struct{} +} + +// NewWriter creates a new shared pipe Writer +func NewWriter() *Writer { + var res = &Writer{ + done: make(chan struct{}), + } + res.curr.Store(&Chunk{done: res.done, ready: make(chan struct{})}) + return res +} + +// Transpile creates a new Reader +func (w *Writer) Transpile() *Reader { + var res = &Reader{ + curr: w.curr.Load(), + done: make(chan struct{}), + } + return res +} + +// Writer represents a shared pipe writer +type Writer struct { + curr atomic.Pointer[Chunk] + done chan struct{} + once sync.Once +} + +// Close closes the pipe for all readers +func (w *Writer) Close() error { + w.once.Do(func() { close(w.done) }) + return nil +} + +func (w *Writer) Write(p []byte) (n int, err error) { + var completePrevious = w.curr.Load().ready + var curr = w.curr.Load() + curr.Next.Store(&Chunk{content: p, ready: make(chan struct{}), done: w.done}) + w.curr.Store(curr.Next.Load()) + close(completePrevious) + return len(p), nil +} + +// Reader is reader from shared pipe, imlements io.Reader interface +type Reader struct { + curr *Chunk + once sync.Once + done chan struct{} + pos int +} + +// Close closes reader stream +func (s *Reader) Close() error { + s.once.Do(func() { + close(s.done) + }) + return nil +} + +func (s *Reader) Read(in []byte) (n int, err error) { + var curr = s.curr + for i := 0; i < len(in); { + for s.pos >= len(curr.content) { + select { + case <-curr.done: + curr = curr.Next.Load() + if curr == nil { + _ = s.Close() + return i, io.EOF + } + case <-s.done: + return i, io.ErrClosedPipe + case <-curr.ready: + curr = curr.Next.Load() + } + s.pos = 0 + s.curr = curr + } + var n = copy(in[i:], curr.content[s.pos:]) + s.pos += n + i += n + } + return len(in), nil +} + +var _ io.Closer = (*Writer)(nil) +var _ io.Closer = (*Reader)(nil) +var _ io.Reader = (*Reader)(nil) +var _ io.Writer = (*Writer)(nil) diff --git a/foundations/stream/internal/pkg/sharedpipe/shared_pipe_bench_test.go b/foundations/stream/internal/pkg/sharedpipe/shared_pipe_bench_test.go new file mode 100644 index 0000000000..cafe282dc0 --- /dev/null +++ b/foundations/stream/internal/pkg/sharedpipe/shared_pipe_bench_test.go @@ -0,0 +1,246 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package sharedpipe + +import ( + "bytes" + "io" + "math/rand" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const sendMessageSize = 8 * 1000 * 1000 +const readerCount = 10 + +func TestStability(t *testing.T) { + for range 10 { + testStability(t) + } +} + +// #nosec +func testStability(t *testing.T) { + var writer = NewWriter() + var readers []io.Reader + + for range 1000 { + readers = append(readers, writer.Transpile()) + } + var buff [4]byte + for range rand.Intn(1000) { + _, _ = writer.Write([]byte("ping")) + for i := range rand.Intn(10) { + _, _ = readers[i].Read(buff[:]) + } + } + _ = writer.Close() + for _, r := range readers { + _, err := io.ReadAll(r) + require.NoError(t, err) + } +} + +func TestBasicWriteRead(t *testing.T) { + writer := NewWriter() + defer func() { _ = writer.Close() }() + reader := writer.Transpile() + + data := []byte("Hello, World!") + n, err := writer.Write(data) + if err != nil { + t.Fatalf("Unexpected error on Write: %v", err) + } + if n != len(data) { + t.Fatalf("Expected to write %d bytes, wrote %d", len(data), n) + } + + readBuf := make([]byte, len(data)) + n, err = reader.Read(readBuf) + if err != nil && err != io.EOF { + t.Fatalf("Unexpected error on Read: %v", err) + } + if !bytes.Equal(readBuf[:n], data) { + t.Fatalf("Expected to read %q, got %q", data, readBuf[:n]) + } +} + +func TestConcurrentWriteRead(t *testing.T) { + writer := NewWriter() + defer func() { _ = writer.Close() }() + reader := writer.Transpile() + + var wg sync.WaitGroup + data := []byte("Hello, Concurrent World!") + readBuf := make([]byte, len(data)) + + wg.Add(2) + + go func() { + defer wg.Done() + _, err := writer.Write(data) + if err != nil { + t.Errorf("Unexpected error on Write: %v", err) + } + }() + + // Reader goroutine + go func() { + defer wg.Done() + n, err := reader.Read(readBuf) + if err != nil && err != io.EOF { + t.Errorf("Unexpected error on Read: %v", err) + } + if n != len(data) { + t.Errorf("Expected to read %d bytes, read %d", len(data), n) + } + if !bytes.Equal(readBuf[:n], data) { + t.Errorf("Expected to read %q, got %q", data, readBuf[:n]) + } + }() + + wg.Wait() +} + +func TestWriterClose(t *testing.T) { + writer := NewWriter() + reader := writer.Transpile() + + _, _ = writer.Write([]byte("Hello")) + _ = writer.Close() + + readBuf := make([]byte, 5) + n, err := reader.Read(readBuf) + if err != nil && err != io.EOF { + t.Fatalf("Unexpected error on Read: %v", err) + } + if n != 5 { + t.Fatalf("Expected to read 5 bytes, read %d", n) + } + + n, err = reader.Read(readBuf) + if err != io.EOF { + t.Fatalf("Expected EOF after writer close, got %v", err) + } + if n != 0 { + t.Fatalf("Expected to read 0 bytes after EOF, read %d", n) + } +} + +// Test reading from an empty writer. +func TestReadFromEmptyWriter(t *testing.T) { + writer := NewWriter() + _ = writer.Close() + reader := writer.Transpile() + + readBuf := make([]byte, 5) + _, err := reader.Read(readBuf) + require.ErrorIs(t, io.EOF, err) +} + +func Test_PipeWait(t *testing.T) { + var writer = NewWriter() + var reader = writer.Transpile() + var buff [4]byte + go func() { + time.Sleep(time.Second / 10) + _, _ = writer.Write([]byte("test")) + }() + _, _ = reader.Read(buff[:]) + require.Equal(t, "test", string(buff[:])) +} + +func Test_Consistent(t *testing.T) { + var writer = NewWriter() + var readers []io.Reader + + for range readerCount { + readers = append(readers, writer.Transpile()) + } + + _, _ = writer.Write([]byte("Hello")) + _, _ = writer.Write([]byte(" ")) + _, _ = writer.Write([]byte("World!")) + _ = writer.Close() + + var res strings.Builder + + for i := range readerCount { + for { + var buffer = make([]byte, 2) + _, err := readers[i].Read(buffer) + if err == io.EOF { + break + } + _, _ = res.WriteString(string(buffer)) + } + require.Equal(t, "Hello World!", res.String()) + res.Reset() + } +} + +// Benchmark_DefaultPipe-8 (4 b) 61956 19177 ns/op 48 B/op 1 allocs/op +// Benchmark_DefaultPipe-8 (8 mb) 24 48471316 ns/op 257 B/op 1 allocs/op +func Benchmark_DefaultPipe(b *testing.B) { + var data [sendMessageSize]byte + var buffer = make([]byte, len(data)) + var readers []io.Reader + var writers []io.Writer + + for range readerCount { + r, w := io.Pipe() + readers = append(readers, r) + writers = append(writers, w) + } + + b.ReportAllocs() + + for b.Loop() { + go func() { + for i := range readerCount { + _, _ = writers[i].Write(data[:]) + } + }() + for i := range readerCount { + _, _ = readers[i].Read(buffer) + } + } +} + +// Benchmark_SharedPipe-8 (4 b) 161847 8131 ns/op 160 B/op 2 allocs/op +// Benchmark_SharedPipe-8 (8 mb) 75 15710658 ns/op 161 B/op 2 allocs/op +func Benchmark_SharedPipe(b *testing.B) { + var data [sendMessageSize]byte + var buffer = make([]byte, len(data)) + var writer = NewWriter() + var readers []io.Reader + + for range readerCount { + readers = append(readers, writer.Transpile()) + } + + b.ReportAllocs() + + for b.Loop() { + _, _ = writer.Write(data[:]) + for i := range readerCount { + _, _ = readers[i].Read(buffer) + } + } +} diff --git a/foundations/stream/internal/pkg/storage/datalake.go b/foundations/stream/internal/pkg/storage/datalake.go new file mode 100644 index 0000000000..b6e47d3de7 --- /dev/null +++ b/foundations/stream/internal/pkg/storage/datalake.go @@ -0,0 +1,646 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/textproto" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/tracing" + "github.com/pkg/errors" + "github.com/valyala/fasthttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" +) + +var tracer = otel.Tracer("storage.datalake") + +type uploadResult struct { + key string + error string +} + +// DatalakeStorage represents datalake storage +type DatalakeStorage struct { + baseURL string + workspace string + token string + logger *zap.Logger + client fasthttp.Client +} + +// NewDatalakeStorage creates a new datalake client +func NewDatalakeStorage(ctx context.Context, baseURL, workspace, token string) Storage { + return &DatalakeStorage{ + baseURL: baseURL, + token: token, + workspace: workspace, + logger: log.FromContext(ctx).With(zap.String("storage", "datalake")), + client: fasthttp.Client{ + MaxIdleConnDuration: 5 * time.Second, + MaxConnsPerHost: 100, + }, + } +} + +var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") + +func escapeQuotes(s string) string { + return quoteEscaper.Replace(s) +} + +// multipart writer CreateFormFile function does not support custom content type +// here we have to have a modified copy that uses actual type instead of application/octet-stream +// see https://github.com/golang/go/issues/49329 +func createFormFile(writer *multipart.Writer, fieldname, filename, contentType string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, escapeQuotes(fieldname), escapeQuotes(filename))) + h.Set("Content-Type", contentType) + return writer.CreatePart(h) +} + +func getObjectKeyFromPath(s string) string { + var _, objectKey = filepath.Split(s) + return objectKey +} + +// PutFile uploads file to the datalake +func (d *DatalakeStorage) PutFile(ctx context.Context, filename string, options PutOptions) error { + _, span := tracer.Start(ctx, "datalake.put_file", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", getObjectKeyFromPath(filename)), + )) + defer span.End() + + // #nosec + file, err := os.Open(filename) + if err != nil { + return err + } + defer func() { + _ = file.Close() + }() + + var objectKey = getObjectKeyFromPath(filename) + var logger = d.logger.With(zap.String("upload", d.workspace), zap.String("fileName", filename)) + + logger.Debug("start") + + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + + part, err := createFormFile(writer, "file", objectKey, getContentType(objectKey)) + if err != nil { + tracing.RecordError(span, err) + return errors.Wrapf(err, "failed to create form file") + } + + if _, err := io.Copy(part, file); err != nil { + tracing.RecordError(span, err) + return errors.Wrapf(err, "failed to copy file data") + } + + if err := writer.Close(); err != nil { + tracing.RecordError(span, err) + return errors.Wrapf(err, "failed to close multipart writer") + } + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + + res := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(res) + + req.SetRequestURI(d.baseURL + "/upload/form-data/" + d.workspace) + req.Header.SetMethod(fasthttp.MethodPost) + req.Header.Add("Authorization", "Bearer "+d.token) + req.Header.SetContentType(writer.FormDataContentType()) + if options.NoCache { + req.Header.Add("Cache-Control", "max-age=0, must-revalidate") + } + req.SetBody(body.Bytes()) + + if err := d.client.Do(req, res); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "upload failed", res) + return errors.Wrapf(err, "upload failed") + } + + var result []uploadResult + if err := json.Unmarshal(res.Body(), &result); err != nil { + tracing.RecordError(span, err) + return errors.Wrapf(err, "parse error") + } + + for _, res := range result { + if res.error != "" { + tracing.RecordError(span, err) + return fmt.Errorf("upload error: %v %v", res.key, res.error) + } + } + + return nil +} + +// DeleteFile deletes file from the datalake +func (d *DatalakeStorage) DeleteFile(ctx context.Context, filename string) error { + _, span := tracer.Start(ctx, "datalake.delete_file", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", getObjectKeyFromPath(filename)), + )) + defer span.End() + + var logger = d.logger.With(zap.String("delete", d.workspace), zap.String("fileName", filename)) + logger.Debug("start") + + var objectKey = getObjectKeyFromPath(filename) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + + res := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(res) + + req.SetRequestURI(d.baseURL + "/blob/" + d.workspace + "/" + objectKey) + req.Header.SetMethod(fasthttp.MethodDelete) + req.Header.Add("Authorization", "Bearer "+d.token) + + if err := d.client.Do(req, res); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "delete failed", res) + return errors.Wrapf(err, "delete failed") + } + + if err := okResponse(res); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "bad status code", res) + return err + } + + return nil +} + +// PatchMeta patches metadata for the object +func (d *DatalakeStorage) PatchMeta(ctx context.Context, filename string, md *Metadata) error { + _, span := tracer.Start(ctx, "datalake.patch_meta", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", getObjectKeyFromPath(filename)), + )) + defer span.End() + + var logger = d.logger.With(zap.String("patch meta", d.workspace), zap.String("fileName", filename)) + logger.Debug("start") + defer logger.Debug("finished") + + var objectKey = getObjectKeyFromPath(filename) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(d.baseURL + "/meta/" + d.workspace + "/" + objectKey) + req.Header.SetMethod(fasthttp.MethodPatch) + req.Header.Add("Authorization", "Bearer "+d.token) + req.Header.SetContentType("application/json") + + b, err := json.Marshal(md) + + if err != nil { + return err + } + req.SetBody(b) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "request failed", resp) + return err + } + + if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "bad status code", resp) + return err + } + + return nil +} + +// GetMeta gets metadata related to the object +func (d *DatalakeStorage) GetMeta(ctx context.Context, filename string) (*Metadata, error) { + _, span := tracer.Start(ctx, "datalake.put_file", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", getObjectKeyFromPath(filename)), + )) + defer span.End() + + var logger = d.logger.With(zap.String("get meta", d.workspace), zap.String("fileName", filename)) + logger.Debug("start") + + var objectKey = getObjectKeyFromPath(filename) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(d.baseURL + "/meta/" + d.workspace + "/" + objectKey) + req.Header.SetMethod(fasthttp.MethodGet) + req.Header.Add("Authorization", "Bearer "+d.token) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "request failed", resp) + return nil, err + } + + if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "bad status code", resp) + return nil, err + } + + var md Metadata + var err = json.Unmarshal(resp.Body(), &md) + + return &md, err +} + +// GetFile gets file from the storage +func (d *DatalakeStorage) GetFile(ctx context.Context, filename, destination string) error { + _, span := tracer.Start(ctx, "datalake.get_file", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", getObjectKeyFromPath(filename)), + )) + defer span.End() + + var logger = d.logger.With(zap.String("get", d.workspace), zap.String("fileName", filename), zap.String("destination", destination)) + logger.Debug("start") + + var objectKey = getObjectKeyFromPath(filename) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(d.baseURL + "/blob/" + d.workspace + "/" + objectKey) + req.Header.SetMethod(fasthttp.MethodGet) + req.Header.Add("Authorization", "Bearer "+d.token) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "request failed", resp) + return err + } + + if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "bad status code", resp) + return err + } + + // #nosec + file, err := os.Create(destination) + if err != nil { + tracing.RecordError(span, err) + logger.Debug("can't create a file", zap.Error(err)) + return err + } + defer func() { + _ = file.Close() + }() + if err = resp.BodyWriteTo(file); err != nil { + tracing.RecordError(span, err) + logger.Debug("can't write to file", zap.Error(err)) + return err + } + + stat, err := os.Stat(destination) + if err != nil { + tracing.RecordError(span, err) + logger.Error("can't stat the file", zap.Error(err)) + return err + } + + logger.Info("file downloaded successfully", zap.Int64("size", stat.Size())) + return nil +} + +// StatFile gets file stat from the storage +func (d *DatalakeStorage) StatFile(ctx context.Context, filename string) (*BlobInfo, error) { + _, span := tracer.Start(ctx, "datalake.stat_file", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", getObjectKeyFromPath(filename)), + )) + defer span.End() + + var logger = d.logger.With(zap.String("head", d.workspace), zap.String("fileName", filename)) + logger.Debug("start") + + var objectKey = getObjectKeyFromPath(filename) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(d.baseURL + "/blob/" + d.workspace + "/" + objectKey) + req.Header.SetMethod(fasthttp.MethodHead) + req.Header.Add("Authorization", "Bearer "+d.token) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "request failed", resp) + return nil, err + } + + if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "bad status code", resp) + return nil, err + } + + var info BlobInfo + info.Size = int64(resp.Header.ContentLength()) + info.Type = string(resp.Header.ContentType()) + info.ETag = string(resp.Header.Peek("ETag")) + + logger.Debug("finished") + return &info, nil +} + +// SetParent updates blob parent reference +func (d *DatalakeStorage) SetParent(ctx context.Context, filename, parent string) error { + _, span := tracer.Start(ctx, "datalake.set_parent", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", getObjectKeyFromPath(filename)), + )) + defer span.End() + + var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("fileName", filename), zap.String("parent", parent)) + + logger.Debug("start") + + var objectKey = getObjectKeyFromPath(filename) + var parentKey = getObjectKeyFromPath(parent) + + if objectKey == parentKey { + // no need to set parent for itself + return nil + } + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(d.baseURL + "/blob/" + d.workspace + "/" + objectKey + "/parent") + req.Header.SetMethod(fasthttp.MethodPatch) + req.Header.Add("Authorization", "Bearer "+d.token) + req.Header.SetContentType("application/json") + + body := map[string]any{ + "parent": parentKey, + } + + if err := json.NewEncoder(req.BodyWriter()).Encode(body); err != nil { + tracing.RecordError(span, err) + logger.Debug("can not encode body", zap.Error(err)) + return err + } + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "request failed", resp) + return err + } + + if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "bad status code", resp) + return err + } + + return nil +} + +// MultipartUploadStart creates a new multipart upload +func (d *DatalakeStorage) MultipartUploadStart(ctx context.Context, objectName, contentType string) (string, error) { + _, span := tracer.Start(ctx, "datalake.multipart_upload_start", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", objectName), + )) + defer span.End() + + var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("objectName", objectName)) + url := fmt.Sprintf("%v/upload/multipart/%v/%v", d.baseURL, d.workspace, objectName) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(url) + req.Header.SetMethod(fasthttp.MethodPost) + req.Header.Add("Authorization", "Bearer "+d.token) + req.Header.SetContentType(contentType) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "request failed", resp) + return "", err + } + + if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "bad status code", resp) + return "", err + } + + var result struct { + UploadID string `json:"uploadId"` + } + err := json.Unmarshal(resp.Body(), &result) + + return result.UploadID, err +} + +// MultipartUploadPart uploads a part of a multipart upload +func (d *DatalakeStorage) MultipartUploadPart(ctx context.Context, objectName, uploadID string, partNumber int, data []byte) (*MultipartPart, error) { + _, span := tracer.Start(ctx, "datalake.multipart_upload_part", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", objectName), + attribute.Int("size", len(data)), + )) + defer span.End() + + var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("uploadID", uploadID), zap.Int("partNumber", partNumber)) + params := url.Values{} + params.Add("uploadId", uploadID) + params.Add("partNumber", strconv.Itoa(partNumber)) + url := fmt.Sprintf("%v/upload/multipart/%v/%v/part?%v", d.baseURL, d.workspace, objectName, params.Encode()) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(url) + req.Header.SetMethod(fasthttp.MethodPut) + req.Header.Add("Authorization", "Bearer "+d.token) + req.Header.SetContentType("application/octet-stream") + req.Header.SetContentLength(len(data)) + req.SetBody(data) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "request failed", resp) + return nil, err + } + + if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "bad status code", resp) + return nil, err + } + + var part MultipartPart + err := json.Unmarshal(resp.Body(), &part) + + return &part, err +} + +// MultipartUploadComplete completes a multipart upload +func (d *DatalakeStorage) MultipartUploadComplete(ctx context.Context, objectName, uploadID string, parts []MultipartPart) error { + _, span := tracer.Start(ctx, "datalake.multipart_upload_complete", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", objectName), + )) + defer span.End() + + var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("uploadID", uploadID), zap.String("objectName", objectName)) + params := url.Values{} + params.Add("uploadId", uploadID) + url := fmt.Sprintf("%v/upload/multipart/%v/%v/complete?%v", d.baseURL, d.workspace, objectName, params.Encode()) + + body, err := json.Marshal(map[string]any{ + "parts": parts, + }) + + if err != nil { + logger.Debug("can not encode body", zap.Error(err)) + return err + } + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(url) + req.Header.SetMethod(fasthttp.MethodPost) + req.Header.Add("Authorization", "Bearer "+d.token) + req.Header.SetContentType("application/json") + req.SetBody(body) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "request failed", resp) + return err + } + + if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "bad status code", resp) + return err + } + + return nil +} + +// MultipartUploadCancel cancels a multipart upload +func (d *DatalakeStorage) MultipartUploadCancel(ctx context.Context, objectName, uploadID string) error { + _, span := tracer.Start(ctx, "datalake.multipart_upload_cancel", trace.WithAttributes( + attribute.String("workspace", d.workspace), + attribute.String("object_key", objectName), + )) + defer span.End() + + var logger = d.logger.With(zap.String("workspace", d.workspace), zap.String("uploadID", uploadID)) + params := url.Values{} + params.Add("uploadId", uploadID) + url := fmt.Sprintf("%v/upload/multipart/%v/%v/abort?%v", d.baseURL, d.workspace, objectName, params.Encode()) + + req := fasthttp.AcquireRequest() + defer fasthttp.ReleaseRequest(req) + req.SetRequestURI(url) + req.Header.SetMethod(fasthttp.MethodPost) + req.Header.Add("Authorization", "Bearer "+d.token) + + resp := fasthttp.AcquireResponse() + defer fasthttp.ReleaseResponse(resp) + + if err := d.client.Do(req, resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "request failed", resp) + return err + } + + if err := okResponse(resp); err != nil { + tracing.RecordError(span, err) + logRequestError(logger, err, "bad status code", resp) + return err + } + + return nil +} + +func okResponse(res *fasthttp.Response) error { + var statusOK = res.StatusCode() >= 200 && res.StatusCode() < 300 + + if !statusOK { + return fmt.Errorf("unexpected status code: %d", res.StatusCode()) + } + + return nil +} + +func logRequestError(logger *zap.Logger, err error, msg string, res *fasthttp.Response) { + logger.Error( + msg, + zap.Error(err), + zap.Int("status", res.StatusCode()), + zap.String("headers", res.Header.String()), + zap.String("response", res.String()), + ) +} + +var _ Storage = (*DatalakeStorage)(nil) +var _ MultipartStorage = (*DatalakeStorage)(nil) +var _ MetaProvider = (*DatalakeStorage)(nil) diff --git a/foundations/stream/internal/pkg/storage/s3.go b/foundations/stream/internal/pkg/storage/s3.go new file mode 100644 index 0000000000..aea0adec5b --- /dev/null +++ b/foundations/stream/internal/pkg/storage/s3.go @@ -0,0 +1,196 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package storage provdies simple storage interface for the remote storages. +package storage + +import ( + "context" + "fmt" + + "os" + "path/filepath" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/hcengineering/stream/internal/pkg/log" + "go.uber.org/zap" +) + +// S3Storage represents S3 storage +type S3Storage struct { + client *s3.Client + bucketName string + logger *zap.Logger +} + +// NewS3 creates a new S3 storage +func NewS3(ctx context.Context, endpoint, bucketName string) Storage { + var accessKeyID = os.Getenv("AWS_ACCESS_KEY_ID") + var accessKeySecret = os.Getenv("AWS_SECRET_ACCESS_KEY") + + cfg, err := config.LoadDefaultConfig(ctx, + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyID, accessKeySecret, "")), + config.WithRegion("auto"), + ) + if err != nil { + panic(err.Error()) + } + + var s3Client = s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = &endpoint + o.UsePathStyle = true + }) + + return &S3Storage{ + client: s3Client, + bucketName: bucketName, + logger: log.FromContext(ctx).With(zap.String("s3", "storage")), + } +} + +// DeleteFile deletes file from the s3 storage +func (u *S3Storage) DeleteFile(ctx context.Context, fileName string) error { + var _, objectKey = filepath.Split(fileName) + var logger = u.logger.With(zap.String("delete", u.bucketName), zap.String("fileName", fileName)) + + logger.Debug("start") + + input := &s3.DeleteObjectInput{ + Bucket: aws.String(u.bucketName), + Key: aws.String(objectKey), + } + + _, err := u.client.DeleteObject(ctx, input) + if err != nil { + return fmt.Errorf("failed to delete file from S3: %w", err) + } + + logger.Debug("deleted") + return nil +} + +// PutFile uploads file to the s3 storage +func (u *S3Storage) PutFile(ctx context.Context, fileName string, options PutOptions) error { + var _, objectKey = filepath.Split(fileName) + var logger = u.logger.With(zap.String("upload", u.bucketName), zap.String("fileName", fileName)) + + logger.Debug("start") + + // #nosec + var file, err = os.Open(fileName) + + if err != nil { + logger.Error("can not open file", zap.Error(err)) + return err + } + + defer func() { + _ = file.Close() + }() + + _, err = u.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(u.bucketName), + Key: aws.String(objectKey), + Body: file, + ContentType: aws.String(getContentType(objectKey)), + }) + + if err != nil { + logger.Error("couldn't upload file", zap.Error(err)) + return err + } + + err = s3.NewObjectExistsWaiter(u.client).Wait( + ctx, &s3.HeadObjectInput{Bucket: aws.String(u.bucketName), Key: aws.String(objectKey)}, time.Minute) + + if err != nil { + logger.Error("Failed attempt to wait for object to exist") + return err + } + + logger.Debug("uploaded") + return nil +} + +// GetFile gets file from the storage and stores it to destination +func (u *S3Storage) GetFile(ctx context.Context, filename, dest string) error { + var logger = u.logger.With(zap.String("get", u.bucketName), zap.String("fileName", filename), zap.String("destination", dest)) + + var result, err = u.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &u.bucketName, + Key: &filename, + }) + + if err != nil { + logger.Error("failed to get object", zap.Error(err)) + return err + } + defer func() { + _ = result.Body.Close() + }() + + // Create a local file to save the downloaded content + // #nosec + file, err := os.Create(dest) + if err != nil { + logger.Error("failed to create file", zap.Error(err)) + return err + } + defer func() { + _ = file.Close() + }() + // Copy the S3 object content to the local file + _, err = file.ReadFrom(result.Body) + if err != nil { + logger.Error("failed to write to file", zap.Error(err)) + return err + } + + logger.Debug("file downloaded successfully") + + return nil +} + +// StatFile gets file stat from the storage +func (u *S3Storage) StatFile(ctx context.Context, filename string) (*BlobInfo, error) { + var logger = u.logger.With(zap.String("head", u.bucketName), zap.String("fileName", filename)) + + var head, err = u.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &u.bucketName, + Key: &filename, + }) + + if err != nil { + logger.Error("failed to head object", zap.Error(err)) + return nil, err + } + + var info BlobInfo + info.Size = *head.ContentLength + info.Type = *head.ContentType + info.ETag = *head.ETag + + return &info, nil +} + +// SetParent updates blob parent reference +func (u *S3Storage) SetParent(ctx context.Context, filename, parent string) error { + // Not implemented for S3 Storage + return nil +} + +var _ Storage = (*S3Storage)(nil) diff --git a/foundations/stream/internal/pkg/storage/storage.go b/foundations/stream/internal/pkg/storage/storage.go new file mode 100644 index 0000000000..cd9c9b8aac --- /dev/null +++ b/foundations/stream/internal/pkg/storage/storage.go @@ -0,0 +1,94 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "net/url" + "strings" + + "github.com/pkg/errors" +) + +// Metadata represents file's metadata +type Metadata map[string]any + +// MetaProvider provides simple api for working with files metadata +type MetaProvider interface { + GetMeta(ctx context.Context, filename string) (*Metadata, error) + PatchMeta(ctx context.Context, filename string, value *Metadata) error +} + +// BlobInfo contains blob stat information +type BlobInfo struct { + Size int64 + Type string + ETag string +} + +// PutOptions represents options for the PutFile operation +type PutOptions struct { + NoCache bool +} + +// Storage represents file-based storage +type Storage interface { + PutFile(ctx context.Context, fileName string, options PutOptions) error + DeleteFile(ctx context.Context, fileName string) error + GetFile(ctx context.Context, fileName, destination string) error + StatFile(ctx context.Context, fileName string) (*BlobInfo, error) + SetParent(ctx context.Context, fileName string, parentName string) error +} + +// MultipartPart represents uploaded multipart part +type MultipartPart struct { + PartNumber int `json:"partNumber"` + ETag string `json:"etag"` +} + +// MultipartStorage represents multipart-based storage +type MultipartStorage interface { + MultipartUploadStart(ctx context.Context, objectName, contentType string) (string, error) + MultipartUploadPart(ctx context.Context, objectName, uploadID string, partNumber int, data []byte) (*MultipartPart, error) + MultipartUploadComplete(ctx context.Context, objectName, uploadID string, parts []MultipartPart) error + MultipartUploadCancel(ctx context.Context, objectName, uploadID string) error +} + +// NewStorageByURL creates a new storage based on the type from the url scheme, for example "datalake://my-datalake-endpoint" +func NewStorageByURL(ctx context.Context, u *url.URL, storageType, token, workspace string) (Storage, error) { + if workspace == "" { + return nil, errors.New("workspace is missed") + } + switch storageType { + case "datalake": + if token == "" { + return nil, errors.New("token is missed") + } + return NewDatalakeStorage(ctx, u.String(), workspace, token), nil + case "s3": + return NewS3(ctx, u.String(), workspace), nil + default: + return nil, errors.New("unknown scheme") + } +} + +func getContentType(objectKey string) string { + if strings.HasSuffix(objectKey, ".ts") { + return "video/mp2t" + } + if strings.HasSuffix(objectKey, ".m3u8") { + return "video/x-mpegurl" + } + return "application/octet-stream" +} diff --git a/foundations/stream/internal/pkg/token/token.go b/foundations/stream/internal/pkg/token/token.go new file mode 100644 index 0000000000..74da213b74 --- /dev/null +++ b/foundations/stream/internal/pkg/token/token.go @@ -0,0 +1,74 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package token provides functions to work with platform tokens +package token + +import ( + "fmt" + + "github.com/golang-jwt/jwt/v5" + "github.com/google/uuid" +) + +// Token represents Claims for the platform token +type Token struct { + jwt.MapClaims + Account string `json:"account"` + Workspace string `json:"workspace,omitempty"` + Extra map[string]any `json:"extra,omitempty"` +} + +// NewToken creates a new platform token +func NewToken(serverSecret, workspace, service string) (string, error) { + var res = Token{ + Account: uuid.NewString(), + Workspace: workspace, + Extra: map[string]any{ + "service": service, + }, + } + return res.Encode(serverSecret) +} + +// Encode signes a token object and converts it to string +func (t *Token) Encode(serverSecret string) (string, error) { + var token = jwt.NewWithClaims(jwt.SigningMethodHS256, t) + var tokenString, err = token.SignedString([]byte(serverSecret)) + + if err != nil { + return "", err + } + + return tokenString, nil +} + +// Decode decodes a token by a passed configuration +func Decode(secretKey, tokenString string) (*Token, error) { + token, err := jwt.ParseWithClaims(tokenString, &Token{}, func(token *jwt.Token) (any, error) { + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) + } + return []byte(secretKey), nil + }) + + if err != nil { + return nil, fmt.Errorf("failed to parse token: %w", err) + } + + if claims, ok := token.Claims.(*Token); ok && token.Valid { + return claims, nil + } + + return nil, fmt.Errorf("invalid token") +} diff --git a/foundations/stream/internal/pkg/token/token_test.go b/foundations/stream/internal/pkg/token/token_test.go new file mode 100644 index 0000000000..7c46879efb --- /dev/null +++ b/foundations/stream/internal/pkg/token/token_test.go @@ -0,0 +1,36 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package token_test + +import ( + "testing" + + "github.com/hcengineering/stream/internal/pkg/token" + "github.com/stretchr/testify/require" +) + +func Test_GenerateSimpleServiceToken(t *testing.T) { + var _, err = token.NewToken("secret", "ws", "stream") + require.NoError(t, err) +} + +func Test_ParseSimpleServiceToken(t *testing.T) { + const secret = "secret" + tokenString, err := token.NewToken(secret, "ws", "stream") + require.NoError(t, err) + tok, err := token.Decode(secret, tokenString) + require.NoError(t, err) + require.Equal(t, "ws", tok.Workspace) + require.Equal(t, "stream", tok.Extra["service"]) +} diff --git a/foundations/stream/internal/pkg/tracing/tracing.go b/foundations/stream/internal/pkg/tracing/tracing.go new file mode 100644 index 0000000000..fee8f4a794 --- /dev/null +++ b/foundations/stream/internal/pkg/tracing/tracing.go @@ -0,0 +1,54 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tracing provides utilities for OpenTelemetry tracing. +package tracing + +import ( + "context" + + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +// RecordError records an error into the provided span and sets its status to Error. +func RecordError(span trace.Span, err error) { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) +} + +// WithSpan wraps a function execution within a span. +func WithSpan(ctx context.Context, tracer trace.Tracer, name string, fn func(ctx context.Context) error) error { + ctx, span := tracer.Start(ctx, name) + defer span.End() + + err := fn(ctx) + if err != nil { + RecordError(span, err) + } + + return err +} + +// WithSpanResult wraps a function returning result execution within a span. +func WithSpanResult[Result any](ctx context.Context, tracer trace.Tracer, name string, fn func(ctx context.Context) (Result, error)) (Result, error) { + ctx, span := tracer.Start(ctx, name) + defer span.End() + + result, err := fn(ctx) + if err != nil { + RecordError(span, err) + } + + return result, err +} diff --git a/foundations/stream/internal/pkg/uploader/options.go b/foundations/stream/internal/pkg/uploader/options.go new file mode 100644 index 0000000000..d3642cca6a --- /dev/null +++ b/foundations/stream/internal/pkg/uploader/options.go @@ -0,0 +1,28 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +package uploader + +import "time" + +// Options reprents options for uploading files +type Options struct { + Dir string + WorkerCount uint32 + BufferSize int + RetryCount int + RetryDelay time.Duration + Timeout time.Duration + Source string + SourceFile string +} diff --git a/foundations/stream/internal/pkg/uploader/uploader.go b/foundations/stream/internal/pkg/uploader/uploader.go new file mode 100644 index 0000000000..78df9f3d2b --- /dev/null +++ b/foundations/stream/internal/pkg/uploader/uploader.go @@ -0,0 +1,444 @@ +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package uploader provides a mechanism for uploading files to a remote storage. +package uploader + +import ( + "context" + "hash/fnv" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + "unsafe" + + "github.com/hcengineering/stream/internal/pkg/log" + "github.com/hcengineering/stream/internal/pkg/storage" + "github.com/pkg/errors" + "go.opentelemetry.io/otel" + "go.uber.org/zap" + "k8s.io/utils/inotify" +) + +var tracer = otel.Tracer("uploader") + +// See at https://man7.org/linux/man-pages/man7/inotify.7.html +const inotifyCloseWrite uint32 = 0x8 // IN_CLOSE_WRITE +const inotifyMovedFrom uint32 = 0x40 // IN_MOVED_FROM +const inotifyMovedTo uint32 = 0x80 // IN_MOVED_TO +const inotifyDelete uint32 = 0x200 // IN_DELETE + +// Uploader represents file uploader +type Uploader interface { + Start() + Stop() + Cancel() +} + +type uploaderImpl struct { + logger *zap.Logger + options *Options + storage storage.Storage + + filesCh chan string + + sentFiles sync.Map // key: string, value: struct{} + + shouldDeleteOnStop func(string) bool + + workersCh []chan func() + + watcherStopCh chan struct{} + watcherDoneCh chan struct{} + + uploadCtx context.Context + uploadCancel context.CancelFunc + + workerWaitGroup sync.WaitGroup +} + +// New creates a new instance of uploader +func New(ctx context.Context, s storage.Storage, opts Options) Uploader { + if s == nil { + panic("storage should not be nil") + } + + var res = &uploaderImpl{ + options: &opts, + storage: s, + logger: log.FromContext(ctx).With(zap.String("uploader", opts.Dir)), + shouldDeleteOnStop: func(s string) bool { + return strings.HasSuffix(s, "m3u8") + }, + filesCh: make(chan string, opts.BufferSize), + watcherStopCh: make(chan struct{}), + watcherDoneCh: make(chan struct{}), + } + + res.workersCh = make([]chan func(), opts.WorkerCount) + + for i := range opts.WorkerCount { + res.workersCh[i] = make(chan func(), opts.BufferSize) + } + + res.logger.Sugar().Debugf("uploader config is %v", opts) + + res.uploadCtx, res.uploadCancel = context.WithCancel(ctx) + + err := os.MkdirAll(opts.Dir, os.ModePerm) + if err != nil { + res.logger.Error("can not create upload directory", zap.Error(err), zap.String("dir", opts.Dir)) + } + + return res +} + +func (u *uploaderImpl) Stop() { + u.logger.Info("stopping upload") + u.stop(u.uploadCtx, false) +} + +func (u *uploaderImpl) Cancel() { + u.logger.Info("canceling upload") + u.stop(u.uploadCtx, true) +} + +func (u *uploaderImpl) scanFiles(ctx context.Context) { + _, span := tracer.Start(ctx, "scanFiles") + defer span.End() + + logger := u.logger.With(zap.String("dir", u.options.Dir)) + + logger.Info("scan files") + files, err := os.ReadDir(u.options.Dir) + if err != nil { + logger.Error("failed to read files", zap.Error(err)) + return + } + + count := 0 + for _, f := range files { + if f.IsDir() { + continue + } + + var filePath = filepath.Join(u.options.Dir, f.Name()) + + // Ignore source file + if filePath == u.options.SourceFile { + continue + } + + if _, uploaded := u.sentFiles.Load(filePath); uploaded { + logger.Debug("file already uploaded", zap.String("file", filePath)) + continue + } + + u.filesCh <- filePath + count++ + } + + logger.Info("scan complete", zap.Int("count", count)) +} + +func (u *uploaderImpl) stop(ctx context.Context, rollback bool) { + ctx, span := tracer.Start(ctx, "stop") + defer span.End() + + // Stop watching for new files + close(u.watcherStopCh) + <-u.watcherDoneCh + + // Scan remaining files in the directory + u.scanFiles(ctx) + + // Close filesCh so no new files added + close(u.filesCh) + + // Wait for all workers to finish processing + u.workerWaitGroup.Wait() + u.logger.Debug("workers done") + + // Perform rollback + if rollback { + u.uploadRollback(ctx) + } + + u.uploadCancel() + + remainingFiles, err := os.ReadDir(u.options.Dir) + if err != nil && !os.IsNotExist(err) { + u.logger.Error("failed to read dir", zap.Error(err)) + } + // log remaining files + if len(remainingFiles) > 0 { + files := make([]string, 0, len(remainingFiles)) + for _, entry := range remainingFiles { + files = append(files, entry.Name()) + } + u.logger.Info("remaining files", zap.Int("count", len(files)), zap.Any("files", files)) + } + + u.sentFiles.Clear() + + u.logger.Debug("stopped", zap.Bool("rollback", rollback)) +} + +func (u *uploaderImpl) uploadRollback(ctx context.Context) { + _, span := tracer.Start(ctx, "uploadRollback") + defer span.End() + + u.logger.Debug("starting rollback...") + + // Create a separate worker pool for rollback + var rollbackWg sync.WaitGroup + rollbackCh := make(chan string, u.options.BufferSize) + + // Start rollback workers + for range u.options.WorkerCount { + rollbackWg.Add(1) + go func() { + defer rollbackWg.Done() + for filename := range rollbackCh { + u.deleteRemoteFile(filename) + } + }() + } + + // Send files to rollback + u.sentFiles.Range(func(key, _ any) bool { + rollbackCh <- key.(string) + return true + }) + + // Close channel and wait for rollback to complete + close(rollbackCh) + rollbackWg.Wait() + u.logger.Debug("rollback done") +} + +func (u *uploaderImpl) Start() { + watcherReady := make(chan struct{}) + + u.startWorkers() + go u.startWatch(watcherReady) + + <-watcherReady + + u.scanFiles(u.uploadCtx) +} + +func (u *uploaderImpl) startWorkers() { + go func() { + var logger = u.logger.With(zap.String("func", "startWorkers")) + logger.Debug("fanout goroutine started") + defer logger.Debug("fanout goroutine stopped") + + h := fnv.New32a() + for f := range u.filesCh { + // #nosec + bytes := unsafe.Slice(unsafe.StringData(f), len(f)) + _, _ = h.Write(bytes) + id := h.Sum32() % u.options.WorkerCount + u.workersCh[id] <- func() { + u.uploadAndDelete(f) + } + h.Reset() + } + for i := range u.options.WorkerCount { + close(u.workersCh[i]) + } + }() + + for i := range u.options.WorkerCount { + var logger = u.logger.With(zap.Uint32("worker", i)) + + u.workerWaitGroup.Add(1) + go func(index uint32) { + logger.Debug("start") + defer logger.Debug("finished") + + defer u.workerWaitGroup.Done() + for work := range u.workersCh[index] { + work() + } + }(i) + } +} + +func (u *uploaderImpl) deleteRemoteFile(f string) { + var logger = u.logger.With(zap.String("delete remote", f)) + logger.Debug("deleting remote file") + + for range u.options.RetryCount { + var ctx, cancel = context.WithTimeout(u.uploadCtx, u.options.Timeout) + var err = u.storage.DeleteFile(ctx, f) + cancel() + + if err != nil { + logger.Error("attempt failed", zap.Error(err)) + } else { + logger.Debug("file deleted in remote storage") + return + } + + time.Sleep(u.options.RetryDelay) + } + + logger.Error("can not delete remote file") +} + +func (u *uploaderImpl) uploadAndDelete(f string) { + var logger = u.logger.With(zap.String("upload and delete", f)) + logger.Debug("uploading file") + + if f == u.options.Dir { + return + } + + // Check if the file has already been uploaded + _, ok := u.sentFiles.Load(f) + if ok && !u.shouldDeleteOnStop(f) { + logger.Debug("file already uploaded") + return + } + + if err := waitFileExists(f); err != nil { + if os.IsNotExist(err) { + logger.Debug("file does not exist", zap.Error(err)) + } else { + logger.Error("failed to stat file", zap.Error(err)) + } + return + } + + for attempt := range u.options.RetryCount { + logger = logger.With(zap.Int("attempt", attempt)) + var putCtx, putCancel = context.WithTimeout(u.uploadCtx, u.options.Timeout) + var putOptions = storage.PutOptions{ + NoCache: u.shouldDeleteOnStop(f), + } + var err = u.storage.PutFile(putCtx, f, putOptions) + putCancel() + + if err != nil { + logger.Error("attempt failed", zap.Error(err)) + } else { + // Mark the file as uploaded + u.sentFiles.Store(f, struct{}{}) + logger.Debug("file uploaded") + + // Update the file's parent if SourceFile is set + if u.options.Source != "" { + var setParentCtx, setParentCancel = context.WithTimeout(u.uploadCtx, u.options.Timeout) + err = u.storage.SetParent(setParentCtx, f, u.options.Source) + setParentCancel() + if err != nil { + logger.Error("can not set blob parent", zap.Error(err), zap.String("filename", f), zap.String("source", u.options.Source)) + } + } + + // Delete the file locally if it should be deleted + if !u.shouldDeleteOnStop(f) { + if err := os.Remove(f); err != nil && !errors.Is(err, syscall.ENOENT) { + logger.Error("failed to remove file locally", zap.Error(err), zap.String("file", f)) + } else { + logger.Debug("removed file locally") + } + } + + break + } + + time.Sleep(u.options.RetryDelay) + } +} + +// startWatch watches for changes in the directory and uploads created files +func (u *uploaderImpl) startWatch(ready chan<- struct{}) { + defer close(u.watcherDoneCh) + + var logger = u.logger.With(zap.String("func", "startWatch")) + var watcher, err = inotify.NewWatcher() + + if err != nil { + logger.Error("can not start file watcher", zap.Error(err)) + close(ready) + return + } + defer func() { + if err := watcher.Close(); err != nil { + logger.Error("can not close watcher", zap.Error(err)) + } + }() + + if err := watcher.AddWatch(u.options.Dir, inotifyCloseWrite|inotifyMovedTo|inotifyDelete|inotifyMovedFrom); err != nil { + logger.Error("can not start watching", zap.Error(err)) + close(ready) + return + } + + close(ready) + logger.Debug("watching for file updates") + defer logger.Debug("done") + + for { + select { + case <-u.watcherStopCh: + return + case event, ok := <-watcher.Event: + if !ok { + logger.Error("file channel was closed") + return + } + if event.Name == u.options.Dir || + event.Name == u.options.SourceFile || + strings.HasSuffix(event.Name, ".tmp") { + continue + } + + if event.Mask&(inotifyDelete|inotifyMovedFrom) != 0 { + logger.Debug("file deleted or moved away", zap.String("event", event.Name), zap.Uint32("mask", event.Mask)) + continue + } + + logger.Debug("received an event", zap.String("event", event.Name), zap.Uint32("mask", event.Mask)) + + u.filesCh <- event.Name + case err, ok := <-watcher.Error: + if !ok { + logger.Error("error channel was closed") + } + logger.Error("received an error", zap.Error(err)) + } + } +} + +func waitFileExists(file string) error { + var err error + var stat os.FileInfo + + for range 10 { + stat, err = os.Stat(file) + if err == nil && stat.Size() > 0 { + return nil + } + + time.Sleep(50 * time.Millisecond) + } + + return err +}