diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1337fe676f..1611d84838 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -192,7 +192,7 @@ jobs: - name: Testing... run: node common/scripts/install-run-rush.js test env: - DB_URL: 'postgresql://postgres:example@localhost:5433' + DB_URL: 'postgresql://root@localhost:26258/defaultdb?sslmode=disable' ELASTIC_URL: 'http://localhost:9201' MONGO_URL: 'mongodb://localhost:27018' uitest: diff --git a/.vscode/launch.json b/.vscode/launch.json index cf5c4226f5..cdacb92688 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -107,6 +107,7 @@ "MONGO_URL": "mongodb://localhost:27017", "DB_URL": "mongodb://localhost:27017", // "DB_URL": "postgresql://postgres:example@localhost:5432", + // "DB_URL": "postgresql://root@host.docker.internal:26257/defaultdb?sslmode=disable", "SERVER_SECRET": "secret", "REGION_INFO":"|Mongo;pg|Postgres;cockroach|CockroachDB", "TRANSACTOR_URL": "ws://host.docker.internal:3333,ws://host.docker.internal:3331;;pg,ws://host.docker.internal:3332;;cockroach", @@ -326,6 +327,7 @@ "MONGO_URL": "mongodb://localhost:27017", "DB_URL": "mongodb://localhost:27017", "ACCOUNTS_URL": "http://localhost:3000", + "ACCOUNT_DB_URL": "mongodb://localhost:27017", "TELEGRAM_DATABASE": "telegram-service", "REKONI_URL": "http://localhost:4004", "MODEL_VERSION": "0.6.421" @@ -562,7 +564,7 @@ "MINIO_SECRET_KEY": "minioadmin", "SERVER_SECRET": "secret", "SERVICE_ID": "analytics-collector", - "SUPPORT_WORKSPACE": "support", + "SUPPORT_WORKSPACE": "09b65664-b3c3-4ea8-b9a1-c9688bde17f0", "ACCOUNTS_URL": "http://localhost:3000" }, "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], @@ -582,7 +584,7 @@ "MONGO_URL": "mongodb://localhost:27017", "PORT": "4010", "SERVER_SECRET": "secret", - "SUPPORT_WORKSPACE": "support", + "SUPPORT_WORKSPACE": "09b65664-b3c3-4ea8-b9a1-c9688bde17f0", "FIRST_NAME": "Jolie", "LAST_NAME": "AI", "PASSWORD": "password", diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 4ab2faf258..ff69f8395d 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -70,6 +70,9 @@ importers: '@rush-temp/account': specifier: file:./projects/account.tgz version: file:projects/account.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) + '@rush-temp/account-client': + specifier: file:./projects/account-client.tgz + version: file:projects/account-client.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) '@rush-temp/account-service': specifier: file:./projects/account-service.tgz version: file:projects/account-service.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3) @@ -1387,6 +1390,9 @@ importers: '@types/pdfjs-dist': specifier: 2.10.378 version: 2.10.378 + '@types/pg': + specifier: ^8.11.6 + version: 8.11.11 '@types/png-chunks-extract': specifier: ^1.0.2 version: 1.0.2 @@ -3800,12 +3806,16 @@ packages: '@remirror/core-constants@3.0.0': resolution: {integrity: sha512-42aWfPrimMfDKDi4YegyS7x+/0tlzaqwPQCULLanv3DMIlu96KTJR0fM5isWX2UViOqlGnX6YFgqWepcX+XMNg==} + '@rush-temp/account-client@file:projects/account-client.tgz': + resolution: {integrity: sha512-alEMy85WshDCFug+rB3K6vcFRWDj/NyeutHceDQiMW1G01RHNzI+uCSQUqKfAgq6aR/Dq11i9l7TuUxBM3mG7w==, tarball: file:projects/account-client.tgz} + version: 0.0.0 + '@rush-temp/account-service@file:projects/account-service.tgz': - resolution: {integrity: sha512-oAW0UD+ThGbtuXdyaoWJoe93XY7bLO0K2uZneePiHsI3KZM3z9st9fjxSbJE4wmC9iE2wmzNYTN6AHrcFxvRrQ==, tarball: file:projects/account-service.tgz} + resolution: {integrity: sha512-SBWP6HBOnzpJC3MIOlYbttcjYw1HoD9VpbLpUS3zRwEL2slU6c0W/hEBU0/rzh3KkLb6bScKssS5utlSx5RrNw==, tarball: file:projects/account-service.tgz} version: 0.0.0 '@rush-temp/account@file:projects/account.tgz': - resolution: {integrity: sha512-xWdmeIZiSKL5eyvVHNUEnn/GXGsaNkayjxoDW+ydFB1GwUWExvSTA7OOs9mfG+IwLNU08GNKNGBYGM3XKZ2PsA==, tarball: file:projects/account.tgz} + resolution: {integrity: sha512-NN+0JN9QcVA0sk5OmSpSwhjf0lhiWqXFztNPSOZbNnyziTzKgufXiLjho7Nwubi+aev+EIL8weUdW93QBYh07A==, tarball: file:projects/account.tgz} version: 0.0.0 '@rush-temp/activity-assets@file:projects/activity-assets.tgz': @@ -3849,7 +3859,7 @@ packages: version: 0.0.0 '@rush-temp/api-client@file:projects/api-client.tgz': - resolution: {integrity: sha512-tyhf+5DHkF+HpWZlimhdgls3Gp3L7PR0UHYEu0YH7e0UmOoZSuvAVbRbwY/6UnVG1am+xQnrs0xiZ93AHIsgBA==, tarball: file:projects/api-client.tgz} + resolution: {integrity: sha512-lFsfqbx6XIWKo8wq95O25j0RYJ9pUMgtb3T/tVUKfWG36NUuImICpStaQRmcm6MPYq7Gtb0KKuOXyQJg3y/80w==, tarball: file:projects/api-client.tgz} version: 0.0.0 '@rush-temp/attachment-assets@file:projects/attachment-assets.tgz': @@ -3957,7 +3967,7 @@ packages: version: 0.0.0 '@rush-temp/collaborator@file:projects/collaborator.tgz': - resolution: {integrity: sha512-+HMz/V8Ne3kDYebHozkhIZW49OAaf89689gEqbe8KXMUs0hAJG7Fe4Ns5uQp7PqZBEiKoILlfNDJEJPvBRR4tQ==, tarball: file:projects/collaborator.tgz} + resolution: {integrity: sha512-QNIE09ZcEdRCPC+oX1x4ltXrEJ+GDrz23EhmqdEZiiU3gLpj6RTVjGnGLeyKTjmXhPWorKWAGmfSwZ1VLBNGhQ==, tarball: file:projects/collaborator.tgz} version: 0.0.0 '@rush-temp/contact-assets@file:projects/contact-assets.tgz': @@ -3965,11 +3975,11 @@ packages: version: 0.0.0 '@rush-temp/contact-resources@file:projects/contact-resources.tgz': - resolution: {integrity: sha512-sOoou1UaCHSN/qYHhEiWyiPICxmqP2p+ACDAeVzhU8fuUa5tejlzuANOdiEYqz4c8+u/NhlCQHd7G7G5WjMPpg==, tarball: file:projects/contact-resources.tgz} + resolution: {integrity: sha512-RWk6026zqPV98SklhNMT0+FjevdIJS6kI3vEf4Y3H2EuP9lFJuj5a2pTiWRXFT+yeoOkYrCtUD0duZ65JrJidw==, tarball: file:projects/contact-resources.tgz} version: 0.0.0 '@rush-temp/contact@file:projects/contact.tgz': - resolution: {integrity: sha512-79dZ9DTczWnB5eBmX1XH8d/mw8RjbGeCK+ryNpym3Bh3kykq7RwuTeTRvIlPTzk+vkCCXzb34A/zSdaslTyfnw==, tarball: file:projects/contact.tgz} + resolution: {integrity: sha512-+QCS1ZJvdStvQoBY8jt91ycyK4CK+W9C0txRhys5R/BCdzPkhdgl8OTZn6k60Fes0T0zKl5ceX3plOJXVtu+Hw==, tarball: file:projects/contact.tgz} version: 0.0.0 '@rush-temp/controlled-documents-assets@file:projects/controlled-documents-assets.tgz': @@ -3977,7 +3987,7 @@ packages: version: 0.0.0 '@rush-temp/controlled-documents-resources@file:projects/controlled-documents-resources.tgz': - resolution: {integrity: sha512-C5G4bSM6HruIz0Ttj++bAXyzlix4UfsgNP7kYw+wHslj3x2zb9Qa3zEFVOtZPeB2MhpMlmlsz2Bu4F2an1Jfxw==, tarball: file:projects/controlled-documents-resources.tgz} + resolution: {integrity: sha512-FhuNC2ZQwCMkG9Jn2qfTOQf6b4Gh6CZnrzQI+cvr+6BK3uFaT3EnDMh2MlE2/lm0X7m98+d9T+2z9oFJmO5vYQ==, tarball: file:projects/controlled-documents-resources.tgz} version: 0.0.0 '@rush-temp/controlled-documents@file:projects/controlled-documents.tgz': @@ -4061,7 +4071,7 @@ packages: version: 0.0.0 '@rush-temp/front@file:projects/front.tgz': - resolution: {integrity: sha512-dohiQMA5N6BQ+LJRe15hzxZOdP3edinNbP2id3xuoM/Xgq/Irs0EfczCo7zy3YcQocIDFMR71AYWGdXgilJbBw==, tarball: file:projects/front.tgz} + resolution: {integrity: sha512-nZ678oTwK/1srj53UHe1oNdkBUp3MrZdjxt0ptqXNkNWdPyFpQwe1PMinygBjU2+EmmygnCgvJTnQ+HtyWXtiQ==, tarball: file:projects/front.tgz} version: 0.0.0 '@rush-temp/github-assets@file:projects/github-assets.tgz': @@ -4097,7 +4107,7 @@ packages: version: 0.0.0 '@rush-temp/guest-resources@file:projects/guest-resources.tgz': - resolution: {integrity: sha512-RfqwhU41ifAGcQchDRFYT12hz9wj0NsQ5ffr1vNeQg5+5s7nEu2o2lIy4jd6nvBpP5/b1d+QpObSmgJ/irvi1A==, tarball: file:projects/guest-resources.tgz} + resolution: {integrity: sha512-kCjOwB3F17OgPj2Op6erq7oht+o/HdR9hwiQtKtropjMf3iq6QALpK0ReHl1tIMpaTvd9MZLFQYmrpBX6BXwHw==, tarball: file:projects/guest-resources.tgz} version: 0.0.0 '@rush-temp/guest@file:projects/guest.tgz': @@ -4169,11 +4179,11 @@ packages: version: 0.0.0 '@rush-temp/login-resources@file:projects/login-resources.tgz': - resolution: {integrity: sha512-NARTLQpe+poR8P/l3cOTAEPFutkH24EQqjw6zY6pIChSPhBVtK4GJu4yRJo7WtZPjBTq8qL7q3bDoZ7+aooAWw==, tarball: file:projects/login-resources.tgz} + resolution: {integrity: sha512-n8jvuqgHgissnRkZBIXW6EP6zXR66DrwC4vVG6+DV2rMmZq25W/PBT9mam+lwFo/L0DcQ5sBGerhyDrLbjq64w==, tarball: file:projects/login-resources.tgz} version: 0.0.0 '@rush-temp/login@file:projects/login.tgz': - resolution: {integrity: sha512-jX5cfakMhaPs02qHZsnzTtFknqMhyz1mI7Wqd0NLthHsSfVP9NIbnB/DhUG9hChVm3Szj7cBrXg+skvv0aDCSQ==, tarball: file:projects/login.tgz} + resolution: {integrity: sha512-V3NnYilBtJGxpAF6i9WDcIlvqzmy84XaRuv0ekKYxIUYjQGGLaKRGs5cwONvsBgm02ZJ9VGh9nqGV5S3YPUTBg==, tarball: file:projects/login.tgz} version: 0.0.0 '@rush-temp/love-assets@file:projects/love-assets.tgz': @@ -4509,7 +4519,7 @@ packages: version: 0.0.0 '@rush-temp/model@file:projects/model.tgz': - resolution: {integrity: sha512-wnsVHHgMBB73uVsYY5S/UigD6u0BDFVUpirCToSOGy8IkQR5Q9YKjgiyAX3K+PFkBY7VaL1a/QoqWZEQCy97ow==, tarball: file:projects/model.tgz} + resolution: {integrity: sha512-MUWU7orOyyibYO9aRQaElxt3VrQFOHbr5qtC9sJYkp9F4PNEuM+/1aYUALDJiuqo14BleVOcNV28030yBXv8CA==, tarball: file:projects/model.tgz} version: 0.0.0 '@rush-temp/mongo@file:projects/mongo.tgz': @@ -4565,7 +4575,7 @@ packages: version: 0.0.0 '@rush-temp/pod-analytics-collector@file:projects/pod-analytics-collector.tgz': - resolution: {integrity: sha512-KuE4/ZuLMoEOqY2QyXv7AD0uPt5J7Uwt6jP70iWXcR2oET1+BhD2QTm/9MNsgOsnfiqp0Tdv06pfu7fb2A2Gcw==, tarball: file:projects/pod-analytics-collector.tgz} + resolution: {integrity: sha512-eCAWgfNhBAyArb1VyMKjMMCzWsoQcocLxyfurr6hM13JPqMMuK7no9z1MGHjd+mPkcvfpRGGh5I03XaPSe9gyg==, tarball: file:projects/pod-analytics-collector.tgz} version: 0.0.0 '@rush-temp/pod-backup@file:projects/pod-backup.tgz': @@ -4589,7 +4599,7 @@ packages: version: 0.0.0 '@rush-temp/pod-github@file:projects/pod-github.tgz': - resolution: {integrity: sha512-FPRCZ36WCkBXnk5ATqZKSIq0KfPHT0mYZpvpzLw1z5J74osXHxMoifo28Lb7bZcHDbXg6o6VOGHotBvPgCYUng==, tarball: file:projects/pod-github.tgz} + resolution: {integrity: sha512-Szu3mv8d6NPVJtTXbnxK4LvO0yCMhJeOjOxX/hQNjizR4bzuWA0RFmihTUxGiByqWhuwciVC5G7ZtjZdprATzQ==, tarball: file:projects/pod-github.tgz} version: 0.0.0 '@rush-temp/pod-gmail@file:projects/pod-gmail.tgz': @@ -4597,11 +4607,11 @@ packages: version: 0.0.0 '@rush-temp/pod-love@file:projects/pod-love.tgz': - resolution: {integrity: sha512-G70rzmJHk/en1EdQHEs2HEK9/D2IY0i/iF0WZtU2FWLtSzlJb5UC0hrpVfxD/aWcnfmmqcLQu+b2siYrnUFSmQ==, tarball: file:projects/pod-love.tgz} + resolution: {integrity: sha512-m+9dPMR45FJpQB+M5ka4PiJcTpV8COlQA5F36is2LkMLwnsDjcB94Sx+TFdIyjhVFpfB8l+t886mSPveHql4ag==, tarball: file:projects/pod-love.tgz} version: 0.0.0 '@rush-temp/pod-print@file:projects/pod-print.tgz': - resolution: {integrity: sha512-uH6mY0Z3/3bbAe+rGjr2WPJmavKnoNgKZ5KoLqToGGF+iDzZOZGLjd8OrwUI90Z3behW7PPIRuPDGiRmFGOtcw==, tarball: file:projects/pod-print.tgz} + resolution: {integrity: sha512-i0Y9ShIeieNqCNMCXax1CZTStiDTw9M156m7fCDSrNS6VV4v2e5jy/HyNLtpyXz/ZRlOKndBdyiKsls+5pDL8g==, tarball: file:projects/pod-print.tgz} version: 0.0.0 '@rush-temp/pod-server@file:projects/pod-server.tgz': @@ -4613,7 +4623,7 @@ packages: version: 0.0.0 '@rush-temp/pod-sign@file:projects/pod-sign.tgz': - resolution: {integrity: sha512-2CvZ6MihYZ/0DXWllTffUrvNyRjLJvvUkcT36EpVmgydo2F4guxAhrnsyaivxNPNwP8NuSy7JJ4uXXOYNeXCQg==, tarball: file:projects/pod-sign.tgz} + resolution: {integrity: sha512-CmOWryDSvF5qg0NTlkxT29gZP0qj0DimD25X2U6sQGtowtEytNvVmmg3cuSNTpwN+WR6+eli4JWyK6gathi+fQ==, tarball: file:projects/pod-sign.tgz} version: 0.0.0 '@rush-temp/pod-stats@file:projects/pod-stats.tgz': @@ -4793,7 +4803,7 @@ packages: version: 0.0.0 '@rush-temp/server-calendar-resources@file:projects/server-calendar-resources.tgz': - resolution: {integrity: sha512-Rpdy7Nx56rXrJ1apUsvlancAVa5USFSwXboZ8iexTJ4Jaebv1yf32j5LUo/fLKwpZyL9KaUHT+A4N+nJKoRVWg==, tarball: file:projects/server-calendar-resources.tgz} + resolution: {integrity: sha512-92TFFDbAxCmUu4V3IaNDOLhbEEqTCB5dRohzpV1nzmroO8r+JneZjdV1j8OPHJF65ml0uqciDFPOct+kMzx5fg==, tarball: file:projects/server-calendar-resources.tgz} version: 0.0.0 '@rush-temp/server-calendar@file:projects/server-calendar.tgz': @@ -4801,7 +4811,7 @@ packages: version: 0.0.0 '@rush-temp/server-chunter-resources@file:projects/server-chunter-resources.tgz': - resolution: {integrity: sha512-anzs0C7AXicSpf9F+jjqdokc3cZunw5O8gC0095fQmEoOW94OmvTyboGrx8iK3i+iYam4Z6Bkt60rXRo3JvXZQ==, tarball: file:projects/server-chunter-resources.tgz} + resolution: {integrity: sha512-kiR/o8nysn7cgV4YGZNzxBpqzpoTqXGPyF7fXO3MJ0aZPa46AWQPushZMQTnR9O7wFs3RUuHR7Bc6VsjlzqDqA==, tarball: file:projects/server-chunter-resources.tgz} version: 0.0.0 '@rush-temp/server-chunter@file:projects/server-chunter.tgz': @@ -4809,7 +4819,7 @@ packages: version: 0.0.0 '@rush-temp/server-client@file:projects/server-client.tgz': - resolution: {integrity: sha512-ixXx7EdCjt/KMKk6pVedAj/27hWqDwCBnHhmTJc694OR8wTEHyIaohP9t8gcjka/ugJRJJjOmAsF1sd8HphW4g==, tarball: file:projects/server-client.tgz} + resolution: {integrity: sha512-uJeov8Q516lmcb4ZfIRdibl6PHP+INgobu/cjGIJ2WyOQaj2qBiABAq0IehGFF6Vb9EGXbQTB2VUuDvbabWEBQ==, tarball: file:projects/server-client.tgz} version: 0.0.0 '@rush-temp/server-collaboration-resources@file:projects/server-collaboration-resources.tgz': @@ -4821,15 +4831,15 @@ packages: version: 0.0.0 '@rush-temp/server-contact-resources@file:projects/server-contact-resources.tgz': - resolution: {integrity: sha512-uBxxy/nVfaqq3u6dPETKCjoQOHCJDRGQFjaau9Lf2ThmJ7ZdxxIZcJ05CfJD4OSgQ1vgf13ZEB4OkDpzqQM5mw==, tarball: file:projects/server-contact-resources.tgz} + resolution: {integrity: sha512-6R2S1cbwLtDKCesTPG9tJVy7mJB6EJogkf8TDAv9RFPRqxBkzdQh6OlgkLQsRMgpuneoJjwmSxXFHhX3btVnug==, tarball: file:projects/server-contact-resources.tgz} version: 0.0.0 '@rush-temp/server-contact@file:projects/server-contact.tgz': - resolution: {integrity: sha512-wH/4AxAjJUi/wU5hR5gbRK1k46L4FW5ydkphun1gbkO/GtNeSSdYAU48PfxJ6MkHTp+Hn0Ww7C5UxHfHjFOjAw==, tarball: file:projects/server-contact.tgz} + resolution: {integrity: sha512-3mD30C7Q8n6xsDge6A0OHJUS6I+9gRI/mw8LyhI98B04o/wJ81fIAM4Fmg6hRPmL1hvgylkNr55Ui1IQr5Cksw==, tarball: file:projects/server-contact.tgz} version: 0.0.0 '@rush-temp/server-controlled-documents-resources@file:projects/server-controlled-documents-resources.tgz': - resolution: {integrity: sha512-atiXw3GqCij5FY7fbz+ZGWnfGVMrQmGOURz34OQbNGVoZRFVimUn7pdMApsU0MP3JZBWDEtairZYPGh2nvs5SA==, tarball: file:projects/server-controlled-documents-resources.tgz} + resolution: {integrity: sha512-OWXWbWiL83nG0kcP9KV1F8PXFxN/YqDaB5NVTmZA9YUwAJpjxNjtHK7VEsdcvldxOiJsi/aDX5vueIjtgYhK4Q==, tarball: file:projects/server-controlled-documents-resources.tgz} version: 0.0.0 '@rush-temp/server-controlled-documents@file:projects/server-controlled-documents.tgz': @@ -4893,7 +4903,7 @@ packages: version: 0.0.0 '@rush-temp/server-hr-resources@file:projects/server-hr-resources.tgz': - resolution: {integrity: sha512-dRJrMwYzzZFaZ64iHkfaKQjP4EAnbFTn9HonG2w59T0KBGSk1cQlldj1G0DObovPhgl3bTcTJyiFTZo8sEKqVQ==, tarball: file:projects/server-hr-resources.tgz} + resolution: {integrity: sha512-sKDUqKJLI+KIEHICNUuvsHH5aGiVd2dOODfwoKmroe3y2bGYFcwHqFmWhWXCX9UyJ5ovfLdh/niY3Bqtx5V26w==, tarball: file:projects/server-hr-resources.tgz} version: 0.0.0 '@rush-temp/server-hr@file:projects/server-hr.tgz': @@ -4921,7 +4931,7 @@ packages: version: 0.0.0 '@rush-temp/server-love-resources@file:projects/server-love-resources.tgz': - resolution: {integrity: sha512-v5wtIer348LTlHf515DVi+LYLwL2iWTmHWpVt0IRGZG459gmW0Sge546mo9Nt2N64uACSWW+LVza6x6rDqzVEQ==, tarball: file:projects/server-love-resources.tgz} + resolution: {integrity: sha512-C+m2Gxq+YZ2PBgCWEH8C4dWcFeWNwWha7rP5ugPzmt4E6InyZBBUVZD36j4hYJpVIiffWUvjJ5Qm9ImO4zplbw==, tarball: file:projects/server-love-resources.tgz} version: 0.0.0 '@rush-temp/server-love@file:projects/server-love.tgz': @@ -4929,7 +4939,7 @@ packages: version: 0.0.0 '@rush-temp/server-notification-resources@file:projects/server-notification-resources.tgz': - resolution: {integrity: sha512-sLf+6eAqeGkx+28mefySh7RB9mMnEh5Cuw/Q036PALbzj/1tihPIIxlst9kMZAYgGr7WRaZGDGK4Pq2yxfCrPQ==, tarball: file:projects/server-notification-resources.tgz} + resolution: {integrity: sha512-XfvqNRHjygDUvXhwdvyQ3w+lp3m1b1iE/p2vXBiylkk54PJU8I0Wot6rJEwnSYIUaPM6ZnvjL5ROoQDYrSue8w==, tarball: file:projects/server-notification-resources.tgz} version: 0.0.0 '@rush-temp/server-notification@file:projects/server-notification.tgz': @@ -5001,7 +5011,7 @@ packages: version: 0.0.0 '@rush-temp/server-time-resources@file:projects/server-time-resources.tgz': - resolution: {integrity: sha512-0arrBFlOQYDRxE3vokplD4aQt7KQ9ucaCbex949FJ+hLrjZClfVs5myB9+rzD4NSHX4AAceIzeHLhF5hKYmf+Q==, tarball: file:projects/server-time-resources.tgz} + resolution: {integrity: sha512-sIJIhhjtHcHXQgRvN2ek6YUOjcxrAakYD9UUBU4R07lzC+tKpyB5ypeWBweZuiRN9DLdAQKMwgijm4ez8Osb+A==, tarball: file:projects/server-time-resources.tgz} version: 0.0.0 '@rush-temp/server-time@file:projects/server-time.tgz': @@ -5013,11 +5023,11 @@ packages: version: 0.0.0 '@rush-temp/server-tool@file:projects/server-tool.tgz': - resolution: {integrity: sha512-AdfPlf8gfUG+0JXgz9ZqM2Bt/BwN7ghTniXpM/dAtSIiJds2MDNkzpkvgOYu5jy0MjUjxlC1oivI8eg6TNRlpQ==, tarball: file:projects/server-tool.tgz} + resolution: {integrity: sha512-XGLZjlEMa+MBHaEr8yUT/c2Rn41CSRIW6ihPJSwA9T0hYn+vkoIXKXV/n7+BnpomXBY58zxlOQE1rSYbBKrtDA==, tarball: file:projects/server-tool.tgz} version: 0.0.0 '@rush-temp/server-tracker-resources@file:projects/server-tracker-resources.tgz': - resolution: {integrity: sha512-Npz6QfpPLP18rNxIGzNvjKf4QEhT8ywVdlYNt7e4QGvA1zetFDava2Cg0QpN31LZue8pTDCC3T94kBQMUiUfqg==, tarball: file:projects/server-tracker-resources.tgz} + resolution: {integrity: sha512-K6rsjo4rNef5+c7LhGFy+mwfwRb4b6YnwRzfP38gveTa0MA5CdMGTlmnaE5/OEciUwCRg77ePOyjA3giW/V1AA==, tarball: file:projects/server-tracker-resources.tgz} version: 0.0.0 '@rush-temp/server-tracker@file:projects/server-tracker.tgz': @@ -5041,11 +5051,11 @@ packages: version: 0.0.0 '@rush-temp/server-ws@file:projects/server-ws.tgz': - resolution: {integrity: sha512-9PP7kE8LXoZ+7r6OwLp26KT+JFAQtSwg5QP/awwXjcsV2t3yphDYKbdAJuVG5xlLfHW52Xn4NRbr0wFhfwS0sg==, tarball: file:projects/server-ws.tgz} + resolution: {integrity: sha512-5g3NGK0JdwcpXIRpsGN75fiev+4fpwl4V33V5rwzrLZzHUNE7CAzTtc/cFRxK5cZIVRwaaTNCYi+WCEfoFI7Tg==, tarball: file:projects/server-ws.tgz} version: 0.0.0 '@rush-temp/server@file:projects/server.tgz': - resolution: {integrity: sha512-PwdTpzGoD5SYmDTSD4hJT7ZkG2b05H030bVz5I+vz5f/AmgAzowxfxa0pbMujI6tEiU+o6hDiK25PQXn2xduHA==, tarball: file:projects/server.tgz} + resolution: {integrity: sha512-378GorQHM5ihU3dZLis+aq/JRjXeIgVOzIV/W3k27KB8EzoZGDbZQbDuKC07vAPxNBaoJ5yYcLTdGWx6XHnwzw==, tarball: file:projects/server.tgz} version: 0.0.0 '@rush-temp/setting-assets@file:projects/setting-assets.tgz': @@ -5053,7 +5063,7 @@ packages: version: 0.0.0 '@rush-temp/setting-resources@file:projects/setting-resources.tgz': - resolution: {integrity: sha512-XudJm31jzp8dEwS9TsP8lpxB2/yYjvafLULunIlgG46LdhQBfDG711m9w9gLb41Jj9O1quj2egHI87fHfmwrrg==, tarball: file:projects/setting-resources.tgz} + resolution: {integrity: sha512-f1+pi6AyYD3Vr5UbZfqt4sOwCRC3ZL5x3EQBDUqD+XKx/XbVfCMF3fp7E1Jw+tA3v/Rp+b7w+DBUQnxa4nVRDQ==, tarball: file:projects/setting-resources.tgz} version: 0.0.0 '@rush-temp/setting@file:projects/setting.tgz': @@ -5257,7 +5267,7 @@ packages: version: 0.0.0 '@rush-temp/workbench-resources@file:projects/workbench-resources.tgz': - resolution: {integrity: sha512-315FeO65RF4AACdkoe6rPuva+WRyhF48FvyYLOg558Te9tZqwEvKjwR/KWrgZPRkm8mcemw9xWBhirfO9Grkqw==, tarball: file:projects/workbench-resources.tgz} + resolution: {integrity: sha512-wnOyuF9Gv2lBWbszQxpfxQ1fF15tG9toi26+Bt8IONSejZsaDrQw/rXhZsW9wgWbtoRnD17Gjg5G2YUPzWZvUg==, tarball: file:projects/workbench-resources.tgz} version: 0.0.0 '@rush-temp/workbench@file:projects/workbench.tgz': @@ -5265,7 +5275,7 @@ packages: version: 0.0.0 '@rush-temp/workspace-service@file:projects/workspace-service.tgz': - resolution: {integrity: sha512-Rj6gppjTr+cKVJqzPxQ7PZy4reEXSEvjJouyZb7e3NApoh7WtqBWYau+N9i/jQw5C1dyGcH3HHaRBoFH9ZZcHA==, tarball: file:projects/workspace-service.tgz} + resolution: {integrity: sha512-tQ49Q9CrzYAD3cibL+LuX1mf9lZ3kQwLJhviHhodOKqx9qmdrMCTBLt1DIJVC96AjblKta8Afa1Y2Wl8FIYhtw==, tarball: file:projects/workspace-service.tgz} version: 0.0.0 '@selderee/plugin-htmlparser2@0.11.0': @@ -6204,6 +6214,9 @@ packages: '@types/pg-pool@2.0.6': resolution: {integrity: sha512-TaAUE5rq2VQYxab5Ts7WZhKNmuN78Q6PiFonTDdpbx8a1H0M1vhy3rhiMjl+e2iHmogyMw7jZF4FrE6eJUy5HQ==} + '@types/pg@8.11.11': + resolution: {integrity: sha512-kGT1qKM8wJQ5qlawUrEkXgvMSXoV213KfMGXcwfDwUIfUHXqXYXOfS1nE1LINRJVVVx5wCm70XnFlMHaIcQAfw==} + '@types/pg@8.6.1': resolution: {integrity: sha512-1Kc4oAGzAl7uqUStZCDvaLFqZrW9qWSjXOmBfdgyBP5La7Us6Mg4GBvRlSoaZMhQF/zSj1C8CtKMBkoiT8eL8w==} @@ -11201,6 +11214,10 @@ packages: resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} engines: {node: '>=4.0.0'} + pg-numeric@1.0.2: + resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==} + engines: {node: '>=4'} + pg-protocol@1.7.0: resolution: {integrity: sha512-hTK/mE36i8fDDhgDFjy6xNOG+LCorxLG3WO17tku+ij6sVHXh1jQUJ8hYAnRhNla4QVD2H8er/FOjc/+EgC6yQ==} @@ -11208,6 +11225,10 @@ packages: resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} engines: {node: '>=4'} + pg-types@4.0.2: + resolution: {integrity: sha512-cRL3JpS3lKMGsKaWndugWQoLOCoP+Cic8oseVcbr0qhPzYD5DWXK+RZ9LY9wxRf7RQia4SCwQlXk0q6FCPrVng==} + engines: {node: '>=10'} + phin@2.9.3: resolution: {integrity: sha512-CzFr90qM24ju5f88quFC/6qohjC144rehe5n6DH900lgXmUe86+xCKc10ev56gRKC4/BkHUoG4uSiQgBiIXwDA==} deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. @@ -11376,18 +11397,37 @@ packages: resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} engines: {node: '>=4'} + postgres-array@3.0.2: + resolution: {integrity: sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==} + engines: {node: '>=12'} + postgres-bytea@1.0.0: resolution: {integrity: sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==} engines: {node: '>=0.10.0'} + postgres-bytea@3.0.0: + resolution: {integrity: sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==} + engines: {node: '>= 6'} + postgres-date@1.0.7: resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} engines: {node: '>=0.10.0'} + postgres-date@2.1.0: + resolution: {integrity: sha512-K7Juri8gtgXVcDfZttFKVmhglp7epKb1K4pgrkLxehjqkrgPhfG6OO8LHLkfaqkbpjNRnra018XwAr1yQFWGcA==} + engines: {node: '>=12'} + postgres-interval@1.2.0: resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} engines: {node: '>=0.10.0'} + postgres-interval@3.0.0: + resolution: {integrity: sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==} + engines: {node: '>=12'} + + postgres-range@1.1.4: + resolution: {integrity: sha512-i/hbxIE9803Alj/6ytL7UHQxRvZkI9O4Sy+J3HGc4F4oo/2eQAjTSNJ0bfxyse3bH0nuVesCk+3IRLaMtG3H6w==} + postgres@3.4.5: resolution: {integrity: sha512-cDWgoah1Gez9rN3H4165peY9qfpEo+SA61oQv65O3cRUE1pOEoJWwddwcqKE8XZYjbblOJlYDlLV4h67HrEVDg==} engines: {node: '>=12'} @@ -15711,6 +15751,32 @@ snapshots: '@remirror/core-constants@3.0.0': {} + '@rush-temp/account-client@file:projects/account-client.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': + dependencies: + '@types/jest': 29.5.12 + '@types/node': 20.11.19 + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.7.3) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.7.3) + cross-env: 7.0.3 + esbuild: 0.24.2 + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.7.3) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) + prettier: 3.2.5 + ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.7.3) + typescript: 5.7.3 + transitivePeerDependencies: + - '@babel/core' + - '@jest/types' + - babel-jest + - babel-plugin-macros + - node-notifier + - supports-color + - ts-node + '@rush-temp/account-service@file:projects/account-service.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)': dependencies: '@koa/cors': 5.0.0 @@ -15757,12 +15823,11 @@ snapshots: '@rush-temp/account@file:projects/account.tgz(@babel/core@7.23.9)(@jest/types@29.6.3)(@types/node@20.11.19)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(gcp-metadata@5.3.0(encoding@0.1.13))(snappy@7.2.2)(socks@2.8.3)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3))': dependencies: - '@types/crypto-js': 4.2.2 '@types/jest': 29.5.12 '@types/otp-generator': 4.0.2 + '@types/pg': 8.11.11 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) - crypto-js: 4.2.0 eslint: 8.56.0 eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.3.3))(eslint@8.56.0)(typescript@5.3.3))(eslint-plugin-import@2.29.1(eslint@8.56.0))(eslint-plugin-n@15.7.0(eslint@8.56.0))(eslint-plugin-promise@6.1.1(eslint@8.56.0))(eslint@8.56.0)(typescript@5.3.3) eslint-plugin-import: 2.29.1(eslint@8.56.0) @@ -17070,6 +17135,7 @@ snapshots: eslint-plugin-promise: 6.1.1(eslint@8.56.0) jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)) prettier: 3.2.5 + svelte: 4.2.19 ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@20.11.19)(ts-node@10.9.2(@types/node@20.11.19)(typescript@5.3.3)))(typescript@5.3.3) typescript: 5.3.3 transitivePeerDependencies: @@ -27033,7 +27099,13 @@ snapshots: '@types/pg-pool@2.0.6': dependencies: - '@types/pg': 8.6.1 + '@types/pg': 8.11.11 + + '@types/pg@8.11.11': + dependencies: + '@types/node': 20.11.19 + pg-protocol: 1.7.0 + pg-types: 4.0.2 '@types/pg@8.6.1': dependencies: @@ -33237,6 +33309,8 @@ snapshots: pg-int8@1.0.1: {} + pg-numeric@1.0.2: {} + pg-protocol@1.7.0: {} pg-types@2.2.0: @@ -33247,6 +33321,16 @@ snapshots: postgres-date: 1.0.7 postgres-interval: 1.2.0 + pg-types@4.0.2: + dependencies: + pg-int8: 1.0.1 + pg-numeric: 1.0.2 + postgres-array: 3.0.2 + postgres-bytea: 3.0.0 + postgres-date: 2.1.0 + postgres-interval: 3.0.0 + postgres-range: 1.1.4 + phin@2.9.3: {} phin@3.7.1: @@ -33400,14 +33484,26 @@ snapshots: postgres-array@2.0.0: {} + postgres-array@3.0.2: {} + postgres-bytea@1.0.0: {} + postgres-bytea@3.0.0: + dependencies: + obuf: 1.1.2 + postgres-date@1.0.7: {} + postgres-date@2.1.0: {} + postgres-interval@1.2.0: dependencies: xtend: 4.0.2 + postgres-interval@3.0.0: {} + + postgres-range@1.1.4: {} + postgres@3.4.5: {} posthog-js@1.122.0: diff --git a/desktop/src/main/backup.ts b/desktop/src/main/backup.ts index 73bad40eaf..d657e731b8 100644 --- a/desktop/src/main/backup.ts +++ b/desktop/src/main/backup.ts @@ -1,5 +1,5 @@ import client, { clientId } from '@hcengineering/client' -import { getWorkspaceId, MeasureMetricsContext, type BackupClient, type Client } from '@hcengineering/core' +import { MeasureMetricsContext, WorkspaceIds, type BackupClient, type Client } from '@hcengineering/core' import { addLocation, getResource, setMetadata } from '@hcengineering/platform' import WebSocket from 'ws' @@ -31,16 +31,15 @@ export async function createClient ( return (await clientFactory(token, transactorUrl)) as unknown as Client & BackupClient } -async function doBackup (dirName: string, token: string, endpoint: string, workspace: string, notify: (command: string, ...args: any[]) => void, backupHugeFiles: boolean): Promise { +async function doBackup (dirName: string, token: string, endpoint: string, wsIds: WorkspaceIds, notify: (command: string, ...args: any[]) => void, backupHugeFiles: boolean): Promise { notify('backup', 0) const ctx = new MeasureMetricsContext('backup', {}) const storage = await createFileBackupStorage(dirName) - const wsid = getWorkspaceId(workspace) const client = await createClient(endpoint, token) try { - ctx.info('do backup', { workspace, endpoint }) - await backup(ctx, endpoint, wsid, storage, { + ctx.info('do backup', { ...wsIds, endpoint }) + await backup(ctx, endpoint, wsIds, storage, { force: true, freshBackup: false, clean: false, @@ -65,7 +64,7 @@ async function doBackup (dirName: string, token: string, endpoint: string, works notify('backup', 100) } -export function startBackup (window: BrowserWindow, token: string, endpoint: string, workspace: string, cmd: (command: string, ...args: any[]) => void): void { +export function startBackup (window: BrowserWindow, token: string, endpoint: string, wsIds: WorkspaceIds, cmd: (command: string, ...args: any[]) => void): void { void dialog .showOpenDialog(window, { properties: ['openDirectory'], @@ -94,7 +93,7 @@ export function startBackup (window: BrowserWindow, token: string, endpoint: str runningBackup = { notify: cmd } - void doBackup(response.filePaths[0], token, endpoint, workspace, cmd, backupKind.checkboxChecked) + void doBackup(response.filePaths[0], token, endpoint, wsIds, cmd, backupKind.checkboxChecked) }) } }) diff --git a/desktop/src/main/start.ts b/desktop/src/main/start.ts index b4e96f7ac0..fee9be59bc 100644 --- a/desktop/src/main/start.ts +++ b/desktop/src/main/start.ts @@ -424,10 +424,10 @@ autoUpdater.on('update-downloaded', (info) => { autoUpdater.quitAndInstall() }) -ipcMain.on('start-backup', (event, token, endpoint, workspace) => { - console.log('start backup', token, endpoint, workspace) +ipcMain.on('start-backup', (event, token, endpoint, wsIds) => { + console.log('start backup', token, endpoint, wsIds) if (mainWindow != null) { - startBackup(mainWindow, token, endpoint, workspace, (cmd: string, ...args: any[]) => { + startBackup(mainWindow, token, endpoint, wsIds, (cmd: string, ...args: any[]) => { mainWindow?.webContents.send(cmd, ...args) }) } diff --git a/desktop/src/ui/index.ts b/desktop/src/ui/index.ts index 8733648dde..ecb4675626 100644 --- a/desktop/src/ui/index.ts +++ b/desktop/src/ui/index.ts @@ -61,13 +61,14 @@ window.addEventListener('DOMContentLoaded', () => { const tokens = fetchMetadataLocalStorage(login.metadata.LoginTokens) if (tokens !== null) { const loc = getCurrentLocation() - loc.path.splice(1, 1) + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete tokens[loc.path[1]] setMetadataLocalStorage(login.metadata.LoginTokens, tokens) } setMetadata(presentation.metadata.Token, null) setMetadataLocalStorage(login.metadata.LastToken, null) setMetadataLocalStorage(login.metadata.LoginEndpoint, null) - setMetadataLocalStorage(login.metadata.LoginEmail, null) + setMetadataLocalStorage(login.metadata.LoginAccount, null) void closeClient().then(() => { navigate({ path: [loginId] }) }) @@ -104,10 +105,17 @@ window.addEventListener('DOMContentLoaded', () => { // We need to obtain current token and endpoint and trigger backup const token = getMetadata(presentation.metadata.Token) const endpoint = getMetadata(presentation.metadata.Endpoint) - const workspace = getMetadata(presentation.metadata.WorkspaceId) + const workspaceUuid = getMetadata(presentation.metadata.WorkspaceUuid) + // const workspaceDataId = getMetadata(presentation.metadata.WorkspaceDataId) + // const workspaceUrl = getMetadata(presentation.metadata.WorkspaceUrl) + // const wsIds = { + // uuid: workspaceUuid, + // dataId: workspaceDataId, + // url: workspaceUrl + // } if (isOwnerOrMaintainer()) { - if (token != null && endpoint != null && workspace != null) { - // ipcMain.startBackup(token, endpoint, workspace) + if (token != null && endpoint != null && workspaceUuid != null) { + // ipcMain.startBackup(token, endpoint, wsIds) closePopup() closePanel() const loc = getCurrentResolvedLocation() diff --git a/desktop/src/ui/notifications.ts b/desktop/src/ui/notifications.ts index ff8e178eac..127d82d00c 100644 --- a/desktop/src/ui/notifications.ts +++ b/desktop/src/ui/notifications.ts @@ -1,4 +1,4 @@ -import contact, { PersonAccount, formatName } from '@hcengineering/contact' +import { getPersonBySocialId, formatName } from '@hcengineering/contact' import { Ref, TxOperations } from '@hcengineering/core' import notification, { DocNotifyContext, CommonInboxNotification, ActivityInboxNotification, InboxNotification } from '@hcengineering/notification' import { IntlString, addEventListener, translate } from '@hcengineering/platform' @@ -67,14 +67,7 @@ async function hydrateNotificationAsYouCan (lastNotification: InboxNotification) body: '' } - const account = await client.getModel().findOne(contact.class.PersonAccount, { _id: lastNotification.modifiedBy as Ref }) - - if (account == null) { - return noPersonData - } - - const person = await client.findOne(contact.class.Person, { _id: account.person }) - + const person = await getPersonBySocialId(client, lastNotification.modifiedBy) if (person == null) { return noPersonData } @@ -122,7 +115,7 @@ export function configureNotifications (): void { let initTimestamp = 0 const notificationHistory = new Map() - addEventListener(workbench.event.NotifyConnection, async (event, account: PersonAccount) => { + addEventListener(workbench.event.NotifyConnection, async () => { client = getClient() const electronAPI: IPCMainExposed = (window as any).electron diff --git a/desktop/src/ui/platform.ts b/desktop/src/ui/platform.ts index f8a31b0521..5cb0b7a5cf 100644 --- a/desktop/src/ui/platform.ts +++ b/desktop/src/ui/platform.ts @@ -340,7 +340,7 @@ export async function configurePlatform (): Promise { initThemeStore() - addEventListener(workbench.event.NotifyConnection, async (evt) => { + addEventListener(workbench.event.NotifyConnection, async () => { await ipcMain.setFrontCookie( config.FRONT_URL, presentation.metadata.Token.replaceAll(':', '-'), diff --git a/desktop/src/ui/preload.ts b/desktop/src/ui/preload.ts index fdf81dd55b..c406608311 100644 --- a/desktop/src/ui/preload.ts +++ b/desktop/src/ui/preload.ts @@ -145,6 +145,6 @@ const expose: IPCMainExposed = { getScreenAccess: () => ipcRenderer.invoke('get-screen-access'), getScreenSources: () => ipcRenderer.invoke('get-screen-sources'), cancelBackup: () => { ipcRenderer.send('cancel-backup') }, - startBackup: (token, endpoint, workspace) => { ipcRenderer.send('start-backup', token, endpoint, workspace) } + startBackup: (token, endpoint, wsIds) => { ipcRenderer.send('start-backup', token, endpoint, wsIds) } } contextBridge.exposeInMainWorld('electron', expose) diff --git a/dev/doc-import-tool/src/config.ts b/dev/doc-import-tool/src/config.ts index 69ea6decbb..fbd23ed88b 100644 --- a/dev/doc-import-tool/src/config.ts +++ b/dev/doc-import-tool/src/config.ts @@ -1,5 +1,5 @@ import { Employee } from '@hcengineering/contact' -import { Ref, WorkspaceId } from '@hcengineering/core' +import { Ref, WorkspaceDataId, WorkspaceUuid } from '@hcengineering/core' import { DocumentSpace } from '@hcengineering/controlled-documents' import { StorageAdapter } from '@hcengineering/server-core' @@ -11,7 +11,8 @@ export interface Config { collaborator?: string collaboratorURL: string uploadURL: string - workspaceId: WorkspaceId + workspaceId: WorkspaceUuid + workspaceDataId?: WorkspaceDataId owner: Ref backend: HtmlConversionBackend space: Ref diff --git a/dev/doc-import-tool/src/import.ts b/dev/doc-import-tool/src/import.ts index c76a8d0b9c..4b891bf271 100644 --- a/dev/doc-import-tool/src/import.ts +++ b/dev/doc-import-tool/src/import.ts @@ -20,9 +20,10 @@ import core, { MeasureContext, Ref, TxOperations, + type WorkspaceDataId, generateId, makeDocCollabId, - systemAccountEmail, + systemAccountUuid, type Blob } from '@hcengineering/core' import { createClient, getTransactorEndpoint } from '@hcengineering/server-client' @@ -40,9 +41,9 @@ export default async function importExtractedFile ( extractedFile: ExtractedFile ): Promise { const { workspaceId } = config - const token = generateToken(systemAccountEmail, workspaceId) + const token = generateToken(systemAccountUuid, workspaceId) const transactorUrl = await getTransactorEndpoint(token, 'external') - console.log(`Connecting to transactor: ${transactorUrl} (ws: '${workspaceId.name}')`) + console.log(`Connecting to transactor: ${transactorUrl} (ws: '${workspaceId}')`) const connection = (await createClient(transactorUrl, token)) as CoreClient & BackupClient try { @@ -252,7 +253,7 @@ export async function processImages ( const dom = parseDocument(section.content) const imageNodes = findAll((n) => n.tagName === 'img', dom.children) - const { storageAdapter, workspaceId, uploadURL } = config + const { storageAdapter, workspaceId, workspaceDataId, uploadURL } = config const imageUploads = imageNodes.map(async (img) => { const src = img.attribs.src @@ -272,7 +273,14 @@ export async function processImages ( // upload const uuid = generateId() - await storageAdapter.put(ctx, workspaceId, uuid, fileContents, mimeType, fileSize) + await storageAdapter.put( + ctx, + workspaceDataId ?? (workspaceId as unknown as WorkspaceDataId), + uuid, + fileContents, + mimeType, + fileSize + ) // attachment const attachmentId: Ref = generateId() diff --git a/dev/doc-import-tool/src/index.ts b/dev/doc-import-tool/src/index.ts index 87f96732bb..cd9dbbf1b8 100644 --- a/dev/doc-import-tool/src/index.ts +++ b/dev/doc-import-tool/src/index.ts @@ -14,7 +14,7 @@ // import { Employee } from '@hcengineering/contact' import documents, { DocumentSpace } from '@hcengineering/controlled-documents' -import { MeasureMetricsContext, Ref, getWorkspaceId, systemAccountEmail } from '@hcengineering/core' +import { MeasureMetricsContext, Ref, systemAccountUuid, WorkspaceUuid, type WorkspaceDataId } from '@hcengineering/core' import { setMetadata } from '@hcengineering/platform' import serverClientPlugin from '@hcengineering/server-client' import { type StorageAdapter } from '@hcengineering/server-core' @@ -74,13 +74,14 @@ export function docImportTool (): void { .description('import doc into workspace') .option('-s|--spec ', 'Specification file') .option('-b|--backend ', 'Conversion backend', 'pandoc') + .option('-d|--dataId ', 'Workspace data ID') .option('--space ', 'Doc space ID', documents.space.QualityDocuments) .action( async ( doc: string, - workspace: string, + workspace: WorkspaceUuid, owner: Ref, - cmd: { backend: string, space: Ref, spec?: string } + cmd: { backend: string, workspaceDataId?: WorkspaceDataId, space: Ref, spec?: string } ) => { console.log( `Importing document '${doc}' into workspace '${workspace}', owner: ${JSON.stringify(owner)}, spec: ${ @@ -89,11 +90,12 @@ export function docImportTool (): void { ) await withStorage(async (storageAdapter) => { - const workspaceId = getWorkspaceId(workspace) + const workspaceId = workspace const config: Config = { doc, workspaceId, + workspaceDataId: cmd.workspaceDataId, owner, backend: getBackend(cmd.backend), specFile: cmd.spec, @@ -102,7 +104,7 @@ export function docImportTool (): void { storageAdapter, collaboratorURL: collaboratorUrl, collaborator, - token: generateToken(systemAccountEmail, workspaceId) + token: generateToken(systemAccountUuid, workspaceId, { service: 'import-tool' }) } await importDoc(ctx, config) diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 8405dbc581..829f047309 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -19,18 +19,6 @@ services: ports: - 27017:27017 restart: unless-stopped - postgres: - image: postgres - container_name: postgres - extra_hosts: - - "host.docker.internal:host-gateway" - environment: - - POSTGRES_PASSWORD=example - volumes: - - dbpg:/data/db - ports: - - 5432:5432 - restart: unless-stopped cockroach: image: cockroachdb/cockroach:latest-v24.2 ports: @@ -86,11 +74,11 @@ services: - ACCOUNT_PORT=3000 - SERVER_SECRET=secret - STATS_URL=http://host.docker.internal:4900 - # - DB_URL=postgresql://postgres:example@postgres:5432 + # - DB_URL=postgresql://root@host.docker.internal:26257/defaultdb?sslmode=disable - DB_URL=${MONGO_URL} # - DB_NS=account-2 # Pass only one region to disallow selection for new workspaces.Ø - - REGION_INFO=|Mongo;pg|Postgres;cockroach|CockroachDB + - REGION_INFO=|Mongo;cockroach|CockroachDB # - REGION_INFO=cockroach|CockroachDB - TRANSACTOR_URL=ws://host.docker.internal:3333,ws://host.docker.internal:3331;;pg,ws://host.docker.internal:3332;;cockroach, - SES_URL= @@ -142,34 +130,6 @@ services: - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG} - BACKUP_BUCKET=${BACKUP_BUCKET_NAME} restart: unless-stopped - workspacepg: - image: hardcoreeng/workspace - extra_hosts: - - 'host.docker.internal:host-gateway' - links: - - postgres - - minio - - stats - volumes: - - ./branding.json:/var/cfg/branding.json - environment: - - WS_OPERATION=all+backup - - SERVER_SECRET=secret - - DB_URL=postgresql://postgres:example@postgres:5432 - - STATS_URL=http://host.docker.internal:4900 - - SES_URL= - - REGION=pg - - STORAGE_CONFIG=${STORAGE_CONFIG} - - FRONT_URL=http://host.docker.internal:8087 - - RESERVED_DB_NAMES=telegram,gmail,github - - MODEL_ENABLED=* - - ACCOUNTS_URL=http://host.docker.internal:3000 - - BRANDING_PATH=/var/cfg/branding.json - # - PARALLEL=2 - # - INIT_WORKSPACE=onboarding - - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG} - - BACKUP_BUCKET=${BACKUP_BUCKET_NAME} - restart: unless-stopped workspace_cockroach: image: hardcoreeng/workspace extra_hosts: @@ -286,43 +246,9 @@ services: - ACCOUNTS_URL=http://host.docker.internal:3000 - LAST_NAME_FIRST=true - BRANDING_PATH=/var/cfg/branding.json - - SUPPORT_WORKSPACE=support + - SUPPORT_WORKSPACE=09b65664-b3c3-4ea8-b9a1-c9688bde17f0 - AI_BOT_URL=http://host.docker.internal:4010 restart: unless-stopped - transactor_pg: - image: hardcoreeng/transactor - extra_hosts: - - 'host.docker.internal:host-gateway' - links: - - postgres - - minio - - account - - stats - # - apm-server - ports: - - 3331:3331 - volumes: - - ./branding.json:/var/cfg/branding.json - environment: - # - SERVER_PROVIDER=uweb - # - UWS_HTTP_MAX_HEADERS_SIZE="32768" - # - UV_THREADPOOL_SIZE=10 - - SERVER_PORT=3331 - - SERVER_SECRET=secret - - ENABLE_COMPRESSION=true - - FULLTEXT_URL=http://host.docker.internal:4701 - - STATS_URL=http://host.docker.internal:4900 - - DB_URL=postgresql://postgres:example@postgres:5432 - - METRICS_CONSOLE=false - - METRICS_FILE=metrics.txt - - STORAGE_CONFIG=${STORAGE_CONFIG} - - FRONT_URL=http://host.docker.internal:8087 - # - APM_SERVER_URL=http://apm-server:8200 - - SES_URL='' - - ACCOUNTS_URL=http://host.docker.internal:3000 - - LAST_NAME_FIRST=true - - BRANDING_PATH=/var/cfg/branding.json - restart: unless-stopped transactor_cockroach: image: hardcoreeng/transactor extra_hosts: @@ -396,26 +322,6 @@ services: - STATS_URL=http://host.docker.internal:4900 - REKONI_URL=http://host.docker.internal:4004 - ACCOUNTS_URL=http://host.docker.internal:3000 - fulltext_pg: - image: hardcoreeng/fulltext - extra_hosts: - - 'host.docker.internal:host-gateway' - restart: unless-stopped - links: - - elastic - - postgres - ports: - - 4701:4701 - environment: - - PORT=4701 - - SERVER_SECRET=secret - - DB_URL=postgresql://postgres:example@postgres:5432 - - FULLTEXT_DB_URL=http://host.docker.internal:9200 - - ELASTIC_INDEX_NAME=local_storage_index - - STORAGE_CONFIG=${STORAGE_CONFIG} - - STATS_URL=http://host.docker.internal:4900 - - REKONI_URL=http://host.docker.internal:4004 - - ACCOUNTS_URL=http://host.docker.internal:3000 fulltext_cockroach: image: hardcoreeng/fulltext extra_hosts: @@ -447,6 +353,7 @@ services: - SECRET=secret - STORAGE_CONFIG=${STORAGE_CONFIG} - STATS_URL=http://host.docker.internal:4900 + - ACCOUNTS_URL=http://host.docker.internal:3000 sign: image: hardcoreeng/sign extra_hosts: @@ -481,7 +388,7 @@ services: - 'MONGO_OPTIONS={"appName":"analytics","maxPoolSize":1}' - SERVICE_ID=analytics-collector-service - ACCOUNTS_URL=http://host.docker.internal:3000 - - SUPPORT_WORKSPACE=support + - SUPPORT_WORKSPACE=09b65664-b3c3-4ea8-b9a1-c9688bde17f0 - STATS_URL=http://host.docker.internal:4900 aiBot: image: hardcoreeng/ai-bot @@ -494,7 +401,7 @@ services: - SERVER_SECRET=secret - MONGO_URL=${MONGO_URL} - ACCOUNTS_URL=http://host.docker.internal:3000 - - SUPPORT_WORKSPACE=support + - SUPPORT_WORKSPACE=09b65664-b3c3-4ea8-b9a1-c9688bde17f0 - STORAGE_CONFIG=${STORAGE_CONFIG} - FIRST_NAME=Jolie - LAST_NAME=AI diff --git a/dev/import-tool/src/index.ts b/dev/import-tool/src/index.ts index 31595b29db..22603b7fdf 100644 --- a/dev/import-tool/src/index.ts +++ b/dev/import-tool/src/index.ts @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // -import { concatLink, TxOperations } from '@hcengineering/core' +import { buildSocialIdString, concatLink, SocialIdType, TxOperations } from '@hcengineering/core' import { ClickupImporter, defaultDocumentPreprocessors, @@ -25,12 +25,7 @@ import { type Logger } from '@hcengineering/importer' import { setMetadata } from '@hcengineering/platform' -import serverClientPlugin, { - createClient, - getUserWorkspaces, - login, - selectWorkspace -} from '@hcengineering/server-client' +import serverClientPlugin, { createClient, getAccountClient } from '@hcengineering/server-client' import { program } from 'commander' import { readFileSync } from 'fs' import * as yaml from 'js-yaml' @@ -79,32 +74,34 @@ export function importTool (): void { console.log('Setting up Accounts URL: ', config.ACCOUNTS_URL) setMetadata(serverClientPlugin.metadata.Endpoint, config.ACCOUNTS_URL) console.log('Trying to login user: ', user) - const userToken = await login(user, password, workspaceUrl) - if (userToken === undefined) { + const unauthAccountClient = getAccountClient() + const { account, token } = await unauthAccountClient.login(user, password) + if (token === undefined || account === undefined) { console.log('Login failed for user: ', user) return } console.log('Looking for workspace: ', workspaceUrl) - const allWorkspaces = await getUserWorkspaces(userToken) - const workspaces = allWorkspaces.filter((ws) => ws.workspaceUrl === workspaceUrl) + const accountClient = getAccountClient(token) + const allWorkspaces = await accountClient.getUserWorkspaces() + const workspaces = allWorkspaces.filter((ws) => ws.url === workspaceUrl) if (workspaces.length < 1) { console.log('Workspace not found: ', workspaceUrl) return } console.log('Workspace found') - const selectedWs = await selectWorkspace(userToken, workspaces[0].workspace) + const selectedWs = await accountClient.selectWorkspace(workspaces[0].url) console.log(selectedWs) console.log('Connecting to Transactor URL: ', selectedWs.endpoint) const connection = await createClient(selectedWs.endpoint, selectedWs.token) - const acc = connection.getModel().getAccountByEmail(user) - if (acc === undefined) { - console.log('Account not found for email: ', user) - return - } - const client = new TxOperations(connection, acc._id) - const fileUploader = new FrontFileUploader(getFrontUrl(), selectedWs.workspaceId, selectedWs.token) + const client = new TxOperations(connection, buildSocialIdString({ type: SocialIdType.EMAIL, value: user })) + const fileUploader = new FrontFileUploader( + getFrontUrl(), + selectedWs.workspace, + selectedWs.workspaceDataId ?? selectedWs.workspace, + selectedWs.token + ) try { await f(client, fileUploader) } catch (err: any) { diff --git a/dev/local-mongo/docker-compose.yaml b/dev/local-mongo/docker-compose.yaml index 234b064008..8ff0ce8b68 100644 --- a/dev/local-mongo/docker-compose.yaml +++ b/dev/local-mongo/docker-compose.yaml @@ -193,7 +193,7 @@ services: - 'MONGO_OPTIONS={"appName":"analytics","maxPoolSize":1}' - SERVICE_ID=analytics-collector-service - ACCOUNTS_URL=http://account:3000 - - SUPPORT_WORKSPACE=support + - SUPPORT_WORKSPACE=09b65664-b3c3-4ea8-b9a1-c9688bde17f0 deploy: resources: limits: @@ -205,7 +205,7 @@ services: - SERVER_SECRET=secret - MONGO_URL=mongodb://host.docker.internal:27017 - ACCOUNTS_URL=http://account:3000 - - SUPPORT_WORKSPACE=support + - SUPPORT_WORKSPACE=09b65664-b3c3-4ea8-b9a1-c9688bde17f0 - FIRST_NAME=Jolie - LAST_NAME=AI - PASSWORD=password diff --git a/dev/prod/public/branding.json b/dev/prod/public/branding.json index 8a9bcbf7a9..1efd403c58 100644 --- a/dev/prod/public/branding.json +++ b/dev/prod/public/branding.json @@ -64,6 +64,7 @@ "defaultApplication": "documents", "defaultSpace": "documents:space:QualityDocuments", "defaultSpecial": "", + "lastNameFirst": "true", "links": [ { "rel": "shortcut icon", diff --git a/dev/tool/package.json b/dev/tool/package.json index 4ab6044cbb..d9c1b9d481 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -20,9 +20,8 @@ "docker:abuild": "docker build -t hardcoreeng/tool . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/tool", "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/tool staging", "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/tool", - "run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --expose-gc --max-old-space-size=18000 ./bundle/bundle.js", - "run-local-pg": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=postgresql://postgres:example@localhost:5432 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --expose-gc --max-old-space-size=18000 ./bundle/bundle.js", - "run-local-cr": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3332 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=postgresql://root@host.docker.internal:26257/defaultdb?sslmode=disable TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --expose-gc --max-old-space-size=18000 ./bundle/bundle.js", + "run-local": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4700 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --expose-gc --max-old-space-size=18000 ./bundle/bundle.js", + "run-local-cr": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret FULLTEXT_URL=http://localhost:4702 ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3332 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=postgresql://root@host.docker.internal:26257/defaultdb?sslmode=disable TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --expose-gc --max-old-space-size=18000 ./bundle/bundle.js", "run-local-brk": "rush bundle --to @hcengineering/tool >/dev/null && cross-env SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000 TRANSACTOR_URL=ws://localhost:3333 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ACCOUNT_DB_URL=mongodb://localhost:27017 DB_URL=mongodb://localhost:27017 TELEGRAM_DATABASE=telegram-service REKONI_URL=http://localhost:4004 MODEL_VERSION=$(node ../../common/scripts/show_version.js) GIT_REVISION=$(git describe --all --long) node --inspect-brk --enable-source-maps --max-old-space-size=18000 ./bundle/bundle.js", "run": "rush bundle --to @hcengineering/tool >/dev/null && cross-env node --max-old-space-size=8000 ./bundle/bundle.js", "upgrade": "rushx run-local upgrade", diff --git a/dev/tool/src/account.ts b/dev/tool/src/account.ts index 24c2228876..fa91735811 100644 --- a/dev/tool/src/account.ts +++ b/dev/tool/src/account.ts @@ -1,202 +1,203 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ import { type Account, type AccountDB, - changeEmail, + // changeEmail, getAccount, - listWorkspacesPure, + // listWorkspacesPure, type Workspace } from '@hcengineering/account' -import core, { getWorkspaceId, type MeasureContext, systemAccountEmail, TxOperations } from '@hcengineering/core' +import core, { type MeasureContext, TxOperations } from '@hcengineering/core' import contact from '@hcengineering/model-contact' import { getTransactorEndpoint } from '@hcengineering/server-client' import { generateToken } from '@hcengineering/server-token' import { connect } from '@hcengineering/server-tool' -export async function renameAccount ( - ctx: MeasureContext, - db: AccountDB, - accountsUrl: string, - oldEmail: string, - newEmail: string -): Promise { - const account = await getAccount(db, oldEmail) - if (account == null) { - throw new Error("Account does'n exists") - } +// export async function renameAccount ( +// ctx: MeasureContext, +// db: AccountDB, +// accountsUrl: string, +// oldEmail: string, +// newEmail: string +// ): Promise { +// const account = await getAccount(db, oldEmail) +// if (account == null) { +// throw new Error("Account does'n exists") +// } - const newAccount = await getAccount(db, newEmail) - if (newAccount != null) { - throw new Error('New Account email already exists:' + newAccount?.email + ' ' + newAccount?._id?.toString()) - } +// const newAccount = await getAccount(db, newEmail) +// if (newAccount != null) { +// throw new Error('New Account email already exists:' + newAccount?.email + ' ' + newAccount?._id?.toString()) +// } - await changeEmail(ctx, db, account, newEmail) +// await changeEmail(ctx, db, account, newEmail) - await fixWorkspaceEmails(account, db, accountsUrl, oldEmail, newEmail) -} +// await fixWorkspaceEmails(account, db, accountsUrl, oldEmail, newEmail) +// } -export async function fixAccountEmails ( - ctx: MeasureContext, - db: AccountDB, - transactorUrl: string, - oldEmail: string, - newEmail: string -): Promise { - const account = await getAccount(db, newEmail) - if (account == null) { - throw new Error("Account does'n exists") - } +// export async function fixAccountEmails ( +// ctx: MeasureContext, +// db: AccountDB, +// transactorUrl: string, +// oldEmail: string, +// newEmail: string +// ): Promise { +// const account = await getAccount(db, newEmail) +// if (account == null) { +// throw new Error("Account does'n exists") +// } - await fixWorkspaceEmails(account, db, transactorUrl, oldEmail, newEmail) -} -async function fixWorkspaceEmails ( - account: Account, - db: AccountDB, - accountsUrl: string, - oldEmail: string, - newEmail: string -): Promise { - const accountWorkspaces = account.workspaces.map((it) => it.toString()) - // We need to update all workspaces - const workspaces = await listWorkspacesPure(db) - for (const ws of workspaces) { - if (!accountWorkspaces.includes(ws._id.toString())) { - continue - } - console.log('checking workspace', ws.workspaceName, ws.workspace) +// await fixWorkspaceEmails(account, db, transactorUrl, oldEmail, newEmail) +// } +// async function fixWorkspaceEmails ( +// account: Account, +// db: AccountDB, +// accountsUrl: string, +// oldEmail: string, +// newEmail: string +// ): Promise { +// const accountWorkspaces = account.workspaces.map((it) => it.toString()) +// // We need to update all workspaces +// const workspaces = await listWorkspacesPure(db) +// for (const ws of workspaces) { +// if (!accountWorkspaces.includes(ws._id.toString())) { +// continue +// } +// console.log('checking workspace', ws.workspaceName, ws.workspace) - const wsid = getWorkspaceId(ws.workspace) - const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid)) +// const wsid = getWorkspaceId(ws.workspace) +// const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid)) - // Let's connect and update account information. - await fixEmailInWorkspace(endpoint, ws, oldEmail, newEmail) - } -} +// // Let's connect and update account information. +// await fixEmailInWorkspace(endpoint, ws, oldEmail, newEmail) +// } +// } -async function fixEmailInWorkspace ( - transactorUrl: string, - ws: Workspace, - oldEmail: string, - newEmail: string -): Promise { - const connection = await connect(transactorUrl, { name: ws.workspace }, undefined, { - mode: 'backup', - model: 'upgrade', // Required for force all clients reload after operation will be complete. - admin: 'true' - }) - try { - const personAccount = await connection.findOne(contact.class.PersonAccount, { email: oldEmail }) +// async function fixEmailInWorkspace ( +// transactorUrl: string, +// ws: Workspace, +// oldEmail: string, +// newEmail: string +// ): Promise { +// const connection = await connect(transactorUrl, { name: ws.workspace }, undefined, { +// mode: 'backup', +// model: 'upgrade', // Required for force all clients reload after operation will be complete. +// admin: 'true' +// }) +// try { +// const personAccount = await connection.findOne(contact.class.PersonAccount, { email: oldEmail }) - if (personAccount !== undefined) { - console.log('update account in ', ws.workspace) - const ops = new TxOperations(connection, core.account.ConfigUser) - await ops.update(personAccount, { email: newEmail }) - } - } catch (err: any) { - console.error(err) - } finally { - await connection.close() - } -} +// if (personAccount !== undefined) { +// console.log('update account in ', ws.workspace) +// const ops = new TxOperations(connection, core.account.ConfigUser) +// await ops.update(personAccount, { email: newEmail }) +// } +// } catch (err: any) { +// console.error(err) +// } finally { +// await connection.close() +// } +// } -interface GithubUserResult { - login: string | null - code: number - rateLimitReset?: number | null -} +// interface GithubUserResult { +// login: string | null +// code: number +// rateLimitReset?: number | null +// } -async function getGithubUser (githubId: string, ghToken?: string): Promise { - const options = - ghToken !== undefined - ? { - headers: { - Authorization: `Bearer ${ghToken}`, - Accept: 'application/vnd.github.v3+json' - } - } - : undefined - const res = await fetch(`https://api.github.com/user/${githubId}`, options) +// async function getGithubUser (githubId: string, ghToken?: string): Promise { +// const options = +// ghToken !== undefined +// ? { +// headers: { +// Authorization: `Bearer ${ghToken}`, +// Accept: 'application/vnd.github.v3+json' +// } +// } +// : undefined +// const res = await fetch(`https://api.github.com/user/${githubId}`, options) - if (res.status === 200) { - return { - login: (await res.json()).login, - code: 200 - } - } +// if (res.status === 200) { +// return { +// login: (await res.json()).login, +// code: 200 +// } +// } - if (res.status === 403) { - const rateLimitReset = res.headers.get('X-RateLimit-Reset') - return { - login: null, - code: res.status, - rateLimitReset: rateLimitReset != null ? parseInt(rateLimitReset) * 1000 : null - } - } +// if (res.status === 403) { +// const rateLimitReset = res.headers.get('X-RateLimit-Reset') +// return { +// login: null, +// code: res.status, +// rateLimitReset: rateLimitReset != null ? parseInt(rateLimitReset) * 1000 : null +// } +// } - return { - login: null, - code: res.status - } -} +// return { +// login: null, +// code: res.status +// } +// } -export async function fillGithubUsers (ctx: MeasureContext, db: AccountDB, ghToken?: string): Promise { - const githubAccounts = await db.account.find({ githubId: { $ne: null } }) - if (githubAccounts.length === 0) { - ctx.info('no github accounts found') - return - } +// export async function fillGithubUsers (ctx: MeasureContext, db: AccountDB, ghToken?: string): Promise { +// const githubAccounts = await db.account.find({ githubId: { $ne: null } }) +// if (githubAccounts.length === 0) { +// ctx.info('no github accounts found') +// return +// } - const accountsToProcess = githubAccounts.filter(({ githubId, githubUser }) => githubUser == null && githubId != null) - if (accountsToProcess.length === 0) { - ctx.info('no github accounts left to fill') - return - } +// const accountsToProcess = githubAccounts.filter(({ githubId, githubUser }) => githubUser == null && githubId != null) +// if (accountsToProcess.length === 0) { +// ctx.info('no github accounts left to fill') +// return +// } - ctx.info('processing github accounts', { total: accountsToProcess.length }) - const defaultRetryTimeout = 1000 * 60 * 5 // 5 minutes - let processed = 0 - for (const account of accountsToProcess) { - while (true) { - try { - if (account.githubId == null) break - let username: string | undefined - if (account.email.startsWith('github:')) { - username = account.email.slice(7) - } else { - const githubUserRes = await getGithubUser(account.githubId, ghToken) - if (githubUserRes.code === 200 && githubUserRes.login != null) { - username = githubUserRes.login - } else if (githubUserRes.code === 404) { - ctx.info('github user not found', { githubId: account.githubId }) - break - } else if (githubUserRes.code === 403) { - const timeout = - githubUserRes.rateLimitReset != null - ? githubUserRes.rateLimitReset - Date.now() + 1000 - : defaultRetryTimeout - ctx.info('rate limit exceeded. Retrying in ', { - githubId: account.githubId, - retryTimeoutMin: Math.ceil(timeout / (1000 * 60)) - }) - await new Promise((resolve) => setTimeout(resolve, timeout)) - } else { - ctx.error('failed to get github user', { githubId: account.githubId, ...githubUserRes }) - break - } - } - if (username != null) { - await db.account.updateOne({ _id: account._id }, { githubUser: username.toLowerCase() }) - ctx.info('github user added', { githubId: account.githubId, githubUser: username.toLowerCase() }) - break - } - } catch (err: any) { - ctx.error('failed to fill github user', { githubId: account.githubId, err }) - break - } - } - processed++ - if (processed % 100 === 0) { - ctx.info('processing accounts:', { processed, of: accountsToProcess.length }) - } - } - ctx.info('finished processing accounts:', { processed, of: accountsToProcess.length }) -} +// ctx.info('processing github accounts', { total: accountsToProcess.length }) +// const defaultRetryTimeout = 1000 * 60 * 5 // 5 minutes +// let processed = 0 +// for (const account of accountsToProcess) { +// while (true) { +// try { +// if (account.githubId == null) break +// let username: string | undefined +// if (account.email.startsWith('github:')) { +// username = account.email.slice(7) +// } else { +// const githubUserRes = await getGithubUser(account.githubId, ghToken) +// if (githubUserRes.code === 200 && githubUserRes.login != null) { +// username = githubUserRes.login +// } else if (githubUserRes.code === 404) { +// ctx.info('github user not found', { githubId: account.githubId }) +// break +// } else if (githubUserRes.code === 403) { +// const timeout = +// githubUserRes.rateLimitReset != null +// ? githubUserRes.rateLimitReset - Date.now() + 1000 +// : defaultRetryTimeout +// ctx.info('rate limit exceeded. Retrying in ', { +// githubId: account.githubId, +// retryTimeoutMin: Math.ceil(timeout / (1000 * 60)) +// }) +// await new Promise((resolve) => setTimeout(resolve, timeout)) +// } else { +// ctx.error('failed to get github user', { githubId: account.githubId, ...githubUserRes }) +// break +// } +// } +// if (username != null) { +// await db.account.updateOne({ _id: account._id }, { githubUser: username.toLowerCase() }) +// ctx.info('github user added', { githubId: account.githubId, githubUser: username.toLowerCase() }) +// break +// } +// } catch (err: any) { +// ctx.error('failed to fill github user', { githubId: account.githubId, err }) +// break +// } +// } +// processed++ +// if (processed % 100 === 0) { +// ctx.info('processing accounts:', { processed, of: accountsToProcess.length }) +// } +// } +// ctx.info('finished processing accounts:', { processed, of: accountsToProcess.length }) +// } diff --git a/dev/tool/src/benchmark.ts b/dev/tool/src/benchmark.ts index 7143746c0e..2b5041adf1 100644 --- a/dev/tool/src/benchmark.ts +++ b/dev/tool/src/benchmark.ts @@ -12,25 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. // - +/* eslint-disable @typescript-eslint/no-unused-vars */ import core, { - AccountRole, MeasureMetricsContext, RateLimiter, TxOperations, concatLink, generateId, - getWorkspaceId, metricsToString, newMetrics, - systemAccountEmail, - type Account, + systemAccountUuid, + buildSocialIdString, + type PersonId, type BackupClient, type BenchmarkDoc, type Client, type Metrics, type Ref, - type WorkspaceId + type WorkspaceUuid, + SocialIdType, + type PersonUuid } from '@hcengineering/core' import { generateToken } from '@hcengineering/server-token' import { connect } from '@hcengineering/server-tool' @@ -42,15 +43,15 @@ import os from 'os' import { Worker, isMainThread, parentPort } from 'worker_threads' import { CSVWriter } from './csv' -import { AvatarType, type PersonAccount } from '@hcengineering/contact' +import { AvatarType, getPersonBySocialId } from '@hcengineering/contact' import contact from '@hcengineering/model-contact' import recruit from '@hcengineering/model-recruit' import { type Vacancy } from '@hcengineering/recruit' import { WebSocket } from 'ws' interface StartMessage { - email: string - workspaceId: WorkspaceId + account: PersonUuid + workspaceId: WorkspaceUuid transactorUrl: string id: number idd: number @@ -87,8 +88,8 @@ interface PendingMsg extends Msg { } export async function benchmark ( - workspaceId: WorkspaceId[], - users: Map, + workspaceId: WorkspaceUuid[], + users: Map, accountsUrl: string, cmd: { from: number @@ -177,11 +178,11 @@ export async function benchmark ( let transfer: number = 0 let oldTransfer: number = 0 - const token = generateToken(systemAccountEmail, workspaceId[0]) + const token = generateToken(systemAccountUuid, workspaceId[0]) setMetadata(serverClientPlugin.metadata.Endpoint, accountsUrl) const endpoint = await getTransactorEndpoint(token, 'external') - console.log('monitor endpoint', endpoint, 'workspace', workspaceId[0].name) + console.log('monitor endpoint', endpoint, 'workspace', workspaceId[0]) const monitorConnection = isMainThread ? ((await ctx.with( 'connect', @@ -303,13 +304,13 @@ export async function benchmark ( .map(async (it) => { const wsid = workspaceId[randNum(workspaceId.length)] const workId = 'w-' + i + '-' + it - const wsUsers = users.get(wsid.name) ?? [] + const wsUsers = users.get(wsid) ?? [] - const token = generateToken(systemAccountEmail, wsid) + const token = generateToken(systemAccountUuid, wsid) const endpoint = await getTransactorEndpoint(token, 'external') - console.log('endpoint', endpoint, 'workspace', wsid.name) + console.log('endpoint', endpoint, 'workspace', wsid) const msg: StartMessage = { - email: wsUsers[randNum(wsUsers.length)], + account: wsUsers[randNum(wsUsers.length)], workspaceId: wsid, transactorUrl: endpoint, id: i, @@ -371,10 +372,11 @@ export function benchmarkWorker (): void { setMetadata(client.metadata.UseProtocolCompression, msg.compression) console.log('connecting to', msg.workspaceId) - connection = await connect(msg.transactorUrl, msg.workspaceId, msg.email) + connection = await connect(msg.transactorUrl, msg.workspaceId, msg.account) if (msg.options.mode === 'find-all') { - const opt = new TxOperations(connection, (core.account.System + '_benchmark') as Ref) + const benchmarkPersonId = (core.account.System + '_benchmark') as PersonId + const opt = new TxOperations(connection, benchmarkPersonId) parentPort?.postMessage({ type: 'operate', workId: msg.workId @@ -479,39 +481,41 @@ export function benchmarkWorker (): void { export type StressBenchmarkMode = 'wrong' | 'connect-disconnect' export async function stressBenchmark (transactor: string, mode: StressBenchmarkMode): Promise { - if (mode === 'wrong') { - console.log('Stress with wrong workspace/email') - let counter = 0 - const rate = new RateLimiter(1) - while (true) { - try { - counter++ - console.log('Attempt', counter) - const token = generateToken(generateId(), { name: generateId() }) - await rate.add(async () => { - try { - const ws = new WebSocket(concatLink(transactor, token)) - await new Promise((resolve) => { - ws.onopen = () => { - resolve() - } - }) - // ws.close() - // await createClient(transactor, token, undefined, 50) - console.log('out') - } catch (err: any) { - console.error(err) - } - }) - } catch (err: any) { - // Ignore - } - } - } + // TODO: FIXME + throw new Error('Not implemented') + // if (mode === 'wrong') { + // console.log('Stress with wrong workspace/email') + // let counter = 0 + // const rate = new RateLimiter(1) + // while (true) { + // try { + // counter++ + // console.log('Attempt', counter) + // const token = generateToken(generateId(), generateId()) + // await rate.add(async () => { + // try { + // const ws = new WebSocket(concatLink(transactor, token)) + // await new Promise((resolve) => { + // ws.onopen = () => { + // resolve() + // } + // }) + // // ws.close() + // // await createClient(transactor, token, undefined, 50) + // console.log('out') + // } catch (err: any) { + // console.error(err) + // } + // }) + // } catch (err: any) { + // // Ignore + // } + // } + // } } -export async function testFindAll (endpoint: string, workspace: string, email: string): Promise { - const connection = await connect(endpoint, getWorkspaceId(workspace), email) +export async function testFindAll (endpoint: string, workspace: WorkspaceUuid, account: PersonUuid): Promise { + const connection = await connect(endpoint, workspace, account) try { const client = new TxOperations(connection, core.account.System) const start = Date.now() @@ -533,22 +537,23 @@ export async function testFindAll (endpoint: string, workspace: string, email: s export async function generateWorkspaceData ( endpoint: string, - workspace: string, + workspace: WorkspaceUuid, parallel: boolean, - user: string + email: string ): Promise { - const connection = await connect(endpoint, getWorkspaceId(workspace)) + const connection = await connect(endpoint, workspace) const client = new TxOperations(connection, core.account.System) try { - const acc = await client.findOne(contact.class.PersonAccount, { email: user }) - if (acc == null) { + const emailSocialString = buildSocialIdString({ type: SocialIdType.EMAIL, value: email }) + const person = await getPersonBySocialId(client, emailSocialString) + if (person == null) { throw new Error('User not found') } - const employees: Ref[] = [acc._id] + const employees: PersonId[] = [emailSocialString] const start = Date.now() for (let i = 0; i < 100; i++) { - const acc = await generateEmployee(client) - employees.push(acc) + const socialString = await generateEmployee(client) + employees.push(socialString) } if (parallel) { const promises: Promise[] = [] @@ -567,24 +572,39 @@ export async function generateWorkspaceData ( } } -export async function generateEmployee (client: TxOperations): Promise> { +export async function generateEmployee (client: TxOperations): Promise { + const personUuid = generateId() as unknown as PersonUuid // TODO: will it work or need to actually be a UUID? const personId = await client.createDoc(contact.class.Person, contact.space.Contacts, { name: generateId().toString(), city: '', - avatarType: AvatarType.COLOR + avatarType: AvatarType.COLOR, + personUuid }) + await client.createMixin(personId, contact.class.Person, contact.space.Contacts, contact.mixin.Employee, { active: true }) - const acc = await client.createDoc(contact.class.PersonAccount, core.space.Model, { - person: personId, - role: AccountRole.User, - email: personId - }) - return acc + + const socialString = buildSocialIdString({ type: SocialIdType.HULY, value: personUuid }) + + await client.addCollection( + contact.class.SocialIdentity, + contact.space.Contacts, + personId, + contact.class.Person, + 'socialIds', + { + type: SocialIdType.HULY, + value: personUuid, + key: socialString, + confirmed: true + } + ) + + return socialString } -async function generateVacancy (client: TxOperations, members: Ref[]): Promise { +async function generateVacancy (client: TxOperations, members: PersonId[]): Promise { // generate vacancies const _id = generateId() await client.createDoc( @@ -602,13 +622,30 @@ async function generateVacancy (client: TxOperations, members: Ref { - console.log('try clean bw miniature for ', workspaceId.name) + console.log('try clean bw miniature for ', workspaceId) const from = new Date(new Date().setDate(new Date().getDate() - 7)).getTime() const list = await storageService.listStream(ctx, workspaceId) let removed = 0 @@ -200,7 +198,7 @@ export async function fixMinioBW ( console.log('FINISH, removed: ', removed) } -export async function cleanRemovedTransactions (workspaceId: WorkspaceId, transactorUrl: string): Promise { +export async function cleanRemovedTransactions (workspaceId: WorkspaceUuid, transactorUrl: string): Promise { const connection = (await connect(transactorUrl, workspaceId, undefined, { mode: 'backup' })) as unknown as CoreClient & BackupClient @@ -232,7 +230,7 @@ export async function cleanRemovedTransactions (workspaceId: WorkspaceId, transa } } -export async function optimizeModel (workspaceId: WorkspaceId, transactorUrl: string): Promise { +export async function optimizeModel (workspaceId: WorkspaceUuid, transactorUrl: string): Promise { const connection = (await connect(transactorUrl, workspaceId, undefined, { mode: 'backup', model: 'upgrade' @@ -303,7 +301,7 @@ export async function optimizeModel (workspaceId: WorkspaceId, transactorUrl: st await connection.close() } } -export async function cleanArchivedSpaces (workspaceId: WorkspaceId, transactorUrl: string): Promise { +export async function cleanArchivedSpaces (workspaceId: WorkspaceUuid, transactorUrl: string): Promise { const connection = (await connect(transactorUrl, workspaceId, undefined, { mode: 'backup' })) as unknown as CoreClient & BackupClient @@ -346,7 +344,7 @@ export async function cleanArchivedSpaces (workspaceId: WorkspaceId, transactorU } } -export async function fixCommentDoubleIdCreate (workspaceId: WorkspaceId, transactorUrl: string): Promise { +export async function fixCommentDoubleIdCreate (workspaceId: WorkspaceUuid, transactorUrl: string): Promise { const connection = (await connect(transactorUrl, workspaceId, undefined, { mode: 'backup' })) as unknown as CoreClient & BackupClient @@ -398,7 +396,7 @@ export async function fixCommentDoubleIdCreate (workspaceId: WorkspaceId, transa const DOMAIN_TAGS = 'tags' as Domain export async function fixSkills ( mongoUrl: string, - workspaceId: WorkspaceId, + workspaceId: WorkspaceUuid, transactorUrl: string, step: string ): Promise { @@ -667,7 +665,7 @@ function groupBy (docs: T[], key: string): Record { export async function restoreRecruitingTaskTypes ( mongoUrl: string, - workspaceId: WorkspaceId, + workspaceId: WorkspaceUuid, transactorUrl: string ): Promise { const connection = (await connect(transactorUrl, workspaceId, undefined, { @@ -831,7 +829,7 @@ export async function restoreRecruitingTaskTypes ( export async function restoreHrTaskTypesFromUpdates ( mongoUrl: string, - workspaceId: WorkspaceId, + workspaceId: WorkspaceUuid, transactorUrl: string ): Promise { const connection = (await connect(transactorUrl, workspaceId, undefined, { @@ -1044,318 +1042,318 @@ export async function removeDuplicateIds ( accountsUrl: string, initWorkspacesStr: string ): Promise { - const state = 'REMOVE_DUPLICATE_IDS' - const [accountsDb, closeAccountsDb] = await getAccountDB(mongodbUri) - const mongoClient = getMongoClient(mongodbUri) - const _client = await mongoClient.getClient() - // disable spaces while change hardocded ids - const skippedDomains: string[] = [DOMAIN_DOC_INDEX_STATE, DOMAIN_BENCHMARK, DOMAIN_TX, DOMAIN_SPACE] - try { - const workspaces = await listWorkspacesRaw(accountsDb) - workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - const initWorkspaces = initWorkspacesStr.split(';') - const initWS = workspaces.filter((p) => initWorkspaces.includes(p.workspace)) - const ids = new Map() - for (const workspace of initWS) { - const workspaceId = getWorkspaceId(workspace.workspace) - const db = getWorkspaceMongoDB(_client, workspaceId) + // TODO: FIXME + throw new Error('Not implemented') + // const state = 'REMOVE_DUPLICATE_IDS' + // const [accountsDb, closeAccountsDb] = await getAccountDB(mongodbUri) + // const mongoClient = getMongoClient(mongodbUri) + // const _client = await mongoClient.getClient() + // // disable spaces while change hardocded ids + // const skippedDomains: string[] = [DOMAIN_DOC_INDEX_STATE, DOMAIN_BENCHMARK, DOMAIN_TX, DOMAIN_SPACE] + // try { + // const workspaces = await listWorkspacesRaw(accountsDb) + // workspaces.sort((a, b) => b.status.lastVisit - a.status.lastVisit) + // const initWorkspaces = initWorkspacesStr.split(';') + // const initWS = workspaces.filter((p) => initWorkspaces.includes(p.uuid)) + // const ids = new Map() + // for (const workspace of initWS) { + // const db = getWorkspaceMongoDB(_client, workspace.dataId) - const txex = await db.collection(DOMAIN_TX).find>({}).toArray() - const txesArr = [] - for (const obj of txex) { - if (obj.objectSpace === core.space.Model && !isPersonAccount(obj)) { - continue - } - txesArr.push({ _id: obj._id, _class: obj._class }) - } - txesArr.filter((it, idx, array) => array.findIndex((pt) => pt._id === it._id) === idx) - ids.set(DOMAIN_TX, txesArr) + // const txex = await db.collection(DOMAIN_TX).find>({}).toArray() + // const txesArr = [] + // for (const obj of txex) { + // if (obj.objectSpace === core.space.Model) { + // continue + // } + // txesArr.push({ _id: obj._id, _class: obj._class }) + // } + // txesArr.filter((it, idx, array) => array.findIndex((pt) => pt._id === it._id) === idx) + // ids.set(DOMAIN_TX, txesArr) - const colls = await db.collections() - for (const coll of colls) { - if (skippedDomains.includes(coll.collectionName)) continue - const arr = ids.get(coll.collectionName) ?? [] - const data = await coll.find({}, { projection: { _id: 1, _class: 1 } }).toArray() - for (const obj of data) { - arr.push(obj) - } - ids.set(coll.collectionName, arr) - } + // const colls = await db.collections() + // for (const coll of colls) { + // if (skippedDomains.includes(coll.collectionName)) continue + // const arr = ids.get(coll.collectionName) ?? [] + // const data = await coll.find({}, { projection: { _id: 1, _class: 1 } }).toArray() + // for (const obj of data) { + // arr.push(obj) + // } + // ids.set(coll.collectionName, arr) + // } - const arr = ids.get(DOMAIN_MODEL) ?? [] - const data = await db - .collection(DOMAIN_TX) - .find>( - { objectSpace: core.space.Model }, - { projection: { objectId: 1, objectClass: 1, modifiedBy: 1 } } - ) - .toArray() - for (const obj of data) { - if ( - (obj.modifiedBy === core.account.ConfigUser || obj.modifiedBy === core.account.System) && - !isPersonAccount(obj) - ) { - continue - } - if (obj.objectId === core.account.ConfigUser || obj.objectId === core.account.System) continue - arr.push({ _id: obj.objectId, _class: obj.objectClass }) - } - arr.filter((it, idx, array) => array.findIndex((pt) => pt._id === it._id) === idx) - ids.set(DOMAIN_MODEL, arr) - } + // const arr = ids.get(DOMAIN_MODEL) ?? [] + // const data = await db + // .collection(DOMAIN_TX) + // .find>( + // { objectSpace: core.space.Model }, + // { projection: { objectId: 1, objectClass: 1, modifiedBy: 1 } } + // ) + // .toArray() + // for (const obj of data) { + // if (obj.modifiedBy === core.account.ConfigUser || obj.modifiedBy === core.account.System) { + // continue + // } + // if (obj.objectId === core.account.ConfigUser || obj.objectId === core.account.System) continue + // arr.push({ _id: obj.objectId, _class: obj.objectClass }) + // } + // arr.filter((it, idx, array) => array.findIndex((pt) => pt._id === it._id) === idx) + // ids.set(DOMAIN_MODEL, arr) + // } - for (let index = 0; index < workspaces.length; index++) { - const workspace = workspaces[index] - // we should skip init workspace first time, for case if something went wrong - if (initWorkspaces.includes(workspace.workspace)) continue + // for (let index = 0; index < workspaces.length; index++) { + // const workspace = workspaces[index] + // // we should skip init workspace first time, for case if something went wrong + // if (initWorkspaces.includes(workspace.uuid)) continue - ctx.info(`Processing workspace ${workspace.workspaceName ?? workspace.workspace}`) - const workspaceId = getWorkspaceId(workspace.workspace) - const db = getWorkspaceMongoDB(_client, workspaceId) - const check = await db.collection(DOMAIN_MIGRATION).findOne({ state, plugin: workspace.workspace }) - if (check != null) continue + // ctx.info(`Processing workspace ${workspace.name ?? workspace.url ?? workspace.uuid}`) + // const workspaceId = workspace.uuid + // const wsDataId = workspace.dataId ?? workspaceId + // const db = getWorkspaceMongoDB(_client, workspace.dataId) + // const plugins = [workspace.uuid] + // if (workspace.dataId != null) { + // plugins.push(workspace.dataId) + // } - const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, workspaceId)) - const wsClient = (await connect(endpoint, workspaceId, undefined, { - model: 'upgrade' - })) as CoreClient & BackupClient - for (const set of ids) { - if (set[1].length === 0) continue - for (const doc of set[1]) { - await updateId(ctx, wsClient, db, storageAdapter, workspaceId, doc) - } - } - await wsClient.sendForceClose() - await wsClient.close() - await db.collection(DOMAIN_MIGRATION).insertOne({ - _id: generateId(), - state, - plugin: workspace.workspace, - space: core.space.Configuration, - modifiedOn: Date.now(), - modifiedBy: core.account.System, - _class: core.class.MigrationState - }) - ctx.info(`Done ${index} / ${workspaces.length - initWorkspaces.length}`) - } - } catch (err: any) { - console.trace(err) - } finally { - mongoClient.close() - closeAccountsDb() - } + // const check = await db.collection(DOMAIN_MIGRATION).findOne({ state, plugin: { $in: plugins } }) + // if (check != null) continue + + // const endpoint = await getTransactorEndpoint(generateToken(systemAccountUuid, workspaceId, { service: 'tool' })) + // const wsClient = (await connect(endpoint, workspaceId, undefined, { + // model: 'upgrade' + // })) as CoreClient & BackupClient + // for (const set of ids) { + // if (set[1].length === 0) continue + // for (const doc of set[1]) { + // await updateId(ctx, wsClient, db, storageAdapter, wsDataId, doc) + // } + // } + // await wsClient.sendForceClose() + // await wsClient.close() + // await db.collection(DOMAIN_MIGRATION).insertOne({ + // _id: generateId(), + // state, + // plugin: workspace.uuid, + // space: core.space.Configuration, + // modifiedOn: Date.now(), + // modifiedBy: core.account.System, + // _class: core.class.MigrationState + // }) + // ctx.info(`Done ${index} / ${workspaces.length - initWorkspaces.length}`) + // } + // } catch (err: any) { + // console.trace(err) + // } finally { + // mongoClient.close() + // closeAccountsDb() + // } } -function isPersonAccount (tx: TxCUD): boolean { - return tx.objectClass === contact.class.PersonAccount -} +// async function update (h: Hierarchy, db: Db, doc: T, update: DocumentUpdate): Promise { +// await db +// .collection(h.getDomain(doc._class)) +// .updateOne({ _id: doc._id }, { $set: { ...update, '%hash%': Date.now().toString(16) } }) +// } -async function update (h: Hierarchy, db: Db, doc: T, update: DocumentUpdate): Promise { - await db - .collection(h.getDomain(doc._class)) - .updateOne({ _id: doc._id }, { $set: { ...update, '%hash%': Date.now().toString(16) } }) -} +// async function updateId ( +// ctx: MeasureContext, +// client: CoreClient & BackupClient, +// db: Db, +// storage: StorageAdapter, +// workspaceId: WorkspaceDataId, +// docRef: RelatedDocument +// ): Promise { +// const h = client.getHierarchy() +// const txop = new TxOperations(client, core.account.System) +// try { +// // chech the doc exists +// const doc = await client.findOne(docRef._class, { _id: docRef._id }) +// if (doc === undefined) return +// const domain = h.getDomain(doc._class) +// const newId = generateId() -async function updateId ( - ctx: MeasureContext, - client: CoreClient & BackupClient, - db: Db, - storage: StorageAdapter, - workspaceId: WorkspaceId, - docRef: RelatedDocument -): Promise { - const h = client.getHierarchy() - const txop = new TxOperations(client, core.account.System) - try { - // chech the doc exists - const doc = await client.findOne(docRef._class, { _id: docRef._id }) - if (doc === undefined) return - const domain = h.getDomain(doc._class) - const newId = generateId() +// // update txes +// await db +// .collection(DOMAIN_TX) +// .updateMany({ objectId: doc._id }, { $set: { objectId: newId, '%hash%': Date.now().toString(16) } }) - // update txes - await db - .collection(DOMAIN_TX) - .updateMany({ objectId: doc._id }, { $set: { objectId: newId, '%hash%': Date.now().toString(16) } }) +// // update nested txes +// await db +// .collection(DOMAIN_TX) +// .updateMany({ 'tx.objectId': doc._id }, { $set: { 'tx.objectId': newId, '%hash%': Date.now().toString(16) } }) - // update nested txes - await db - .collection(DOMAIN_TX) - .updateMany({ 'tx.objectId': doc._id }, { $set: { 'tx.objectId': newId, '%hash%': Date.now().toString(16) } }) +// // we have generated ids for calendar, let's update in +// if (h.isDerived(doc._class, core.class.Account)) { +// await updateId(ctx, client, db, storage, workspaceId, { +// _id: `${doc._id}_calendar` as Ref, +// _class: calendar.class.Calendar +// }) +// } - // we have generated ids for calendar, let's update in - if (h.isDerived(doc._class, core.class.Account)) { - await updateId(ctx, client, db, storage, workspaceId, { - _id: `${doc._id}_calendar` as Ref, - _class: calendar.class.Calendar - }) - } +// // update backlinks +// const backlinks = await client.findAll(activity.class.ActivityReference, { attachedTo: doc._id }) +// for (const backlink of backlinks) { +// const contentDoc = await client.findOne(backlink.attachedDocClass ?? backlink.srcDocClass, { +// _id: backlink.attachedDocId ?? backlink.srcDocClass +// }) +// if (contentDoc !== undefined) { +// const attrs = h.getAllAttributes(contentDoc._class) +// for (const [attrName, attr] of attrs) { +// if (attr.type._class === core.class.TypeMarkup) { +// const markup = (contentDoc as any)[attrName] as Markup +// const newMarkup = markup.replaceAll(doc._id, newId) +// await update(h, db, contentDoc, { [attrName]: newMarkup }) +// } else if (attr.type._class === core.class.TypeCollaborativeDoc) { +// const collabId = makeDocCollabId(contentDoc, attr.name) +// await updateYDoc(ctx, collabId, storage, workspaceId, contentDoc, newId, doc) +// } +// } +// } +// await update(h, db, backlink, { attachedTo: newId, message: backlink.message.replaceAll(doc._id, newId) }) +// } - // update backlinks - const backlinks = await client.findAll(activity.class.ActivityReference, { attachedTo: doc._id }) - for (const backlink of backlinks) { - const contentDoc = await client.findOne(backlink.attachedDocClass ?? backlink.srcDocClass, { - _id: backlink.attachedDocId ?? backlink.srcDocClass - }) - if (contentDoc !== undefined) { - const attrs = h.getAllAttributes(contentDoc._class) - for (const [attrName, attr] of attrs) { - if (attr.type._class === core.class.TypeMarkup) { - const markup = (contentDoc as any)[attrName] as Markup - const newMarkup = markup.replaceAll(doc._id, newId) - await update(h, db, contentDoc, { [attrName]: newMarkup }) - } else if (attr.type._class === core.class.TypeCollaborativeDoc) { - const collabId = makeDocCollabId(contentDoc, attr.name) - await updateYDoc(ctx, collabId, storage, workspaceId, contentDoc, newId, doc) - } - } - } - await update(h, db, backlink, { attachedTo: newId, message: backlink.message.replaceAll(doc._id, newId) }) - } +// // blobs - // blobs +// await updateRefs(txop, newId, doc) - await updateRefs(txop, newId, doc) +// await updateArrRefs(txop, newId, doc) - await updateArrRefs(txop, newId, doc) +// // update docIndexState +// const docIndexState = await client.findOne(core.class.DocIndexState, { doc: doc._id }) +// if (docIndexState !== undefined) { +// const { _id, space, modifiedBy, modifiedOn, createdBy, createdOn, _class, ...data } = docIndexState +// await txop.createDoc(docIndexState._class, docIndexState.space, { +// ...data, +// removed: false +// }) +// await txop.update(docIndexState, { removed: true, needIndex: true }) +// } - // update docIndexState - const docIndexState = await client.findOne(core.class.DocIndexState, { doc: doc._id }) - if (docIndexState !== undefined) { - const { _id, space, modifiedBy, modifiedOn, createdBy, createdOn, _class, ...data } = docIndexState - await txop.createDoc(docIndexState._class, docIndexState.space, { - ...data, - removed: false - }) - await txop.update(docIndexState, { removed: true, needIndex: true }) - } +// if (domain !== DOMAIN_MODEL) { +// const raw = await db.collection(domain).findOne({ _id: doc._id }) +// await db.collection(domain).insertOne({ +// ...raw, +// _id: newId as any, +// '%hash%': Date.now().toString(16) +// }) +// await db.collection(domain).deleteOne({ _id: doc._id }) +// } +// } catch (err: any) { +// console.error('Error processing', docRef._id) +// } +// } - if (domain !== DOMAIN_MODEL) { - const raw = await db.collection(domain).findOne({ _id: doc._id }) - await db.collection(domain).insertOne({ - ...raw, - _id: newId as any, - '%hash%': Date.now().toString(16) - }) - await db.collection(domain).deleteOne({ _id: doc._id }) - } - } catch (err: any) { - console.error('Error processing', docRef._id) - } -} +// async function updateYDoc ( +// ctx: MeasureContext, +// _id: CollaborativeDoc, +// storage: StorageAdapter, +// workspaceId: WorkspaceDataId, +// contentDoc: Doc, +// newId: Ref, +// doc: RelatedDocument +// ): Promise { +// try { +// const ydoc = await loadCollabYdoc(ctx, storage, workspaceId, _id) +// if (ydoc === undefined) { +// ctx.error('document content not found', { document: contentDoc._id }) +// return +// } +// const buffer = yDocToBuffer(ydoc) -async function updateYDoc ( - ctx: MeasureContext, - _id: CollaborativeDoc, - storage: StorageAdapter, - workspaceId: WorkspaceId, - contentDoc: Doc, - newId: Ref, - doc: RelatedDocument -): Promise { - try { - const ydoc = await loadCollabYdoc(ctx, storage, workspaceId, _id) - if (ydoc === undefined) { - ctx.error('document content not found', { document: contentDoc._id }) - return - } - const buffer = yDocToBuffer(ydoc) +// const updatedYDoc = updateYDocContent(buffer, (body: Record) => { +// const str = JSON.stringify(body) +// const updated = str.replaceAll(doc._id, newId) +// return JSON.parse(updated) +// }) - const updatedYDoc = updateYDocContent(buffer, (body: Record) => { - const str = JSON.stringify(body) - const updated = str.replaceAll(doc._id, newId) - return JSON.parse(updated) - }) +// if (updatedYDoc !== undefined) { +// await saveCollabYdoc(ctx, storage, workspaceId, _id, updatedYDoc) +// } +// } catch { +// // do nothing, the collaborative doc does not sem to exist yet +// } +// } - if (updatedYDoc !== undefined) { - await saveCollabYdoc(ctx, storage, workspaceId, _id, updatedYDoc) - } - } catch { - // do nothing, the collaborative doc does not sem to exist yet - } -} +// async function updateRefs (client: TxOperations, newId: Ref, doc: RelatedDocument): Promise { +// const h = client.getHierarchy() +// const ancestors = h.getAncestors(doc._class) +// const reftos = (await client.findAll(core.class.Attribute, { 'type._class': core.class.RefTo })).filter((it) => { +// const to = it.type as RefTo +// return ancestors.includes(h.getBaseClass(to.to)) +// }) +// for (const attr of reftos) { +// if (attr.name === '_id') { +// continue +// } +// const descendants = h.getDescendants(attr.attributeOf) +// for (const d of descendants) { +// if (h.isDerived(d, core.class.BenchmarkDoc)) { +// continue +// } +// if (h.isDerived(d, core.class.Tx)) { +// continue +// } +// if (h.findDomain(d) !== undefined) { +// while (true) { +// const values = await client.findAll(d, { [attr.name]: doc._id }, { limit: 100 }) +// if (values.length === 0) { +// break +// } -async function updateRefs (client: TxOperations, newId: Ref, doc: RelatedDocument): Promise { - const h = client.getHierarchy() - const ancestors = h.getAncestors(doc._class) - const reftos = (await client.findAll(core.class.Attribute, { 'type._class': core.class.RefTo })).filter((it) => { - const to = it.type as RefTo - return ancestors.includes(h.getBaseClass(to.to)) - }) - for (const attr of reftos) { - if (attr.name === '_id') { - continue - } - const descendants = h.getDescendants(attr.attributeOf) - for (const d of descendants) { - if (h.isDerived(d, core.class.BenchmarkDoc)) { - continue - } - if (h.isDerived(d, core.class.Tx)) { - continue - } - if (h.findDomain(d) !== undefined) { - while (true) { - const values = await client.findAll(d, { [attr.name]: doc._id }, { limit: 100 }) - if (values.length === 0) { - break - } +// const builder = client.apply(doc._id) +// for (const v of values) { +// await updateAttribute(builder, v, d, { key: attr.name, attr }, newId, true) +// } +// const modelTxes = builder.txes.filter((p) => p.objectSpace === core.space.Model) +// builder.txes = builder.txes.filter((p) => p.objectSpace !== core.space.Model) +// for (const modelTx of modelTxes) { +// await client.tx(modelTx) +// } +// await builder.commit() +// } +// } +// } +// } +// } - const builder = client.apply(doc._id) - for (const v of values) { - await updateAttribute(builder, v, d, { key: attr.name, attr }, newId, true) - } - const modelTxes = builder.txes.filter((p) => p.objectSpace === core.space.Model) - builder.txes = builder.txes.filter((p) => p.objectSpace !== core.space.Model) - for (const modelTx of modelTxes) { - await client.tx(modelTx) - } - await builder.commit() - } - } - } - } -} - -async function updateArrRefs (client: TxOperations, newId: Ref, doc: RelatedDocument): Promise { - const h = client.getHierarchy() - const ancestors = h.getAncestors(doc._class) - const arrs = await client.findAll(core.class.Attribute, { 'type._class': core.class.ArrOf }) - for (const attr of arrs) { - if (attr.name === '_id') { - continue - } - const to = attr.type as ArrOf - if (to.of._class !== core.class.RefTo) continue - const refto = to.of as RefTo - if (ancestors.includes(h.getBaseClass(refto.to))) { - const descendants = h.getDescendants(attr.attributeOf) - for (const d of descendants) { - if (h.isDerived(d, core.class.BenchmarkDoc)) { - continue - } - if (h.isDerived(d, core.class.Tx)) { - continue - } - if (h.findDomain(d) !== undefined) { - while (true) { - const values = await client.findAll(attr.attributeOf, { [attr.name]: doc._id }, { limit: 100 }) - if (values.length === 0) { - break - } - const builder = client.apply(doc._id) - for (const v of values) { - await updateAttribute(builder, v, d, { key: attr.name, attr }, newId, true) - } - const modelTxes = builder.txes.filter((p) => p.objectSpace === core.space.Model) - builder.txes = builder.txes.filter((p) => p.objectSpace !== core.space.Model) - for (const modelTx of modelTxes) { - await client.tx(modelTx) - } - await builder.commit() - } - } - } - } - } -} +// async function updateArrRefs (client: TxOperations, newId: Ref, doc: RelatedDocument): Promise { +// const h = client.getHierarchy() +// const ancestors = h.getAncestors(doc._class) +// const arrs = await client.findAll(core.class.Attribute, { 'type._class': core.class.ArrOf }) +// for (const attr of arrs) { +// if (attr.name === '_id') { +// continue +// } +// const to = attr.type as ArrOf +// if (to.of._class !== core.class.RefTo) continue +// const refto = to.of as RefTo +// if (ancestors.includes(h.getBaseClass(refto.to))) { +// const descendants = h.getDescendants(attr.attributeOf) +// for (const d of descendants) { +// if (h.isDerived(d, core.class.BenchmarkDoc)) { +// continue +// } +// if (h.isDerived(d, core.class.Tx)) { +// continue +// } +// if (h.findDomain(d) !== undefined) { +// while (true) { +// const values = await client.findAll(attr.attributeOf, { [attr.name]: doc._id }, { limit: 100 }) +// if (values.length === 0) { +// break +// } +// const builder = client.apply(doc._id) +// for (const v of values) { +// await updateAttribute(builder, v, d, { key: attr.name, attr }, newId, true) +// } +// const modelTxes = builder.txes.filter((p) => p.objectSpace === core.space.Model) +// builder.txes = builder.txes.filter((p) => p.objectSpace !== core.space.Model) +// for (const modelTx of modelTxes) { +// await client.tx(modelTx) +// } +// await builder.commit() +// } +// } +// } +// } +// } +// } diff --git a/dev/tool/src/configuration.ts b/dev/tool/src/configuration.ts index 5ebdb259c0..c560b8d2f4 100644 --- a/dev/tool/src/configuration.ts +++ b/dev/tool/src/configuration.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import core, { type BackupClient, type Client as CoreClient, TxFactory, type WorkspaceId } from '@hcengineering/core' +import core, { type BackupClient, type Client as CoreClient, TxFactory, type WorkspaceUuid } from '@hcengineering/core' import { connect } from '@hcengineering/server-tool' function toLen (val: string, sep: string, len: number): string { @@ -23,7 +23,7 @@ function toLen (val: string, sep: string, len: number): string { return val } export async function changeConfiguration ( - workspaceId: WorkspaceId, + workspaceId: WorkspaceUuid, transactorUrl: string, cmd: { enable?: string, disable?: string, list?: boolean } ): Promise { diff --git a/dev/tool/src/db.ts b/dev/tool/src/db.ts index a45ef841b3..c3a7ef1192 100644 --- a/dev/tool/src/db.ts +++ b/dev/tool/src/db.ts @@ -1,21 +1,12 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { type AccountDB, type Workspace, getAccount, getWorkspaceById } from '@hcengineering/account' import { - type AccountDB, - listAccounts, - listWorkspacesPure, - listInvites, - updateWorkspace, - type Workspace, - type ObjectId, - getAccount, - getWorkspaceById -} from '@hcengineering/account' -import { + systemAccountUuid, type BackupClient, type Client, - getWorkspaceId, + type Doc, MeasureMetricsContext, - systemAccountEmail, - type Doc + type WorkspaceUuid } from '@hcengineering/core' import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo' import { @@ -55,7 +46,7 @@ export async function moveFromMongoToPG ( await moveWorkspace(accountDb, mongo, pgClient, ws, region) console.log('Move workspace', index, workspaces.length) } catch (err) { - console.log('Error when move workspace', ws.workspaceName ?? ws.workspace, err) + console.log('Error when move workspace', ws.name ?? ws.url, err) throw err } } @@ -73,9 +64,10 @@ async function moveWorkspace ( force = false ): Promise { try { - console.log('move workspace', ws.workspaceName ?? ws.workspace) - const wsId = getWorkspaceId(ws.workspace) - const mongoDB = getWorkspaceMongoDB(mongo, wsId) + console.log('move workspace', ws.name ?? ws.url) + const wsId = ws.uuid + // TODO: get workspace mongoDB + const mongoDB = getWorkspaceMongoDB(mongo, ws.dataId ?? wsId) const collections = await mongoDB.collections() let tables = collections.map((c) => c.collectionName) if (include !== undefined) { @@ -83,7 +75,7 @@ async function moveWorkspace ( } await createTables(new MeasureMetricsContext('', {}), pgClient, '', tables) - const token = generateToken(systemAccountEmail, wsId) + const token = generateToken(systemAccountUuid, wsId, { service: 'tool' }) const endpoint = await getTransactorEndpoint(token, 'external') const connection = (await connect(endpoint, wsId, undefined, { model: 'upgrade' @@ -94,8 +86,7 @@ async function moveWorkspace ( continue } const cursor = collection.find() - const current = - await pgClient`SELECT _id FROM ${pgClient(domain)} WHERE "workspaceId" = ${ws.uuid ?? ws.workspace}` + const current = await pgClient`SELECT _id FROM ${pgClient(domain)} WHERE "workspaceId" = ${ws.uuid}` const currentIds = new Set(current.map((r) => r._id)) console.log('move domain', domain) const docs: Doc[] = [] @@ -123,7 +114,7 @@ async function moveWorkspace ( const part = toRemove.splice(0, 100) await retryTxn(pgClient, async (client) => { await client.unsafe( - `DELETE FROM ${translateDomain(domain)} WHERE "workspaceId" = '${ws.workspace}' AND _id IN (${part.map((c) => `'${c}'`).join(', ')})` + `DELETE FROM ${translateDomain(domain)} WHERE "workspaceId" = '${ws.uuid}' AND _id IN (${part.map((c) => `'${c}'`).join(', ')})` ) }) } @@ -133,7 +124,7 @@ async function moveWorkspace ( const values: DBDoc[] = [] for (let i = 0; i < part.length; i++) { const doc = part[i] - const d = convertDoc(domain, doc, ws.workspace) + const d = convertDoc(domain, doc, wsId) values.push(d) } try { @@ -146,11 +137,12 @@ async function moveWorkspace ( } } } - await updateWorkspace(accountDb, ws, { region }) + // TODO: FIXME + // await updateWorkspace(accountDb, ws, { region }) await connection.sendForceClose() await connection.close() } catch (err) { - console.log('Error when move workspace', ws.workspaceName ?? ws.workspace, err) + console.log('Error when move workspace', ws.name ?? ws.url, err) throw err } } @@ -182,100 +174,102 @@ export async function moveAccountDbFromMongoToPG ( mongoDb: AccountDB, pgDb: AccountDB ): Promise { + // TODO: FIXME + throw new Error('Not implemented') // [accountId, workspaceId] - const workspaceAssignments: [ObjectId, ObjectId][] = [] - const accounts = await listAccounts(mongoDb) - const workspaces = await listWorkspacesPure(mongoDb) - const invites = await listInvites(mongoDb) + // const workspaceAssignments: [string, WorkspaceUuid][] = [] + // const accounts = await listAccounts(mongoDb) + // const workspaces = await listWorkspacesPure(mongoDb) + // const invites = await listInvites(mongoDb) - for (const mongoAccount of accounts) { - const pgAccount = { - ...mongoAccount, - _id: mongoAccount._id.toString() - } + // for (const mongoAccount of accounts) { + // const pgAccount = { + // ...mongoAccount, + // _id: mongoAccount._id.toString() + // } - delete (pgAccount as any).workspaces + // delete (pgAccount as any).workspaces - if (pgAccount.createdOn == null) { - pgAccount.createdOn = Date.now() - } + // if (pgAccount.createdOn == null) { + // pgAccount.createdOn = Date.now() + // } - if (pgAccount.first == null) { - pgAccount.first = 'NotSet' - } + // if (pgAccount.first == null) { + // pgAccount.first = 'NotSet' + // } - if (pgAccount.last == null) { - pgAccount.last = 'NotSet' - } + // if (pgAccount.last == null) { + // pgAccount.last = 'NotSet' + // } - for (const workspaceString of new Set(mongoAccount.workspaces.map((w) => w.toString()))) { - workspaceAssignments.push([pgAccount._id, workspaceString]) - } + // for (const workspaceString of new Set(mongoAccount.workspaces.map((w) => w.toString()))) { + // workspaceAssignments.push([pgAccount._id, workspaceString]) + // } - const exists = await getAccount(pgDb, pgAccount.email) - if (exists === null) { - await pgDb.account.insertOne(pgAccount) - ctx.info('Moved account', { email: pgAccount.email }) - } - } + // const exists = await getAccount(pgDb, pgAccount.email) + // if (exists === null) { + // await pgDb.account.insertOne(pgAccount) + // ctx.info('Moved account', { email: pgAccount.email }) + // } + // } - for (const mongoWorkspace of workspaces) { - const pgWorkspace = { - ...mongoWorkspace, - _id: mongoWorkspace._id.toString() - } + // for (const mongoWorkspace of workspaces) { + // const pgWorkspace = { + // ...mongoWorkspace, + // _id: mongoWorkspace._id.toString() + // } - if (pgWorkspace.createdOn == null) { - pgWorkspace.createdOn = Date.now() - } + // if (pgWorkspace.createdOn == null) { + // pgWorkspace.createdOn = Date.now() + // } - // delete deprecated fields - delete (pgWorkspace as any).createProgress - delete (pgWorkspace as any).creating - delete (pgWorkspace as any).productId - delete (pgWorkspace as any).organisation + // // delete deprecated fields + // delete (pgWorkspace as any).createProgress + // delete (pgWorkspace as any).creating + // delete (pgWorkspace as any).productId + // delete (pgWorkspace as any).organisation - // assigned separately - delete (pgWorkspace as any).accounts + // // assigned separately + // delete (pgWorkspace as any).accounts - const exists = await getWorkspaceById(pgDb, pgWorkspace.workspace) - if (exists === null) { - await pgDb.workspace.insertOne(pgWorkspace) - ctx.info('Moved workspace', { - workspace: pgWorkspace.workspace, - workspaceName: pgWorkspace.workspaceName, - workspaceUrl: pgWorkspace.workspaceUrl - }) - } - } + // const exists = await getWorkspaceById(pgDb, pgWorkspace.workspace) + // if (exists === null) { + // await pgDb.workspace.insertOne(pgWorkspace) + // ctx.info('Moved workspace', { + // workspace: pgWorkspace.workspace, + // workspaceName: pgWorkspace.workspaceName, + // workspaceUrl: pgWorkspace.workspaceUrl + // }) + // } + // } - for (const mongoInvite of invites) { - const pgInvite = { - ...mongoInvite, - _id: mongoInvite._id.toString() - } + // for (const mongoInvite of invites) { + // const pgInvite = { + // ...mongoInvite, + // _id: mongoInvite._id.toString() + // } - const exists = await pgDb.invite.findOne({ _id: pgInvite._id }) - if (exists === null) { - await pgDb.invite.insertOne(pgInvite) - } - } + // const exists = await pgDb.invite.findOne({ _id: pgInvite._id }) + // if (exists === null) { + // await pgDb.invite.insertOne(pgInvite) + // } + // } - const pgAssignments = (await listAccounts(pgDb)).reduce>((assignments, acc) => { - assignments[acc._id] = acc.workspaces + // const pgAssignments = (await listAccounts(pgDb)).reduce>((assignments, acc) => { + // assignments[acc._id] = acc.workspaces - return assignments - }, {}) - const assignmentsToInsert = workspaceAssignments.filter( - ([accountId, workspaceId]) => - pgAssignments[accountId] === undefined || !pgAssignments[accountId].includes(workspaceId) - ) + // return assignments + // }, {}) + // const assignmentsToInsert = workspaceAssignments.filter( + // ([accountId, workspaceId]) => + // pgAssignments[accountId] === undefined || !pgAssignments[accountId].includes(workspaceId) + // ) - for (const [accountId, workspaceId] of assignmentsToInsert) { - await pgDb.assignWorkspace(accountId, workspaceId) - } + // for (const [accountId, workspaceId] of assignmentsToInsert) { + // await pgDb.assignWorkspace(accountId, workspaceId) + // } - ctx.info('Assignments made', { count: assignmentsToInsert.length }) + // ctx.info('Assignments made', { count: assignmentsToInsert.length }) } export async function generateUuidMissingWorkspaces ( @@ -283,18 +277,20 @@ export async function generateUuidMissingWorkspaces ( db: AccountDB, dryRun = false ): Promise { - const workspaces = await listWorkspacesPure(db) - let updated = 0 - for (const ws of workspaces) { - if (ws.uuid !== undefined) continue + // TODO: FIXME + throw new Error('Not implemented') + // const workspaces = await listWorkspacesPure(db) + // let updated = 0 + // for (const ws of workspaces) { + // if (ws.uuid !== undefined) continue - const uuid = new UUID().toJSON() - if (!dryRun) { - await db.workspace.updateOne({ _id: ws._id }, { uuid }) - } - updated++ - } - ctx.info('Assigned uuids to workspaces', { updated, total: workspaces.length }) + // const uuid = new UUID().toJSON() + // if (!dryRun) { + // await db.workspace.updateOne({ _id: ws._id }, { uuid }) + // } + // updated++ + // } + // ctx.info('Assigned uuids to workspaces', { updated, total: workspaces.length }) } export async function updateDataWorkspaceIdToUuid ( @@ -314,12 +310,12 @@ export async function updateDataWorkspaceIdToUuid ( // Generate uuids for all workspaces or verify they exist await generateUuidMissingWorkspaces(ctx, accountDb, dryRun) - const workspaces = await listWorkspacesPure(accountDb) - const noUuidWss = workspaces.filter((ws) => ws.uuid === undefined) - if (noUuidWss.length > 0) { - ctx.error('Workspace uuid is required but not defined', { workspaces: noUuidWss.map((it) => it.workspace) }) - throw new Error('workspace uuid is required but not defined') - } + const workspaces: Workspace[] = [] // TODO: FIXME await listWorkspacesPure(accountDb) + // const noUuidWss = workspaces.filter((ws) => ws.uuid === undefined) + // if (noUuidWss.length > 0) { + // ctx.error('Workspace uuid is required but not defined', { workspaces: noUuidWss.map((it) => it.workspace) }) + // throw new Error('workspace uuid is required but not defined') + // } const res = await pgClient`select t.table_name from information_schema.columns as c join information_schema.tables as t on @@ -342,13 +338,11 @@ export async function updateDataWorkspaceIdToUuid ( await retryTxn(pgClient, async (client) => { for (const ws of workspaces) { - const uuid = ws.uuid - if (uuid === undefined) { - ctx.error('Workspace uuid is required but not defined', { workspace: ws.workspace }) - throw new Error('workspace uuid is required but not defined') - } + if (ws.dataId === undefined) continue - await client`UPDATE ${client(table)} SET "workspaceId" = ${uuid} WHERE "workspaceIdOld" = ${ws.workspace} OR "workspaceIdOld" = ${uuid}` + const uuid = ws.uuid + + await client`UPDATE ${client(table)} SET "workspaceId" = ${uuid} WHERE "workspaceIdOld" = ${ws.dataId} OR "workspaceIdOld" = ${uuid}` } }) diff --git a/dev/tool/src/elastic.ts b/dev/tool/src/elastic.ts index acd3217579..539f316037 100644 --- a/dev/tool/src/elastic.ts +++ b/dev/tool/src/elastic.ts @@ -14,20 +14,20 @@ // import { Client as ElasticClient } from '@elastic/elasticsearch' -import core, { DOMAIN_DOC_INDEX_STATE, toWorkspaceString, type WorkspaceId } from '@hcengineering/core' +import core, { DOMAIN_DOC_INDEX_STATE } from '@hcengineering/core' import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo' import { type StorageAdapter } from '@hcengineering/server-core' export async function rebuildElastic ( mongoUrl: string, - workspaceId: WorkspaceId, + dataId: string, storageAdapter: StorageAdapter, elasticUrl: string ): Promise { const client = getMongoClient(mongoUrl) try { const _client = await client.getClient() - const db = getWorkspaceMongoDB(_client, workspaceId) + const db = getWorkspaceMongoDB(_client, dataId) await db .collection(DOMAIN_DOC_INDEX_STATE) .updateMany({ _class: core.class.DocIndexState }, { $set: { elastic: false } }) @@ -35,15 +35,15 @@ export async function rebuildElastic ( client.close() } - await dropElastic(elasticUrl, workspaceId) + await dropElastic(elasticUrl, dataId) } -async function dropElastic (elasticUrl: string, workspaceId: WorkspaceId): Promise { +async function dropElastic (elasticUrl: string, dataId: string): Promise { console.log('drop existing elastic docment') const client = new ElasticClient({ node: elasticUrl }) - const productWs = toWorkspaceString(workspaceId) + const productWs = dataId await new Promise((resolve, reject) => { client.indices.exists( { diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 64f853a095..dd9d1e5219 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -13,29 +13,23 @@ // See the License for the specific language governing permissions and // limitations under the License. // - +/* eslint-disable @typescript-eslint/no-unused-vars */ import accountPlugin, { - assignAccountToWs, + assignWorkspace, confirmEmail, - createAcc, - createWorkspace as createWorkspaceRecord, - dropAccount, - dropWorkspace, - dropWorkspaceFull, getAccount, - getAccountDB, getWorkspaceById, - listAccounts, - listWorkspacesByAccount, - listWorkspacesPure, - listWorkspacesRaw, - replacePassword, - setAccountAdmin, - setRole, updateArchiveInfo, - updateWorkspace, + signUpByEmail, + createWorkspaceRecord, + updateWorkspaceInfo, + getAccountDB, + getWorkspaceInfoWithStatusById, + flattenStatus, + type WorkspaceInfoWithStatus, type AccountDB, - type Workspace + type Workspace, + getEmailSocialId } from '@hcengineering/account' import { backupWorkspace } from '@hcengineering/backup-service' import { setMetadata } from '@hcengineering/platform' @@ -55,8 +49,7 @@ import serverClientPlugin, { BlobClient, createClient, getTransactorEndpoint, - listAccountWorkspaces, - updateBackupInfo + getAccountClient } from '@hcengineering/server-client' import { createBackupPipeline, @@ -83,20 +76,17 @@ import { diffWorkspace, updateField } from './workspace' import core, { AccountRole, generateId, - getWorkspaceId, isActiveMode, isArchivingMode, MeasureMetricsContext, metricsToString, RateLimiter, - systemAccountEmail, versionToString, type Data, type Doc, type Ref, type Tx, - type Version, - type WorkspaceId + type Version } from '@hcengineering/core' import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model' import contact from '@hcengineering/model-contact' @@ -123,7 +113,7 @@ import type { PipelineFactory, StorageAdapter, StorageAdapterEx } from '@hcengin import { deepEqual } from 'fast-equals' import { createWriteStream, readFileSync } from 'fs' import { getAccountDBUrl, getMongoDBUrl } from './__start' -import { fillGithubUsers, fixAccountEmails, renameAccount } from './account' +// import { fillGithubUsers, fixAccountEmails, renameAccount } from './account' import { benchmark, benchmarkWorker, @@ -155,7 +145,9 @@ import { import { reindexWorkspace } from './fulltext' import { restoreControlledDocContentMongo, restoreMarkupRefsMongo, restoreWikiContentMongo } from './markup' import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin' + import { copyToDatalake, moveFiles, showLostFiles } from './storage' +import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils' const colorConstants = { colorRed: '\u001b[31m', @@ -265,83 +257,84 @@ export function devTool ( .requiredOption('-p, --password ', 'user password') .requiredOption('-f, --first ', 'first name') .requiredOption('-l, --last ', 'last name') - .action(async (email: string, cmd) => { + .option('-n, --notconfirmed', 'creates not confirmed account', false) + .action(async (email: string, cmd: { password: string, first: string, last: string, notconfirmed: boolean }) => { await withAccountDatabase(async (db) => { - console.log(`creating account ${cmd.first as string} ${cmd.last as string} (${email})...`) - await createAcc(toolCtx, db, null, email, cmd.password, cmd.first, cmd.last, true) + console.log(`creating account ${cmd.first} ${cmd.last} (${email})...`) + await signUpByEmail(toolCtx, db, null, email, cmd.password, cmd.first, cmd.last, !cmd.notconfirmed) }) }) - program - .command('reset-account ') - .description('create user and corresponding account in master database') - .option('-p, --password ', 'new user password') - .action(async (email: string, cmd) => { - await withAccountDatabase(async (db) => { - console.log(`update account ${email} ${cmd.first as string} ${cmd.last as string}...`) - await replacePassword(db, email, cmd.password) - }) - }) + // program + // .command('reset-account ') + // .description('create user and corresponding account in master database') + // .option('-p, --password ', 'new user password') + // .action(async (email: string, cmd) => { + // await withAccountDatabase(async (db) => { + // console.log(`update account ${email} ${cmd.first as string} ${cmd.last as string}...`) + // await replacePassword(db, email, cmd.password) + // }) + // }) - program - .command('reset-email ') - .description('rename account in accounts and all workspaces') - .action(async (email: string, newEmail: string, cmd) => { - await withAccountDatabase(async (db) => { - console.log(`update account ${email} to ${newEmail}`) - await renameAccount(toolCtx, db, accountsUrl, email, newEmail) - }) - }) + // program + // .command('reset-email ') + // .description('rename account in accounts and all workspaces') + // .action(async (email: string, newEmail: string, cmd) => { + // await withAccountDatabase(async (db) => { + // console.log(`update account ${email} to ${newEmail}`) + // await renameAccount(toolCtx, db, accountsUrl, email, newEmail) + // }) + // }) - program - .command('fix-email ') - .description('fix email in all workspaces to be proper one') - .action(async (email: string, newEmail: string, cmd) => { - await withAccountDatabase(async (db) => { - console.log(`update account ${email} to ${newEmail}`) - await fixAccountEmails(toolCtx, db, accountsUrl, email, newEmail) - }) - }) + // program + // .command('fix-email ') + // .description('fix email in all workspaces to be proper one') + // .action(async (email: string, newEmail: string, cmd) => { + // await withAccountDatabase(async (db) => { + // console.log(`update account ${email} to ${newEmail}`) + // await fixAccountEmails(toolCtx, db, accountsUrl, email, newEmail) + // }) + // }) - program - .command('compact-db-mongo') - .description('compact all db collections') - .option('-w, --workspace ', 'A selected "workspace" only', '') - .action(async (cmd: { workspace: string }) => { - const dbUrl = getMongoDBUrl() - await withAccountDatabase(async (db) => { - console.log('compacting db ...') - let gtotal: number = 0 - const client = getMongoClient(dbUrl) - const _client = await client.getClient() - try { - const workspaces = await listWorkspacesPure(db) - for (const workspace of workspaces) { - if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { - continue - } - let total: number = 0 - const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - const collections = wsDb.listCollections() - while (true) { - const collInfo = await collections.next() - if (collInfo === null) { - break - } - const result = await wsDb.command({ compact: collInfo.name }) - total += result.bytesFreed - } - gtotal += total - console.log('total feed for db', workspace.workspaceName, Math.round(total / (1024 * 1024))) - } - console.log('global total feed', Math.round(gtotal / (1024 * 1024))) - } catch (err: any) { - console.error(err) - } finally { - client.close() - } - }) - }) + // program + // .command('compact-db-mongo') + // .description('compact all db collections') + // .option('-w, --workspace ', 'A selected "workspace" only', '') + // .action(async (cmd: { workspace: string }) => { + // const dbUrl = getMongoDBUrl() + // await withAccountDatabase(async (db) => { + // console.log('compacting db ...') + // let gtotal: number = 0 + // const client = getMongoClient(dbUrl) + // const _client = await client.getClient() + // try { + // const workspaces = await listWorkspacesPure(db) + // for (const workspace of workspaces) { + // if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { + // continue + // } + // let total: number = 0 + // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) + // const collections = wsDb.listCollections() + // while (true) { + // const collInfo = await collections.next() + // if (collInfo === null) { + // break + // } + // const result = await wsDb.command({ compact: collInfo.name }) + // total += result.bytesFreed + // } + // gtotal += total + // console.log('total feed for db', workspace.workspaceName, Math.round(total / (1024 * 1024))) + // } + // console.log('global total feed', Math.round(gtotal / (1024 * 1024))) + // } catch (err: any) { + // console.error(err) + // } finally { + // client.close() + // } + // }) + // }) program .command('assign-workspace ') @@ -350,85 +343,87 @@ export function devTool ( await withAccountDatabase(async (db) => { console.log(`assigning user ${email} to ${workspace}...`) try { - const workspaceInfo = await getWorkspaceById(db, workspace) - if (workspaceInfo === null) { - throw new Error(`workspace ${workspace} not found`) + const ws = await getWorkspace(db, workspace) + if (ws === null) { + throw new Error(`Workspace ${workspace} not found`) } - const token = generateToken(systemAccountEmail, { name: workspaceInfo.workspace }) - const endpoint = await getTransactorEndpoint(token, 'external') - console.log('assigning to workspace', workspaceInfo, endpoint) - const client = await createClient(endpoint, token) - console.log('assigning to workspace connected', workspaceInfo, endpoint) - await assignAccountToWs( - toolCtx, - db, - null, - email, - workspaceInfo.workspace, - AccountRole.User, - undefined, - undefined, - client - ) - await client.close() + + await assignWorkspace(toolCtx, db, null, getToolToken(), email, ws.uuid, AccountRole.User) } catch (err: any) { console.error(err) } }) }) - program - .command('show-user ') - .description('show user') - .action(async (email) => { - await withAccountDatabase(async (db) => { - const info = await getAccount(db, email) - console.log(info) - }) - }) + // program + // .command('show-user ') + // .description('show user') + // .action(async (email) => { + // await withAccountDatabase(async (db) => { + // const info = await getAccount(db, email) + // console.log(info) + // }) + // }) program - .command('create-workspace ') + .command('create-workspace ') .description('create workspace') - .requiredOption('-w, --workspaceName ', 'Workspace name') - .option('-e, --email ', 'Author email', 'platform@email.com') .option('-i, --init ', 'Init from workspace') .option('-r, --region ', 'Region') .option('-b, --branding ', 'Branding key') - .action( - async ( - workspace, - cmd: { email: string, workspaceName: string, init?: string, branding?: string, region?: string } - ) => { - const { txes, version, migrateOperations } = prepareTools() - await withAccountDatabase(async (db) => { - const measureCtx = new MeasureMetricsContext('create-workspace', {}) - const brandingObj = - cmd.branding !== undefined || cmd.init !== undefined ? { key: cmd.branding, initWorkspace: cmd.init } : null - const wsInfo = await createWorkspaceRecord( - measureCtx, - db, - brandingObj, - cmd.email, - cmd.workspaceName, - workspace, - cmd.region, - 'manual-creation' - ) + .action(async (name, socialString, cmd: { account: string, init?: string, branding?: string, region?: string }) => { + const { txes, version, migrateOperations } = prepareTools() + await withAccountDatabase(async (db) => { + const measureCtx = new MeasureMetricsContext('create-workspace', {}) + const brandingObj = + cmd.branding !== undefined || cmd.init !== undefined ? { key: cmd.branding, initWorkspace: cmd.init } : null + const socialId = await db.socialId.findOne({ key: socialString }) + if (socialId == null) { + throw new Error(`Social id ${socialString} not found`) + } - await createWorkspace(measureCtx, version, brandingObj, wsInfo, txes, migrateOperations, undefined, true) + const res = await createWorkspaceRecord( + measureCtx, + db, + brandingObj, + name, + socialId.personUuid, + cmd.region, + 'manual-creation' + ) + const wsInfo = await getWorkspaceInfoWithStatusById(db, res.workspaceUuid) - await updateWorkspace(db, wsInfo, { - mode: 'active', - progress: 100, - disabled: false, - version - }) + if (wsInfo == null) { + throw new Error(`Created workspace record ${res.workspaceUuid} not found`) + } + const coreWsInfo = flattenStatus(wsInfo) + const accountClient = getAccountClient(getToolToken()) - console.log('create-workspace done') - }) - } - ) + await createWorkspace( + measureCtx, + version, + brandingObj, + coreWsInfo, + txes, + migrateOperations, + accountClient, + undefined, + true + ) + await updateWorkspaceInfo( + measureCtx, + db, + brandingObj, + getToolToken(), + res.workspaceUuid, + 'create-done', + version, + 100 + ) + + console.log('create-workspace done') + }) + }) program .command('set-user-role ') @@ -436,28 +431,29 @@ export function devTool ( .action(async (email: string, workspace: string, role: AccountRole, cmd) => { console.log(`set user ${email} role for ${workspace}...`) await withAccountDatabase(async (db) => { - const workspaceInfo = await getWorkspaceById(db, workspace) - if (workspaceInfo === null) { - throw new Error(`workspace ${workspace} not found`) + const rolesArray = ['DocGuest', 'GUEST', 'USER', 'MAINTAINER', 'OWNER'] + if (!rolesArray.includes(role)) { + throw new Error(`Invalid role ${role}. Valid roles are ${rolesArray.join(', ')}`) } - console.log('assigning to workspace', workspaceInfo) - const token = generateToken(systemAccountEmail, { name: workspaceInfo.workspace }) - const endpoint = await getTransactorEndpoint(token, 'external') - const client = await createClient(endpoint, token) - await setRole(toolCtx, db, email, workspace, role, client) - await client.close() + + const ws = await getWorkspace(db, workspace) + if (ws === null) { + throw new Error(`Workspace ${workspace} not found`) + } + + await assignWorkspace(toolCtx, db, null, getToolToken(), email, ws.uuid, role) }) }) - program - .command('set-user-admin ') - .description('set user role') - .action(async (email: string, role: string) => { - console.log(`set user ${email} admin...`) - await withAccountDatabase(async (db) => { - await setAccountAdmin(db, email, role === 'true') - }) - }) + // program + // .command('set-user-admin ') + // .description('set user role') + // .action(async (email: string, role: string) => { + // console.log(`set user ${email} admin...`) + // await withAccountDatabase(async (db) => { + // await setAccountAdmin(db, email, role === 'true') + // }) + // }) program .command('upgrade-workspace ') @@ -468,19 +464,27 @@ export function devTool ( const { version, txes, migrateOperations } = prepareTools() await withAccountDatabase(async (db) => { - const info = await getWorkspaceById(db, workspace) + const info = await getWorkspace(db, workspace) if (info === null) { throw new Error(`workspace ${workspace} not found`) } + const wsInfo = await getWorkspaceInfoWithStatusById(db, info.uuid) + if (wsInfo === null) { + throw new Error(`workspace ${workspace} not found`) + } + + const coreWsInfo = flattenStatus(wsInfo) const measureCtx = new MeasureMetricsContext('upgrade-workspace', {}) + const accountClient = getAccountClient(getToolToken(wsInfo.uuid)) await upgradeWorkspace( measureCtx, version, txes, migrateOperations, - info, + accountClient, + coreWsInfo, consoleModelLogger, async () => {}, cmd.force, @@ -488,511 +492,507 @@ export function devTool ( true ) - await updateWorkspace(db, info, { - mode: 'active', - progress: 100, - version, - attempts: 0 - }) + await updateWorkspaceInfo(measureCtx, db, null, getToolToken(), info.uuid, 'upgrade-done', version, 100) console.log(metricsToString(measureCtx.metrics, 'upgrade', 60)) console.log('upgrade-workspace done') }) }) - program - .command('upgrade') - .description('upgrade') - .option('-l|--logs ', 'Default logs folder', './logs') - .option('-i|--ignore [ignore]', 'Ignore workspaces', '') - .option('-r|--region [region]', 'Region of workspaces', '') - .option( - '-c|--console', - 'Display all information into console(default will create logs folder with {workspace}.log files', - false - ) - .option('-f|--force [force]', 'Force update', false) - .action(async (cmd: { logs: string, force: boolean, console: boolean, ignore: string, region: string }) => { - const { version, txes, migrateOperations } = prepareTools() - await withAccountDatabase(async (db) => { - const workspaces = (await listWorkspacesRaw(db, cmd.region)).filter((ws) => !cmd.ignore.includes(ws.workspace)) - workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - const measureCtx = new MeasureMetricsContext('upgrade', {}) + // program + // .command('upgrade') + // .description('upgrade') + // .option('-l|--logs ', 'Default logs folder', './logs') + // .option('-i|--ignore [ignore]', 'Ignore workspaces', '') + // .option('-r|--region [region]', 'Region of workspaces', '') + // .option( + // '-c|--console', + // 'Display all information into console(default will create logs folder with {workspace}.log files', + // false + // ) + // .option('-f|--force [force]', 'Force update', false) + // .action(async (cmd: { logs: string, force: boolean, console: boolean, ignore: string, region: string }) => { + // const { version, txes, migrateOperations } = prepareTools() + // await withAccountDatabase(async (db) => { + // const workspaces = (await listWorkspacesRaw(db, cmd.region)).filter((ws) => !cmd.ignore.includes(ws.workspace)) + // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) + // const measureCtx = new MeasureMetricsContext('upgrade', {}) - for (const ws of workspaces) { - console.warn('UPGRADING', ws.workspaceName) - const logger = cmd.console - ? consoleModelLogger - : new FileModelLogger(path.join(cmd.logs, `${ws.workspace}.log`)) + // for (const ws of workspaces) { + // console.warn('UPGRADING', ws.workspaceName) + // const logger = cmd.console + // ? consoleModelLogger + // : new FileModelLogger(path.join(cmd.logs, `${ws.workspace}.log`)) - try { - await upgradeWorkspace( - measureCtx, - version, - txes, - migrateOperations, - ws, - logger, - async () => {}, - cmd.force, - false, - true - ) + // try { + // await upgradeWorkspace( + // measureCtx, + // version, + // txes, + // migrateOperations, + // ws, + // logger, + // async () => {}, + // cmd.force, + // false, + // true + // ) - await updateWorkspace(db, ws, { - mode: 'active', - progress: 100, - version, - attempts: 0 - }) - } catch (err: any) { - toolCtx.error('failed to upgrade', { err, workspace: ws.workspace, workspaceName: ws.workspaceName }) - continue - } - } - console.log('upgrade done') - }) - }) + // await updateWorkspace(db, ws, { + // mode: 'active', + // progress: 100, + // version, + // attempts: 0 + // }) + // } catch (err: any) { + // toolCtx.error('failed to upgrade', { err, workspace: ws.workspace, workspaceName: ws.workspaceName }) + // continue + // } + // } + // console.log('upgrade done') + // }) + // }) - program - .command('list-unused-workspaces') - .description('list unused workspaces. Without it will only mark them disabled') - .option('-t|--timeout [timeout]', 'Timeout in days', '60') - .action(async (cmd: { disable: boolean, exclude: string, timeout: string }) => { - await withAccountDatabase(async (db) => { - const workspaces = new Map((await listWorkspacesPure(db)).map((p) => [p._id.toString(), p])) + // program + // .command('list-unused-workspaces') + // .description('list unused workspaces. Without it will only mark them disabled') + // .option('-t|--timeout [timeout]', 'Timeout in days', '60') + // .action(async (cmd: { disable: boolean, exclude: string, timeout: string }) => { + // await withAccountDatabase(async (db) => { + // const workspaces = new Map((await listWorkspacesPure(db)).map((p) => [p._id.toString(), p])) - const accounts = await listAccounts(db) + // const accounts = await listAccounts(db) - const _timeout = parseInt(cmd.timeout) ?? 7 + // const _timeout = parseInt(cmd.timeout) ?? 7 - let used = 0 - let unused = 0 + // let used = 0 + // let unused = 0 - for (const a of accounts) { - const authored = a.workspaces - .map((it) => workspaces.get(it.toString())) - .filter((it) => it !== undefined && it.createdBy?.trim() === a.email?.trim()) as Workspace[] - authored.sort((a, b) => b.lastVisit - a.lastVisit) - if (authored.length > 0) { - const lastLoginDays = Math.floor((Date.now() - a.lastVisit) / 1000 / 3600 / 24) - toolCtx.info(a.email, { - workspaces: a.workspaces.length, - firstName: a.first, - lastName: a.last, - lastLoginDays - }) - for (const ws of authored) { - const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) + // for (const a of accounts) { + // const authored = a.workspaces + // .map((it) => workspaces.get(it.toString())) + // .filter((it) => it !== undefined && it.createdBy?.trim() === a.email?.trim()) as Workspace[] + // authored.sort((a, b) => b.lastVisit - a.lastVisit) + // if (authored.length > 0) { + // const lastLoginDays = Math.floor((Date.now() - a.lastVisit) / 1000 / 3600 / 24) + // toolCtx.info(a.email, { + // workspaces: a.workspaces.length, + // firstName: a.first, + // lastName: a.last, + // lastLoginDays + // }) + // for (const ws of authored) { + // const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) - if (lastVisitDays > _timeout) { - unused++ - toolCtx.warn(' --- unused', { - url: ws.workspaceUrl, - id: ws.workspace, - lastVisitDays - }) - } else { - used++ - toolCtx.warn(' +++ used', { - url: ws.workspaceUrl, - id: ws.workspace, - createdBy: ws.createdBy, - lastVisitDays - }) - } - } - } - } + // if (lastVisitDays > _timeout) { + // unused++ + // toolCtx.warn(' --- unused', { + // url: ws.workspaceUrl, + // id: ws.workspace, + // lastVisitDays + // }) + // } else { + // used++ + // toolCtx.warn(' +++ used', { + // url: ws.workspaceUrl, + // id: ws.workspace, + // createdBy: ws.createdBy, + // lastVisitDays + // }) + // } + // } + // } + // } - console.log('Used: ', used, 'Unused: ', unused) - }) - }) - program - .command('archive-workspaces') - .description('Archive and delete non visited workspaces...') - .option('-r|--remove [remove]', 'Pass to remove all data', false) - .option('--region [region]', 'Pass to remove all data', '') - .option('-t|--timeout [timeout]', 'Timeout in days', '60') - .option('-w|--workspace [workspace]', 'Force backup of selected workspace', '') - .action( - async (cmd: { - disable: boolean - exclude: string - timeout: string - remove: boolean - workspace: string - region: string - }) => { - const { dbUrl, txes } = prepareTools() - await withAccountDatabase(async (db) => { - const workspaces = (await listWorkspacesPure(db)) - .sort((a, b) => a.lastVisit - b.lastVisit) - .filter((it) => cmd.workspace === '' || cmd.workspace === it.workspace) + // console.log('Used: ', used, 'Unused: ', unused) + // }) + // }) - const _timeout = parseInt(cmd.timeout) ?? 7 + // program + // .command('archive-workspaces') + // .description('Archive and delete non visited workspaces...') + // .option('-r|--remove [remove]', 'Pass to remove all data', false) + // .option('--region [region]', 'Pass to remove all data', '') + // .option('-t|--timeout [timeout]', 'Timeout in days', '60') + // .option('-w|--workspace [workspace]', 'Force backup of selected workspace', '') + // .action( + // async (cmd: { + // disable: boolean + // exclude: string + // timeout: string + // remove: boolean + // workspace: string + // region: string + // }) => { + // const { dbUrl, txes } = prepareTools() + // await withAccountDatabase(async (db) => { + // const workspaces = (await listWorkspacesPure(db)) + // .sort((a, b) => a.lastVisit - b.lastVisit) + // .filter((it) => cmd.workspace === '' || cmd.workspace === it.workspace) - let unused = 0 - for (const ws of workspaces) { - const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) + // const _timeout = parseInt(cmd.timeout) ?? 7 - if (lastVisitDays > _timeout && isActiveMode(ws.mode)) { - unused++ - toolCtx.warn('--- unused', { - url: ws.workspaceUrl, - id: ws.workspace, - lastVisitDays, - mode: ws.mode - }) - try { - await backupWorkspace( - toolCtx, - ws, - (dbUrl, storageAdapter) => { - const factory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, { - externalStorage: storageAdapter, - usePassedCtx: true - }) - return factory - }, - (ctx, dbUrls, workspace, branding, externalStorage) => { - return getConfig(ctx, dbUrls, ctx, { - externalStorage, - disableTriggers: true - }) - }, - cmd.region, - true, - true, - 5000, // 5 gigabytes per blob - sharedPipelineContextVars, - async (storage, workspaceStorage) => { - if (cmd.remove) { - await updateArchiveInfo(toolCtx, db, ws.workspace, true) - const files = await workspaceStorage.listStream(toolCtx, { name: ws.workspace }) + // let unused = 0 + // for (const ws of workspaces) { + // const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) - while (true) { - const docs = await files.next() - if (docs.length === 0) { - break - } - await workspaceStorage.remove( - toolCtx, - { name: ws.workspace }, - docs.map((it) => it._id) - ) - } + // if (lastVisitDays > _timeout && isActiveMode(ws.mode)) { + // unused++ + // toolCtx.warn('--- unused', { + // url: ws.workspaceUrl, + // id: ws.workspace, + // lastVisitDays, + // mode: ws.mode + // }) + // try { + // await backupWorkspace( + // toolCtx, + // ws, + // (dbUrl, storageAdapter) => { + // const factory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, { + // externalStorage: storageAdapter, + // usePassedCtx: true + // }) + // return factory + // }, + // (ctx, dbUrls, workspace, branding, externalStorage) => { + // return getConfig(ctx, dbUrls, ctx, { + // externalStorage, + // disableTriggers: true + // }) + // }, + // cmd.region, + // true, + // true, + // 5000, // 5 gigabytes per blob + // sharedPipelineContextVars, + // async (storage, workspaceStorage) => { + // if (cmd.remove) { + // await updateArchiveInfo(toolCtx, db, ws.workspace, true) + // const files = await workspaceStorage.listStream(toolCtx, { name: ws.workspace }) - const destroyer = getWorkspaceDestroyAdapter(dbUrl) + // while (true) { + // const docs = await files.next() + // if (docs.length === 0) { + // break + // } + // await workspaceStorage.remove( + // toolCtx, + // { name: ws.workspace }, + // docs.map((it) => it._id) + // ) + // } - await destroyer.deleteWorkspace(toolCtx, sharedPipelineContextVars, { name: ws.workspace }) - } - } - ) - } catch (err: any) { - toolCtx.error('Failed to backup/archive workspace', { workspace: ws.workspace }) - } - } - } - console.log('Processed unused workspaces', unused) - }) - } - ) + // const destroyer = getWorkspaceDestroyAdapter(dbUrl) - program - .command('backup-all') - .description('Backup all workspaces...') - .option('--region [region]', 'Force backup of selected workspace', '') - .option('-w|--workspace [workspace]', 'Force backup of selected workspace', '') - .action(async (cmd: { workspace: string, region: string }) => { - const { txes } = prepareTools() - await withAccountDatabase(async (db) => { - const workspaces = (await listWorkspacesPure(db)) - .sort((a, b) => a.lastVisit - b.lastVisit) - .filter((it) => cmd.workspace === '' || cmd.workspace === it.workspace) + // await destroyer.deleteWorkspace(toolCtx, sharedPipelineContextVars, { name: ws.workspace }) + // } + // } + // ) + // } catch (err: any) { + // toolCtx.error('Failed to backup/archive workspace', { workspace: ws.workspace }) + // } + // } + // } + // console.log('Processed unused workspaces', unused) + // }) + // } + // ) - let processed = 0 + // program + // .command('backup-all') + // .description('Backup all workspaces...') + // .option('--region [region]', 'Force backup of selected workspace', '') + // .option('-w|--workspace [workspace]', 'Force backup of selected workspace', '') + // .action(async (cmd: { workspace: string, region: string }) => { + // const { txes } = prepareTools() + // await withAccountDatabase(async (db) => { + // const workspaces = (await listWorkspacesPure(db)) + // .sort((a, b) => a.lastVisit - b.lastVisit) + // .filter((it) => cmd.workspace === '' || cmd.workspace === it.workspace) - // We need to update workspaces with missing workspaceUrl - for (const ws of workspaces) { - try { - if ( - await backupWorkspace( - toolCtx, - ws, - (dbUrl, storageAdapter) => { - const factory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, { - externalStorage: storageAdapter, - usePassedCtx: true - }) - return factory - }, - (ctx, dbUrls, workspace, branding, externalStorage) => { - return getConfig(ctx, dbUrls, ctx, { - externalStorage, - disableTriggers: true - }) - }, - cmd.region, - false, - false, - 100, - sharedPipelineContextVars - ) - ) { - processed++ - } - } catch (err: any) { - toolCtx.error('Failed to backup workspace', { workspace: ws.workspace }) - } - } - console.log('Processed workspaces', processed) - }) - }) + // let processed = 0 - program - .command('drop-workspace ') - .description('drop workspace') - .option('--full [full]', 'Force remove all data', false) - .action(async (workspace, cmd: { full: boolean }) => { - const { dbUrl } = prepareTools() + // // We need to update workspaces with missing workspaceUrl + // for (const ws of workspaces) { + // try { + // if ( + // await backupWorkspace( + // toolCtx, + // ws, + // (dbUrl, storageAdapter) => { + // const factory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, { + // externalStorage: storageAdapter, + // usePassedCtx: true + // }) + // return factory + // }, + // (ctx, dbUrls, workspace, branding, externalStorage) => { + // return getConfig(ctx, dbUrls, ctx, { + // externalStorage, + // disableTriggers: true + // }) + // }, + // cmd.region, + // false, + // false, + // 100, + // sharedPipelineContextVars + // ) + // ) { + // processed++ + // } + // } catch (err: any) { + // toolCtx.error('Failed to backup workspace', { workspace: ws.workspace }) + // } + // } + // console.log('Processed workspaces', processed) + // }) + // }) - await withStorage(async (storageAdapter) => { - await withAccountDatabase(async (db) => { - const ws = await getWorkspaceById(db, workspace) - if (ws === null) { - console.log('no workspace exists') - return - } - if (cmd.full) { - await dropWorkspaceFull(toolCtx, db, dbUrl, null, workspace, storageAdapter) - } else { - await dropWorkspace(toolCtx, db, null, workspace) - } - }) - }) - }) + // program + // .command('drop-workspace ') + // .description('drop workspace') + // .option('--full [full]', 'Force remove all data', false) + // .action(async (workspace, cmd: { full: boolean }) => { + // const { dbUrl } = prepareTools() - program - .command('drop-workspace-by-email ') - .description('drop workspace') - .option('--full [full]', 'Force remove all data', false) - .action(async (email, cmd: { full: boolean }) => { - const { dbUrl } = prepareTools() - await withStorage(async (storageAdapter) => { - await withAccountDatabase(async (db) => { - for (const workspace of await listWorkspacesByAccount(db, email)) { - if (cmd.full) { - await dropWorkspaceFull(toolCtx, db, dbUrl, null, workspace.workspace, storageAdapter) - } else { - await dropWorkspace(toolCtx, db, null, workspace.workspace) - } - } - }) - }) - }) - program - .command('list-workspace-by-email ') - .description('drop workspace') - .option('--full [full]', 'Force remove all data', false) - .action(async (email, cmd: { full: boolean }) => { - await withAccountDatabase(async (db) => { - for (const workspace of await listWorkspacesByAccount(db, email)) { - console.log(workspace.workspace, workspace.workspaceUrl, workspace.workspaceName) - } - }) - }) + // await withStorage(async (storageAdapter) => { + // await withAccountDatabase(async (db) => { + // const ws = await getWorkspaceById(db, workspace) + // if (ws === null) { + // console.log('no workspace exists') + // return + // } + // if (cmd.full) { + // await dropWorkspaceFull(toolCtx, db, dbUrl, null, workspace, storageAdapter) + // } else { + // await dropWorkspace(toolCtx, db, null, workspace) + // } + // }) + // }) + // }) - program - .command('drop-workspace-last-visit') - .description('drop old workspaces') - .action(async (cmd: any) => { - const { dbUrl } = prepareTools() + // program + // .command('drop-workspace-by-email ') + // .description('drop workspace') + // .option('--full [full]', 'Force remove all data', false) + // .action(async (email, cmd: { full: boolean }) => { + // const { dbUrl } = prepareTools() + // await withStorage(async (storageAdapter) => { + // await withAccountDatabase(async (db) => { + // for (const workspace of await listWorkspacesByAccount(db, email)) { + // if (cmd.full) { + // await dropWorkspaceFull(toolCtx, db, dbUrl, null, workspace.workspace, storageAdapter) + // } else { + // await dropWorkspace(toolCtx, db, null, workspace.workspace) + // } + // } + // }) + // }) + // }) + // program + // .command('list-workspace-by-email ') + // .description('drop workspace') + // .option('--full [full]', 'Force remove all data', false) + // .action(async (email, cmd: { full: boolean }) => { + // await withAccountDatabase(async (db) => { + // for (const workspace of await listWorkspacesByAccount(db, email)) { + // console.log(workspace.workspace, workspace.workspaceUrl, workspace.workspaceName) + // } + // }) + // }) - await withStorage(async (storageAdapter) => { - await withAccountDatabase(async (db) => { - const workspacesJSON = await listWorkspacesPure(db) - for (const ws of workspacesJSON) { - const lastVisit = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) - if (lastVisit > 60) { - await dropWorkspaceFull(toolCtx, db, dbUrl, null, ws.workspace, storageAdapter) - } - } - }) - }) - }) + // program + // .command('drop-workspace-last-visit') + // .description('drop old workspaces') + // .action(async (cmd: any) => { + // const { dbUrl } = prepareTools() - program - .command('list-workspaces') - .description('List workspaces') - .option('-e|--expired [expired]', 'Show only expired', false) - .action(async (cmd: { expired: boolean }) => { - const { version } = prepareTools() - await withAccountDatabase(async (db) => { - const workspacesJSON = await listWorkspacesPure(db) - for (const ws of workspacesJSON) { - let lastVisit = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) - if (cmd.expired && lastVisit <= 7) { - continue - } - console.log( - colorConstants.colorBlue + - '####################################################################################################' + - colorConstants.reset - ) - console.log('id:', colorConstants.colorWhiteCyan + ws.workspace + colorConstants.reset) - console.log('url:', ws.workspaceUrl, 'name:', ws.workspaceName) - console.log( - 'version:', - ws.version !== undefined ? versionToString(ws.version) : 'not-set', - !deepEqual(ws.version, version) ? `upgrade to ${versionToString(version)} is required` : '' - ) - console.log('disabled:', ws.disabled) - console.log('mode:', ws.mode) - console.log('created by:', ws.createdBy) - console.log('members:', (ws.accounts ?? []).length) - if (Number.isNaN(lastVisit)) { - lastVisit = 365 - } - if (lastVisit > 30) { - console.log(colorConstants.colorRed + `last visit: ${lastVisit} days ago` + colorConstants.reset) - } else if (lastVisit > 7) { - console.log(colorConstants.colorRedYellow + `last visit: ${lastVisit} days ago` + colorConstants.reset) - } else { - console.log('last visit:', lastVisit, 'days ago') - } - } + // await withStorage(async (storageAdapter) => { + // await withAccountDatabase(async (db) => { + // const workspacesJSON = await listWorkspacesPure(db) + // for (const ws of workspacesJSON) { + // const lastVisit = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) + // if (lastVisit > 60) { + // await dropWorkspaceFull(toolCtx, db, dbUrl, null, ws.workspace, storageAdapter) + // } + // } + // }) + // }) + // }) - console.log('latest model version:', JSON.stringify(version)) - }) - }) + // program + // .command('list-workspaces') + // .description('List workspaces') + // .option('-e|--expired [expired]', 'Show only expired', false) + // .action(async (cmd: { expired: boolean }) => { + // const { version } = prepareTools() + // await withAccountDatabase(async (db) => { + // const workspacesJSON = await listWorkspacesPure(db) + // for (const ws of workspacesJSON) { + // let lastVisit = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) + // if (cmd.expired && lastVisit <= 7) { + // continue + // } + // console.log( + // colorConstants.colorBlue + + // '####################################################################################################' + + // colorConstants.reset + // ) + // console.log('id:', colorConstants.colorWhiteCyan + ws.workspace + colorConstants.reset) + // console.log('url:', ws.workspaceUrl, 'name:', ws.workspaceName) + // console.log( + // 'version:', + // ws.version !== undefined ? versionToString(ws.version) : 'not-set', + // !deepEqual(ws.version, version) ? `upgrade to ${versionToString(version)} is required` : '' + // ) + // console.log('disabled:', ws.disabled) + // console.log('mode:', ws.mode) + // console.log('created by:', ws.createdBy) + // console.log('members:', (ws.accounts ?? []).length) + // if (Number.isNaN(lastVisit)) { + // lastVisit = 365 + // } + // if (lastVisit > 30) { + // console.log(colorConstants.colorRed + `last visit: ${lastVisit} days ago` + colorConstants.reset) + // } else if (lastVisit > 7) { + // console.log(colorConstants.colorRedYellow + `last visit: ${lastVisit} days ago` + colorConstants.reset) + // } else { + // console.log('last visit:', lastVisit, 'days ago') + // } + // } - program.command('fix-person-accounts-mongo').action(async () => { - const { version } = prepareTools() - const mongodbUri = getMongoDBUrl() - await withAccountDatabase(async (db) => { - const ws = await listWorkspacesPure(db) - const client = getMongoClient(mongodbUri) - const _client = await client.getClient() - try { - for (const w of ws) { - const wsDb = getWorkspaceMongoDB(_client, { name: w.workspace }) - await wsDb.collection('tx').updateMany( - { - objectClass: contact.class.PersonAccount, - objectSpace: null - }, - { $set: { objectSpace: core.space.Model } } - ) - } - } finally { - client.close() - } + // console.log('latest model version:', JSON.stringify(version)) + // }) + // }) - console.log('latest model version:', JSON.stringify(version)) - }) - }) + // program.command('fix-person-accounts-mongo').action(async () => { + // const { version } = prepareTools() + // const mongodbUri = getMongoDBUrl() + // await withAccountDatabase(async (db) => { + // const ws = await listWorkspacesPure(db) + // const client = getMongoClient(mongodbUri) + // const _client = await client.getClient() + // try { + // for (const w of ws) { + // const wsDb = getWorkspaceMongoDB(_client, { name: w.workspace }) + // await wsDb.collection('tx').updateMany( + // { + // objectClass: contact.class.PersonAccount, + // objectSpace: null + // }, + // { $set: { objectSpace: core.space.Model } } + // ) + // } + // } finally { + // client.close() + // } - program - .command('show-accounts') - .description('Show accounts') - .action(async () => { - await withAccountDatabase(async (db) => { - const workspaces = await listWorkspacesPure(db) - const accounts = await listAccounts(db) - for (const a of accounts) { - const wss = a.workspaces.map((it) => it.toString()) - console.info( - a.email, - a.confirmed, - workspaces.filter((it) => wss.includes(it._id.toString())).map((it) => it.workspaceUrl ?? it.workspace) - ) - } - }) - }) + // console.log('latest model version:', JSON.stringify(version)) + // }) + // }) - program - .command('drop-account ') - .description('drop account') - .action(async (email: string, cmd) => { - await withAccountDatabase(async (db) => { - await dropAccount(toolCtx, db, null, email) - }) - }) + // program + // .command('show-accounts') + // .description('Show accounts') + // .action(async () => { + // await withAccountDatabase(async (db) => { + // const workspaces = await listWorkspacesPure(db) + // const accounts = await listAccounts(db) + // for (const a of accounts) { + // const wss = a.workspaces.map((it) => it.toString()) + // console.info( + // a.email, + // a.confirmed, + // workspaces.filter((it) => wss.includes(it._id.toString())).map((it) => it.workspaceUrl ?? it.workspace) + // ) + // } + // }) + // }) - program - .command('backup ') - .description('dump workspace transactions and minio resources') - .option('-i, --include ', 'A list of ; separated domain names to include during backup', '*') - .option('-s, --skip ', 'A list of ; separated domain names to skip during backup', '') - .option( - '-ct, --contentTypes ', - 'A list of ; separated content types for blobs to skip download if size >= limit', - '' - ) - .option('-bl, --blobLimit ', 'A blob size limit in megabytes (default 15mb)', '15') - .option('-f, --force', 'Force backup', false) - .option('-f, --fresh', 'Force fresh backup', false) - .option('-c, --clean', 'Force clean of old backup files, only with fresh backup option', false) - .option('-t, --timeout ', 'Connect timeout in seconds', '30') - .action( - async ( - dirName: string, - workspace: string, - cmd: { - skip: string - force: boolean - fresh: boolean - clean: boolean - timeout: string - include: string - blobLimit: string - contentTypes: string - } - ) => { - const storage = await createFileBackupStorage(dirName) - const wsid = getWorkspaceId(workspace) - const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - await backup(toolCtx, endpoint, wsid, storage, { - force: cmd.force, - freshBackup: cmd.fresh, - clean: cmd.clean, - include: cmd.include === '*' ? undefined : new Set(cmd.include.split(';').map((it) => it.trim())), - skipDomains: (cmd.skip ?? '').split(';').map((it) => it.trim()), - timeout: 0, - connectTimeout: parseInt(cmd.timeout) * 1000, - blobDownloadLimit: parseInt(cmd.blobLimit), - skipBlobContentTypes: cmd.contentTypes - .split(';') - .map((it) => it.trim()) - .filter((it) => it.length > 0) - }) - } - ) - program - .command('backup-find ') - .description('dump workspace transactions and minio resources') - .option('-d, --domain ', 'Check only domain') - .action(async (dirName: string, fileId: string, cmd: { domain: string | undefined }) => { - const storage = await createFileBackupStorage(dirName) - await backupFind(storage, fileId as unknown as Ref, cmd.domain) - }) + // program + // .command('drop-account ') + // .description('drop account') + // .action(async (email: string, cmd) => { + // await withAccountDatabase(async (db) => { + // await dropAccount(toolCtx, db, null, email) + // }) + // }) - program - .command('backup-compact ') - .description('Compact a given backup, will create one snapshot clean unused resources') - .option('-f, --force', 'Force compact.', false) - .action(async (dirName: string, cmd: { force: boolean }) => { - const storage = await createFileBackupStorage(dirName) - await compactBackup(toolCtx, storage, cmd.force) - }) - program - .command('backup-check ') - .description('Compact a given backup, will create one snapshot clean unused resources') - .action(async (dirName: string, cmd: any) => { - const storage = await createFileBackupStorage(dirName) - await checkBackupIntegrity(toolCtx, storage) - }) + // program + // .command('backup ') + // .description('dump workspace transactions and minio resources') + // .option('-i, --include ', 'A list of ; separated domain names to include during backup', '*') + // .option('-s, --skip ', 'A list of ; separated domain names to skip during backup', '') + // .option( + // '-ct, --contentTypes ', + // 'A list of ; separated content types for blobs to skip download if size >= limit', + // '' + // ) + // .option('-bl, --blobLimit ', 'A blob size limit in megabytes (default 15mb)', '15') + // .option('-f, --force', 'Force backup', false) + // .option('-f, --fresh', 'Force fresh backup', false) + // .option('-c, --clean', 'Force clean of old backup files, only with fresh backup option', false) + // .option('-t, --timeout ', 'Connect timeout in seconds', '30') + // .action( + // async ( + // dirName: string, + // workspace: string, + // cmd: { + // skip: string + // force: boolean + // fresh: boolean + // clean: boolean + // timeout: string + // include: string + // blobLimit: string + // contentTypes: string + // } + // ) => { + // const storage = await createFileBackupStorage(dirName) + // const wsid = getWorkspaceId(workspace) + // const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') + // await backup(toolCtx, endpoint, wsIds, storage, { + // force: cmd.force, + // freshBackup: cmd.fresh, + // clean: cmd.clean, + // include: cmd.include === '*' ? undefined : new Set(cmd.include.split(';').map((it) => it.trim())), + // skipDomains: (cmd.skip ?? '').split(';').map((it) => it.trim()), + // timeout: 0, + // connectTimeout: parseInt(cmd.timeout) * 1000, + // blobDownloadLimit: parseInt(cmd.blobLimit), + // skipBlobContentTypes: cmd.contentTypes + // .split(';') + // .map((it) => it.trim()) + // .filter((it) => it.length > 0) + // }) + // } + // ) + // program + // .command('backup-find ') + // .description('dump workspace transactions and minio resources') + // .option('-d, --domain ', 'Check only domain') + // .action(async (dirName: string, fileId: string, cmd: { domain: string | undefined }) => { + // const storage = await createFileBackupStorage(dirName) + // await backupFind(storage, fileId as unknown as Ref, cmd.domain) + // }) + + // program + // .command('backup-compact ') + // .description('Compact a given backup, will create one snapshot clean unused resources') + // .option('-f, --force', 'Force compact.', false) + // .action(async (dirName: string, cmd: { force: boolean }) => { + // const storage = await createFileBackupStorage(dirName) + // await compactBackup(toolCtx, storage, cmd.force) + // }) + // program + // .command('backup-check ') + // .description('Compact a given backup, will create one snapshot clean unused resources') + // .action(async (dirName: string, cmd: any) => { + // const storage = await createFileBackupStorage(dirName) + // await checkBackupIntegrity(toolCtx, storage) + // }) program .command('backup-restore [date]') @@ -1011,7 +1011,7 @@ export function devTool ( .action( async ( dirName: string, - workspace: string, + workspaceId: string, date, cmd: { merge: boolean @@ -1023,749 +1023,768 @@ export function devTool ( historyFile: string } ) => { - const storage = await createFileBackupStorage(dirName) - const wsid = getWorkspaceId(workspace) - const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - const storageConfig = cmd.useStorage !== '' ? storageConfigFromEnv(process.env[cmd.useStorage]) : undefined - - const workspaceStorage: StorageAdapter | undefined = - storageConfig !== undefined ? buildStorageFromConfig(storageConfig) : undefined - await restore(toolCtx, endpoint, wsid, storage, { - date: parseInt(date ?? '-1'), - merge: cmd.merge, - parallel: parseInt(cmd.parallel ?? '1'), - recheck: cmd.recheck, - include: cmd.include === '*' ? undefined : new Set(cmd.include.split(';')), - skip: new Set(cmd.skip.split(';')), - storageAdapter: workspaceStorage, - historyFile: cmd.historyFile - }) - await workspaceStorage?.close() - } - ) - - program - .command('backup-list ') - .description('list snaphost ids for backup') - .action(async (dirName: string, cmd) => { - const storage = await createFileBackupStorage(dirName) - await backupList(storage) - }) - - program - .command('backup-s3 ') - .description('dump workspace transactions and minio resources') - .action(async (bucketName: string, dirName: string, workspace: string, cmd) => { - await withStorage(async (adapter) => { - const storage = await createStorageBackupStorage(toolCtx, adapter, getWorkspaceId(bucketName), dirName) - const wsid = getWorkspaceId(workspace) - const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - await backup(toolCtx, endpoint, wsid, storage) - }) - }) - program - .command('backup-s3-clean ') - .description('dump workspace transactions and minio resources') - .action(async (bucketName: string, days: string, cmd) => { - const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) - const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - - const daysInterval = Date.now() - parseInt(days) * 24 * 60 * 60 * 1000 - try { - const token = generateToken(systemAccountEmail, { name: 'any' }) - const workspaces = (await listAccountWorkspaces(token)).filter((it) => { - const lastBackup = it.backupInfo?.lastBackup ?? 0 - if (lastBackup > daysInterval) { - // No backup required, interval not elapsed - return true - } - - if (it.lastVisit == null) { - return false - } - - return false - }) - workspaces.sort((a, b) => { - return (b.backupInfo?.backupSize ?? 0) - (a.backupInfo?.backupSize ?? 0) - }) - - for (const ws of workspaces) { - const storage = await createStorageBackupStorage( - toolCtx, - storageAdapter, - getWorkspaceId(bucketName), - ws.workspace - ) - await backupRemoveLast(storage, daysInterval) - await updateBackupInfo(generateToken(systemAccountEmail, { name: 'any' }), { - backups: ws.backupInfo?.backups ?? 0, - backupSize: ws.backupInfo?.backupSize ?? 0, - blobsSize: ws.backupInfo?.blobsSize ?? 0, - dataSize: ws.backupInfo?.dataSize ?? 0, - lastBackup: daysInterval - }) - } - } finally { - await storageAdapter.close() - } - }) - program - .command('backup-clean ') - .description('dump workspace transactions and minio resources') - .action(async (dirName: string, days: string, cmd) => { - const daysInterval = Date.now() - parseInt(days) * 24 * 60 * 60 * 1000 - const storage = await createFileBackupStorage(dirName) - await backupRemoveLast(storage, daysInterval) - }) - - program - .command('backup-s3-compact ') - .description('Compact a given backup to just one snapshot') - .option('-f, --force', 'Force compact.', false) - .action(async (bucketName: string, dirName: string, cmd: { force: boolean, print: boolean }) => { - const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) - const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - try { - const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) - await compactBackup(toolCtx, storage, cmd.force) - } catch (err: any) { - toolCtx.error('failed to size backup', { err }) - } - await storageAdapter.close() - }) - program - .command('backup-s3-check ') - .description('Compact a given backup to just one snapshot') - .action(async (bucketName: string, dirName: string, cmd: any) => { - const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) - const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - try { - const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) - await checkBackupIntegrity(toolCtx, storage) - } catch (err: any) { - toolCtx.error('failed to size backup', { err }) - } - await storageAdapter.close() - }) - - program - .command('backup-s3-restore [date]') - .description('dump workspace transactions and minio resources') - .action(async (bucketName: string, dirName: string, workspace: string, date, cmd) => { - const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) - const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - try { - const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) - const wsid = getWorkspaceId(workspace) - const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - await restore(toolCtx, endpoint, wsid, storage, { - date: parseInt(date ?? '-1') - }) - } catch (err: any) { - toolCtx.error('failed to size backup', { err }) - } - await storageAdapter.close() - }) - program - .command('backup-s3-list ') - .description('list snaphost ids for backup') - .action(async (bucketName: string, dirName: string, cmd) => { - const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) - const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - try { - const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) - await backupList(storage) - } catch (err: any) { - toolCtx.error('failed to size backup', { err }) - } - await storageAdapter.close() - }) - - program - .command('backup-s3-size ') - .description('list snaphost ids for backup') - .action(async (bucketName: string, dirName: string, cmd) => { - const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) - const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - try { - const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) - await backupSize(storage) - } catch (err: any) { - toolCtx.error('failed to size backup', { err }) - } - await storageAdapter.close() - }) - - program - .command('backup-s3-download ') - .description('Download a full backup from s3 to local dir') - .action(async (bucketName: string, dirName: string, storeIn: string, cmd) => { - const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) - const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - try { - const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) - await backupDownload(storage, storeIn) - } catch (err: any) { - toolCtx.error('failed to size backup', { err }) - } - await storageAdapter.close() - }) - - program - .command('copy-s3-datalake') - .description('copy files from s3 to datalake') - .option('-w, --workspace ', 'Selected workspace only', '') - .option('-c, --concurrency ', 'Number of files being processed concurrently', '10') - .option('-s, --skip ', 'Number of workspaces to skip', '0') - .option('-e, --existing', 'Copy existing blobs', false) - .action(async (cmd: { workspace: string, concurrency: string, existing: boolean, skip: string }) => { - const params = { - concurrency: parseInt(cmd.concurrency), - existing: cmd.existing - } - const skip = parseInt(cmd.skip) - - const storageConfig = storageConfigFromEnv(process.env.STORAGE) - - const storages = storageConfig.storages.filter((p) => p.kind === S3_CONFIG_KIND) as S3Config[] - if (storages.length === 0) { - throw new Error('S3 storage config is required') - } - - const datalakeConfig = storageConfig.storages.find((p) => p.kind === DATALAKE_CONFIG_KIND) - if (datalakeConfig === undefined) { - throw new Error('Datalake storage config is required') - } - - toolCtx.info('using datalake', { datalake: datalakeConfig }) - - let workspaces: Workspace[] = [] - await withAccountDatabase(async (db) => { - workspaces = await listWorkspacesPure(db) - workspaces = workspaces - .filter((p) => isActiveMode(p.mode) || isArchivingMode(p.mode)) - .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) - // .sort((a, b) => b.lastVisit - a.lastVisit) - .sort((a, b) => { - if (a.backupInfo !== undefined && b.backupInfo !== undefined) { - return b.backupInfo.blobsSize - a.backupInfo.blobsSize - } else if (b.backupInfo !== undefined) { - return 1 - } else if (a.backupInfo !== undefined) { - return -1 - } else { - return b.lastVisit - a.lastVisit - } - }) - }) - - const count = workspaces.length - console.log('found workspaces', count) - - let index = 0 - for (const workspace of workspaces) { - index++ - if (index <= skip) { - toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count }) - continue - } - - toolCtx.info('processing workspace', { - workspace: workspace.workspace, - index, - count, - blobsSize: workspace.backupInfo?.blobsSize ?? 0 - }) - const workspaceId = getWorkspaceId(workspace.workspace) - const token = generateToken(systemAccountEmail, workspaceId) - const datalake = createDatalakeClient(datalakeConfig as DatalakeConfig, token) - - for (const config of storages) { - const storage = new S3Service(config) - await copyToDatalake(toolCtx, workspaceId, config, storage, datalake, params) - } - } - }) - - program - .command('restore-wiki-content-mongo') - .description('restore wiki document contents') - .option('-w, --workspace ', 'Selected workspace only', '') - .option('-d, --dryrun', 'Dry run', false) - .action(async (cmd: { workspace: string, dryrun: boolean }) => { - const params = { - dryRun: cmd.dryrun - } - - const { version } = prepareTools() - - let workspaces: Workspace[] = [] - await withAccountDatabase(async (db) => { - workspaces = await listWorkspacesPure(db) - workspaces = workspaces - .filter((p) => isActiveMode(p.mode)) - .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) - .sort((a, b) => b.lastVisit - a.lastVisit) - }) - - console.log('found workspaces', workspaces.length) - - await withStorage(async (storageAdapter) => { - const mongodbUri = getMongoDBUrl() - const client = getMongoClient(mongodbUri) - const _client = await client.getClient() - - try { - const count = workspaces.length - let index = 0 - for (const workspace of workspaces) { - index++ - - toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count }) - if (workspace.version === undefined || !deepEqual(workspace.version, version)) { - console.log(`upgrade to ${versionToString(version)} is required`) - continue - } - - const workspaceId = getWorkspaceId(workspace.workspace) - const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - - await restoreWikiContentMongo(toolCtx, wsDb, workspaceId, storageAdapter, params) - } - } finally { - client.close() - } - }) - }) - - program - .command('restore-controlled-content-mongo') - .description('restore controlled document contents') - .option('-w, --workspace ', 'Selected workspace only', '') - .option('-d, --dryrun', 'Dry run', false) - .option('-f, --force', 'Force update', false) - .action(async (cmd: { workspace: string, dryrun: boolean, force: boolean }) => { - const params = { - dryRun: cmd.dryrun - } - - const { version } = prepareTools() - - let workspaces: Workspace[] = [] - await withAccountDatabase(async (db) => { - workspaces = await listWorkspacesPure(db) - workspaces = workspaces - .filter((p) => p.mode !== 'archived') - .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) - .sort((a, b) => b.lastVisit - a.lastVisit) - }) - - console.log('found workspaces', workspaces.length) - - await withStorage(async (storageAdapter) => { await withAccountDatabase(async (db) => { - const mongodbUri = getMongoDBUrl() - const client = getMongoClient(mongodbUri) - const _client = await client.getClient() - - try { - const count = workspaces.length - let index = 0 - for (const workspace of workspaces) { - index++ - - toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count }) - - if (!cmd.force && (workspace.version === undefined || !deepEqual(workspace.version, version))) { - console.log(`upgrade to ${versionToString(version)} is required`) - continue - } - - const workspaceId = getWorkspaceId(workspace.workspace) - const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - - await restoreControlledDocContentMongo(toolCtx, wsDb, workspaceId, storageAdapter, params) - } - } finally { - client.close() - } - }) - }) - }) - - program - .command('restore-markup-ref-mongo') - .description('restore markup document content refs') - .option('-w, --workspace ', 'Selected workspace only', '') - .option('-f, --force', 'Force update', false) - .action(async (cmd: { workspace: string, force: boolean }) => { - const { txes, version } = prepareTools() - - const { hierarchy } = await buildModel(toolCtx, txes) - - let workspaces: Workspace[] = [] - await withAccountDatabase(async (db) => { - workspaces = await listWorkspacesPure(db) - workspaces = workspaces - .filter((p) => isActiveMode(p.mode)) - .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) - .sort((a, b) => b.lastVisit - a.lastVisit) - }) - - console.log('found workspaces', workspaces.length) - - await withStorage(async (storageAdapter) => { - const mongodbUri = getMongoDBUrl() - const client = getMongoClient(mongodbUri) - const _client = await client.getClient() - - try { - const count = workspaces.length - let index = 0 - for (const workspace of workspaces) { - index++ - - toolCtx.info('processing workspace', { - workspace: workspace.workspace, - version: workspace.version, - index, - count - }) - - if (!cmd.force && (workspace.version === undefined || !deepEqual(workspace.version, version))) { - console.log(`upgrade to ${versionToString(version)} is required`) - continue - } - - const workspaceId = getWorkspaceId(workspace.workspace) - const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - - await restoreMarkupRefsMongo(toolCtx, wsDb, workspaceId, hierarchy, storageAdapter) - } - } finally { - client.close() - } - }) - }) - - program - .command('confirm-email ') - .description('confirm user email') - .action(async (email: string, cmd) => { - await withAccountDatabase(async (db) => { - const account = await getAccount(db, email) - if (account?.confirmed === true) { - console.log(`Already confirmed:${email}`) - } else { - await confirmEmail(db, email) - } - }) - }) - - program - .command('diff-workspace ') - .description('restore workspace transactions and minio resources from previous dump.') - .action(async (workspace: string, cmd) => { - const { dbUrl, txes } = prepareTools() - await diffWorkspace(dbUrl, getWorkspaceId(workspace), txes) - }) - - program - .command('clear-telegram-history ') - .description('clear telegram history') - .option('-w, --workspace ', 'target workspace') - .action(async (workspace: string, cmd) => { - const { dbUrl } = prepareTools() - await withStorage(async (adapter) => { - const telegramDB = process.env.TELEGRAM_DATABASE - if (telegramDB === undefined) { - console.error('please provide TELEGRAM_DATABASE.') - process.exit(1) - } - - console.log(`clearing ${workspace} history:`) - await clearTelegramHistory(toolCtx, dbUrl, getWorkspaceId(workspace), telegramDB, adapter) - }) - }) - - program - .command('clear-telegram-all-history') - .description('clear telegram history') - .action(async (cmd) => { - const { dbUrl } = prepareTools() - await withStorage(async (adapter) => { - await withAccountDatabase(async (db) => { - const telegramDB = process.env.TELEGRAM_DATABASE - if (telegramDB === undefined) { - console.error('please provide TELEGRAM_DATABASE.') - process.exit(1) + const ws = await getWorkspace(db, workspaceId) + if (ws === null) { + throw new Error(`workspace ${workspaceId} not found`) } - const workspaces = await listWorkspacesPure(db) - - for (const w of workspaces) { - console.log(`clearing ${w.workspace} history:`) - await clearTelegramHistory(toolCtx, dbUrl, getWorkspaceId(w.workspace), telegramDB, adapter) + const workspace = ws.uuid + const wsIds = { + uuid: ws.uuid, + dataId: ws.dataId, + url: ws.url } - }) - }) - }) + const storage = await createFileBackupStorage(dirName) + const storageConfig = cmd.useStorage !== '' ? storageConfigFromEnv(process.env[cmd.useStorage]) : undefined - program - .command('generate-token ') - .description('generate token') - .option('--admin', 'Generate token with admin access', false) - .action(async (name: string, workspace: string, opt: { admin: boolean }) => { - console.log(generateToken(name, getWorkspaceId(workspace), { ...(opt.admin ? { admin: 'true' } : {}) })) - }) - program - .command('decode-token ') - .description('decode token') - .action(async (token) => { - console.log(decodeToken(token)) - }) - - program - .command('clean-workspace ') - .description('clean workspace') - .option('--recruit', 'Clean recruit', false) - .option('--tracker', 'Clean tracker', false) - .option('--removedTx', 'Clean removed transactions', false) - .action(async (workspace: string, cmd: { recruit: boolean, tracker: boolean, removedTx: boolean }) => { - const { dbUrl } = prepareTools() - await withStorage(async (adapter) => { - const wsid = getWorkspaceId(workspace) - const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - await cleanWorkspace(toolCtx, dbUrl, wsid, adapter, endpoint, cmd) - }) - }) - program - .command('clean-empty-buckets') - .option('--prefix [prefix]', 'Prefix', '') - .action(async (cmd: { prefix: string }) => { - await withStorage(async (adapter) => { - const buckets = await adapter.listBuckets(toolCtx) - for (const ws of buckets) { - if (ws.name.startsWith(cmd.prefix)) { - console.log('Checking', ws.name) - const l = await ws.list() - const docs = await l.next() - if (docs.length === 0) { - await l.close() - // No data, we could delete it. - console.log('Clean bucket', ws.name) - await ws.delete() - } else { - await l.close() - } - } - } - }) - }) - program - .command('upload-file ') - .action(async (workspace: string, local: string, remote: string, contentType: string, cmd: any) => { - const wsId: WorkspaceId = { - name: workspace - } - const token = generateToken(systemAccountEmail, wsId) - const endpoint = await getTransactorEndpoint(token) - const blobClient = new BlobClient(endpoint, token, wsId) - const buffer = readFileSync(local) - await blobClient.upload(toolCtx, remote, buffer.length, contentType, buffer) - }) - - program - .command('download-file ') - .action(async (workspace: string, remote: string, local: string, cmd: any) => { - const wsId: WorkspaceId = { - name: workspace - } - const token = generateToken(systemAccountEmail, wsId) - const endpoint = await getTransactorEndpoint(token) - const blobClient = new BlobClient(endpoint, token, wsId) - const wrstream = createWriteStream(local) - await blobClient.writeTo(toolCtx, remote, -1, { - write: (buffer, cb) => { - wrstream.write(buffer, cb) - }, - end: (cb) => { - wrstream.end(cb) - } - }) - }) - - program - .command('move-files') - .option('-w, --workspace ', 'Selected workspace only', '') - .option('-m, --move ', 'When set to true, the files will be moved, otherwise copied', 'false') - .option('-bl, --blobLimit ', 'A blob size limit in megabytes (default 50mb)', '999999') - .option('-c, --concurrency ', 'Number of files being processed concurrently', '10') - .option('--disabled', 'Include disabled workspaces', false) - .action( - async (cmd: { workspace: string, move: string, blobLimit: string, concurrency: string, disabled: boolean }) => { - const params = { - concurrency: parseInt(cmd.concurrency), - move: cmd.move === 'true' - } - - await withAccountDatabase(async (db) => { - await withStorage(async (adapter) => { - try { - const exAdapter = adapter as StorageAdapterEx - if (exAdapter.adapters === undefined || exAdapter.adapters.length < 2) { - throw new Error('bad storage config, at least two storage providers are required') - } - - console.log('moving files to storage provider', exAdapter.adapters[0].name) - - let index = 1 - const workspaces = await listWorkspacesPure(db) - workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - - const rateLimit = new RateLimiter(10) - for (const workspace of workspaces) { - if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { - continue - } - if (!isActiveMode(workspace.mode)) { - console.log('ignore non active workspace', workspace.workspace, workspace.mode) - continue - } - if (workspace.disabled === true && !cmd.disabled) { - console.log('ignore disabled workspace', workspace.workspace) - continue - } - - await rateLimit.exec(async () => { - console.log('start', workspace.workspace, index, '/', workspaces.length) - await moveFiles(toolCtx, getWorkspaceId(workspace.workspace), exAdapter, params) - console.log('done', workspace.workspace) - index += 1 - }) - } - await rateLimit.waitProcessing() - } catch (err: any) { - console.error(err) - } + const workspaceStorage: StorageAdapter | undefined = + storageConfig !== undefined ? buildStorageFromConfig(storageConfig) : undefined + await restore(toolCtx, await getWorkspaceTransactorEndpoint(workspace), wsIds, storage, { + date: parseInt(date ?? '-1'), + merge: cmd.merge, + parallel: parseInt(cmd.parallel ?? '1'), + recheck: cmd.recheck, + include: cmd.include === '*' ? undefined : new Set(cmd.include.split(';')), + skip: new Set(cmd.skip.split(';')), + storageAdapter: workspaceStorage, + historyFile: cmd.historyFile }) + await workspaceStorage?.close() }) } ) - program - .command('show-lost-files-mongo') - .option('-w, --workspace ', 'Selected workspace only', '') - .option('--disabled', 'Include disabled workspaces', false) - .option('--all', 'Show all files', false) - .action(async (cmd: { workspace: string, disabled: boolean, all: boolean }) => { - await withAccountDatabase(async (db) => { - await withStorage(async (adapter) => { - const mongodbUri = getMongoDBUrl() - const client = getMongoClient(mongodbUri) - const _client = await client.getClient() - try { - let index = 1 - const workspaces = await listWorkspacesPure(db) - workspaces.sort((a, b) => b.lastVisit - a.lastVisit) + // program + // .command('backup-list ') + // .description('list snaphost ids for backup') + // .action(async (dirName: string, cmd) => { + // const storage = await createFileBackupStorage(dirName) + // await backupList(storage) + // }) - for (const workspace of workspaces) { - if (!isActiveMode(workspace.mode)) { - console.log('ignore non active workspace', workspace.workspace, workspace.mode) - continue - } - if (workspace.disabled === true && !cmd.disabled) { - console.log('ignore disabled workspace', workspace.workspace) - continue - } + // program + // .command('backup-s3 ') + // .description('dump workspace transactions and minio resources') + // .action(async (bucketName: string, dirName: string, workspace: string, cmd) => { + // await withStorage(async (adapter) => { + // const storage = await createStorageBackupStorage(toolCtx, adapter, getWorkspaceId(bucketName), dirName) + // const wsid = getWorkspaceId(workspace) + // const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') + // await backup(toolCtx, endpoint, wsIds, storage) + // }) + // }) - if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { - continue - } + // program + // .command('backup-s3-clean ') + // .description('dump workspace transactions and minio resources') + // .action(async (bucketName: string, days: string, cmd) => { + // const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) + // const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) - try { - console.log('start', workspace.workspace, index, '/', workspaces.length) - const workspaceId = getWorkspaceId(workspace.workspace) - const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - await showLostFiles(toolCtx, workspaceId, wsDb, adapter, { showAll: cmd.all }) - console.log('done', workspace.workspace) - } catch (err) { - console.error(err) - } + // const daysInterval = Date.now() - parseInt(days) * 24 * 60 * 60 * 1000 + // try { + // const token = generateToken(systemAccountUuid, undefined, { service: 'tool' }) + // const accountClient = getAccountClient(token) + // const workspaces = (await accountClient.listWorkspaces(null, 'active')).filter((it) => { + // const lastBackup = it.backupInfo?.lastBackup ?? 0 + // if (lastBackup > daysInterval) { + // // No backup required, interval not elapsed + // return true + // } - index += 1 - } - } catch (err: any) { - console.error(err) - } finally { - client.close() - } - }) - }) - }) + // if (it.lastVisit == null) { + // return false + // } - program.command('fix-bw-workspace ').action(async (workspace: string) => { - await withStorage(async (adapter) => { - await fixMinioBW(toolCtx, getWorkspaceId(workspace), adapter) - }) - }) + // return false + // }) + // workspaces.sort((a, b) => { + // return (b.backupInfo?.backupSize ?? 0) - (a.backupInfo?.backupSize ?? 0) + // }) - program - .command('clean-removed-transactions ') - .description('clean removed transactions') - .action(async (workspace: string, cmd: any) => { - const wsid = getWorkspaceId(workspace) - const token = generateToken(systemAccountEmail, wsid) - const endpoint = await getTransactorEndpoint(token) - await cleanRemovedTransactions(wsid, endpoint) - }) + // for (const ws of workspaces) { + // const storage = await createStorageBackupStorage( + // toolCtx, + // storageAdapter, + // getWorkspaceId(bucketName), + // ws.workspace + // ) + // await backupRemoveLast(storage, daysInterval) + // const accountClient = getAccountClient(token) + // await accountClient.updateBackupInfo({ + // backups: ws.backupInfo?.backups ?? 0, + // backupSize: ws.backupInfo?.backupSize ?? 0, + // blobsSize: ws.backupInfo?.blobsSize ?? 0, + // dataSize: ws.backupInfo?.dataSize ?? 0, + // lastBackup: daysInterval + // }) + // } + // } finally { + // await storageAdapter.close() + // } + // }) - program - .command('clean-archived-spaces ') - .description('clean archived spaces') - .action(async (workspace: string, cmd: any) => { - const wsid = getWorkspaceId(workspace) - const token = generateToken(systemAccountEmail, wsid) - const endpoint = await getTransactorEndpoint(token) - await cleanArchivedSpaces(wsid, endpoint) - }) + // program + // .command('backup-clean ') + // .description('dump workspace transactions and minio resources') + // .action(async (dirName: string, days: string, cmd) => { + // const daysInterval = Date.now() - parseInt(days) * 24 * 60 * 60 * 1000 + // const storage = await createFileBackupStorage(dirName) + // await backupRemoveLast(storage, daysInterval) + // }) - program - .command('chunter-fix-comments ') - .description('chunter-fix-comments') - .action(async (workspace: string, cmd: any) => { - const wsid = getWorkspaceId(workspace) - const token = generateToken(systemAccountEmail, wsid) - const endpoint = await getTransactorEndpoint(token) - await fixCommentDoubleIdCreate(wsid, endpoint) - }) + // program + // .command('backup-s3-compact ') + // .description('Compact a given backup to just one snapshot') + // .option('-f, --force', 'Force compact.', false) + // .action(async (bucketName: string, dirName: string, cmd: { force: boolean, print: boolean }) => { + // const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) + // const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) + // try { + // const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) + // await compactBackup(toolCtx, storage, cmd.force) + // } catch (err: any) { + // toolCtx.error('failed to size backup', { err }) + // } + // await storageAdapter.close() + // }) + // program + // .command('backup-s3-check ') + // .description('Compact a given backup to just one snapshot') + // .action(async (bucketName: string, dirName: string, cmd: any) => { + // const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) + // const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) + // try { + // const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) + // await checkBackupIntegrity(toolCtx, storage) + // } catch (err: any) { + // toolCtx.error('failed to size backup', { err }) + // } + // await storageAdapter.close() + // }) - program - .command('mixin-show-foreign-attributes ') - .description('mixin-show-foreign-attributes') - .option('--mixin ', 'Mixin class', '') - .option('--property ', 'Property name', '') - .option('--detail ', 'Show details', false) - .action(async (workspace: string, cmd: { detail: boolean, mixin: string, property: string }) => { - const wsid = getWorkspaceId(workspace) - const token = generateToken(systemAccountEmail, wsid) - const endpoint = await getTransactorEndpoint(token) - await showMixinForeignAttributes(wsid, endpoint, cmd) - }) + // program + // .command('backup-s3-restore [date]') + // .description('dump workspace transactions and minio resources') + // .action(async (bucketName: string, dirName: string, workspace: string, date, cmd) => { + // const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) + // const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) + // try { + // const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) + // const wsid = getWorkspaceId(workspace) + // const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') + // await restore(toolCtx, endpoint, wsid, storage, { + // date: parseInt(date ?? '-1') + // }) + // } catch (err: any) { + // toolCtx.error('failed to size backup', { err }) + // } + // await storageAdapter.close() + // }) + // program + // .command('backup-s3-list ') + // .description('list snaphost ids for backup') + // .action(async (bucketName: string, dirName: string, cmd) => { + // const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) + // const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) + // try { + // const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) + // await backupList(storage) + // } catch (err: any) { + // toolCtx.error('failed to size backup', { err }) + // } + // await storageAdapter.close() + // }) - program - .command('mixin-fix-foreign-attributes-mongo ') - .description('mixin-fix-foreign-attributes') - .option('--mixin ', 'Mixin class', '') - .option('--property ', 'Property name', '') - .action(async (workspace: string, cmd: { mixin: string, property: string }) => { - const mongodbUri = getMongoDBUrl() - const wsid = getWorkspaceId(workspace) - const token = generateToken(systemAccountEmail, wsid) - const endpoint = await getTransactorEndpoint(token) - await fixMixinForeignAttributes(mongodbUri, wsid, endpoint, cmd) - }) + // program + // .command('backup-s3-size ') + // .description('list snaphost ids for backup') + // .action(async (bucketName: string, dirName: string, cmd) => { + // const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) + // const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) + // try { + // const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) + // await backupSize(storage) + // } catch (err: any) { + // toolCtx.error('failed to size backup', { err }) + // } + // await storageAdapter.close() + // }) + + // program + // .command('backup-s3-download ') + // .description('Download a full backup from s3 to local dir') + // .action(async (bucketName: string, dirName: string, storeIn: string, cmd) => { + // const backupStorageConfig = storageConfigFromEnv(process.env.STORAGE) + // const storageAdapter = createStorageFromConfig(backupStorageConfig.storages[0]) + // try { + // const storage = await createStorageBackupStorage(toolCtx, storageAdapter, getWorkspaceId(bucketName), dirName) + // await backupDownload(storage, storeIn) + // } catch (err: any) { + // toolCtx.error('failed to size backup', { err }) + // } + // await storageAdapter.close() + // }) + + // program + // .command('copy-s3-datalake') + // .description('copy files from s3 to datalake') + // .option('-w, --workspace ', 'Selected workspace only', '') + // .option('-c, --concurrency ', 'Number of files being processed concurrently', '10') + // .option('-s, --skip ', 'Number of workspaces to skip', '0') + // .option('-e, --existing', 'Copy existing blobs', false) + // .action(async (cmd: { workspace: string, concurrency: string, existing: boolean, skip: string }) => { + // const params = { + // concurrency: parseInt(cmd.concurrency), + // existing: cmd.existing + // } + // const skip = parseInt(cmd.skip) + + // const storageConfig = storageConfigFromEnv(process.env.STORAGE) + + // const storages = storageConfig.storages.filter((p) => p.kind === S3_CONFIG_KIND) as S3Config[] + // if (storages.length === 0) { + // throw new Error('S3 storage config is required') + // } + + // const datalakeConfig = storageConfig.storages.find((p) => p.kind === DATALAKE_CONFIG_KIND) + // if (datalakeConfig === undefined) { + // throw new Error('Datalake storage config is required') + // } + + // toolCtx.info('using datalake', { datalake: datalakeConfig }) + + // let workspaces: Workspace[] = [] + // await withAccountDatabase(async (db) => { + // workspaces = await listWorkspacesPure(db) + // workspaces = workspaces + // .filter((p) => isActiveMode(p.mode) || isArchivingMode(p.mode)) + // .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) + // // .sort((a, b) => b.lastVisit - a.lastVisit) + // .sort((a, b) => { + // if (a.backupInfo !== undefined && b.backupInfo !== undefined) { + // return b.backupInfo.blobsSize - a.backupInfo.blobsSize + // } else if (b.backupInfo !== undefined) { + // return 1 + // } else if (a.backupInfo !== undefined) { + // return -1 + // } else { + // return b.lastVisit - a.lastVisit + // } + // }) + // }) + + // const count = workspaces.length + // console.log('found workspaces', count) + + // let index = 0 + // for (const workspace of workspaces) { + // index++ + // if (index <= skip) { + // toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count }) + // continue + // } + + // toolCtx.info('processing workspace', { + // workspace: workspace.workspace, + // index, + // count, + // blobsSize: workspace.backupInfo?.blobsSize ?? 0 + // }) + // const workspaceId = getWorkspaceId(workspace.workspace) + // const token = generateToken(systemAccountEmail, workspaceId) + // const datalake = createDatalakeClient(datalakeConfig as DatalakeConfig, token) + + // for (const config of storages) { + // const storage = new S3Service(config) + // await copyToDatalake(toolCtx, workspaceId, config, storage, datalake, params) + // } + // } + // }) + + // program + // .command('restore-wiki-content-mongo') + // .description('restore wiki document contents') + // .option('-w, --workspace ', 'Selected workspace only', '') + // .option('-d, --dryrun', 'Dry run', false) + // .action(async (cmd: { workspace: string, dryrun: boolean }) => { + // const params = { + // dryRun: cmd.dryrun + // } + + // const { version } = prepareTools() + + // let workspaces: Workspace[] = [] + // await withAccountDatabase(async (db) => { + // workspaces = await listWorkspacesPure(db) + // workspaces = workspaces + // .filter((p) => isActiveMode(p.mode)) + // .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) + // .sort((a, b) => b.lastVisit - a.lastVisit) + // }) + + // console.log('found workspaces', workspaces.length) + + // await withStorage(async (storageAdapter) => { + // const mongodbUri = getMongoDBUrl() + // const client = getMongoClient(mongodbUri) + // const _client = await client.getClient() + + // try { + // const count = workspaces.length + // let index = 0 + // for (const workspace of workspaces) { + // index++ + + // toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count }) + // if (workspace.version === undefined || !deepEqual(workspace.version, version)) { + // console.log(`upgrade to ${versionToString(version)} is required`) + // continue + // } + + // const workspaceId = getWorkspaceId(workspace.workspace) + // const workspaceDataId = workspace.dataId ?? workspace.uuid + // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) + + // await restoreWikiContentMongo(toolCtx, wsDb, workspaceDataId, storageAdapter, params) + // } + // } finally { + // client.close() + // } + // }) + // }) + + // program + // .command('restore-controlled-content-mongo') + // .description('restore controlled document contents') + // .option('-w, --workspace ', 'Selected workspace only', '') + // .option('-d, --dryrun', 'Dry run', false) + // .option('-f, --force', 'Force update', false) + // .action(async (cmd: { workspace: string, dryrun: boolean, force: boolean }) => { + // const params = { + // dryRun: cmd.dryrun + // } + + // const { version } = prepareTools() + + // let workspaces: Workspace[] = [] + // await withAccountDatabase(async (db) => { + // workspaces = await listWorkspacesPure(db) + // workspaces = workspaces + // .filter((p) => p.mode !== 'archived') + // .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) + // .sort((a, b) => b.lastVisit - a.lastVisit) + // }) + + // console.log('found workspaces', workspaces.length) + + // await withStorage(async (storageAdapter) => { + // await withAccountDatabase(async (db) => { + // const mongodbUri = getMongoDBUrl() + // const client = getMongoClient(mongodbUri) + // const _client = await client.getClient() + + // try { + // const count = workspaces.length + // let index = 0 + // for (const workspace of workspaces) { + // index++ + + // toolCtx.info('processing workspace', { workspace: workspace.workspace, index, count }) + + // if (!cmd.force && (workspace.version === undefined || !deepEqual(workspace.version, version))) { + // console.log(`upgrade to ${versionToString(version)} is required`) + // continue + // } + + // const workspaceId = getWorkspaceId(workspace.workspace) + // const workspaceDataId = workspace.dataId ?? workspace.uuid + // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) + + // await restoreControlledDocContentMongo(toolCtx, wsDb, workspaceDataId, storageAdapter, params) + // } + // } finally { + // client.close() + // } + // }) + // }) + // }) + + // program + // .command('restore-markup-ref-mongo') + // .description('restore markup document content refs') + // .option('-w, --workspace ', 'Selected workspace only', '') + // .option('-f, --force', 'Force update', false) + // .action(async (cmd: { workspace: string, force: boolean }) => { + // const { txes, version } = prepareTools() + + // const { hierarchy } = await buildModel(toolCtx, txes) + + // let workspaces: Workspace[] = [] + // await withAccountDatabase(async (db) => { + // workspaces = await listWorkspacesPure(db) + // workspaces = workspaces + // .filter((p) => isActiveMode(p.mode)) + // .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) + // .sort((a, b) => b.lastVisit - a.lastVisit) + // }) + + // console.log('found workspaces', workspaces.length) + + // await withStorage(async (storageAdapter) => { + // const mongodbUri = getMongoDBUrl() + // const client = getMongoClient(mongodbUri) + // const _client = await client.getClient() + + // try { + // const count = workspaces.length + // let index = 0 + // for (const workspace of workspaces) { + // index++ + + // toolCtx.info('processing workspace', { + // workspace: workspace.workspace, + // version: workspace.version, + // index, + // count + // }) + + // if (!cmd.force && (workspace.version === undefined || !deepEqual(workspace.version, version))) { + // console.log(`upgrade to ${versionToString(version)} is required`) + // continue + // } + + // const workspaceId = getWorkspaceId(workspace.workspace) + // const workspaceDataId = workspace.dataId ?? workspace.uuid + // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) + + // await restoreMarkupRefsMongo(toolCtx, wsDb, workspaceDataId, hierarchy, storageAdapter) + // } + // } finally { + // client.close() + // } + // }) + // }) + + // program + // .command('confirm-email ') + // .description('confirm user email') + // .action(async (email: string, cmd) => { + // await withAccountDatabase(async (db) => { + // const account = await getAccount(db, email) + // if (account?.confirmed === true) { + // console.log(`Already confirmed:${email}`) + // } else { + // await confirmEmail(db, email) + // } + // }) + // }) + + // program + // .command('diff-workspace ') + // .description('restore workspace transactions and minio resources from previous dump.') + // .action(async (workspace: string, cmd) => { + // const { dbUrl, txes } = prepareTools() + // await diffWorkspace(dbUrl, workspace, txes) + // }) + + // program + // .command('clear-telegram-history ') + // .description('clear telegram history') + // .option('-w, --workspace ', 'target workspace') + // .action(async (workspace: string, cmd) => { + // const { dbUrl } = prepareTools() + // await withStorage(async (adapter) => { + // const telegramDB = process.env.TELEGRAM_DATABASE + // if (telegramDB === undefined) { + // console.error('please provide TELEGRAM_DATABASE.') + // process.exit(1) + // } + + // console.log(`clearing ${workspace} history:`) + // await clearTelegramHistory(toolCtx, dbUrl, getWorkspaceId(workspace), telegramDB, adapter) + // }) + // }) + + // program + // .command('clear-telegram-all-history') + // .description('clear telegram history') + // .action(async (cmd) => { + // const { dbUrl } = prepareTools() + // await withStorage(async (adapter) => { + // await withAccountDatabase(async (db) => { + // const telegramDB = process.env.TELEGRAM_DATABASE + // if (telegramDB === undefined) { + // console.error('please provide TELEGRAM_DATABASE.') + // process.exit(1) + // } + + // const workspaces = await listWorkspacesPure(db) + + // for (const w of workspaces) { + // console.log(`clearing ${w.workspace} history:`) + // await clearTelegramHistory(toolCtx, dbUrl, getWorkspaceId(w.workspace), telegramDB, adapter) + // } + // }) + // }) + // }) + + // program + // .command('generate-token ') + // .description('generate token') + // .option('--admin', 'Generate token with admin access', false) + // .action(async (name: string, workspace: string, opt: { admin: boolean }) => { + // console.log(generateToken(name, getWorkspaceId(workspace), { ...(opt.admin ? { admin: 'true' } : {}) })) + // }) + // program + // .command('decode-token ') + // .description('decode token') + // .action(async (token) => { + // console.log(decodeToken(token)) + // }) + + // program + // .command('clean-workspace ') + // .description('clean workspace') + // .option('--recruit', 'Clean recruit', false) + // .option('--tracker', 'Clean tracker', false) + // .option('--removedTx', 'Clean removed transactions', false) + // .action(async (workspace: string, cmd: { recruit: boolean, tracker: boolean, removedTx: boolean }) => { + // const { dbUrl } = prepareTools() + // await withStorage(async (adapter) => { + // const wsid = getWorkspaceId(workspace) + // const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') + // await cleanWorkspace(toolCtx, dbUrl, wsid, adapter, endpoint, cmd) + // }) + // }) + // program + // .command('clean-empty-buckets') + // .option('--prefix [prefix]', 'Prefix', '') + // .action(async (cmd: { prefix: string }) => { + // await withStorage(async (adapter) => { + // const buckets = await adapter.listBuckets(toolCtx) + // for (const ws of buckets) { + // if (ws.name.startsWith(cmd.prefix)) { + // console.log('Checking', ws.name) + // const l = await ws.list() + // const docs = await l.next() + // if (docs.length === 0) { + // await l.close() + // // No data, we could delete it. + // console.log('Clean bucket', ws.name) + // await ws.delete() + // } else { + // await l.close() + // } + // } + // } + // }) + // }) + // program + // .command('upload-file ') + // .action(async (workspace: string, local: string, remote: string, contentType: string, cmd: any) => { + // const wsId: WorkspaceId = { + // name: workspace + // } + // const token = generateToken(systemAccountEmail, wsId) + // const endpoint = await getTransactorEndpoint(token) + // const blobClient = new BlobClient(endpoint, token, wsId) + // const buffer = readFileSync(local) + // await blobClient.upload(toolCtx, remote, buffer.length, contentType, buffer) + // }) + + // program + // .command('download-file ') + // .action(async (workspace: string, remote: string, local: string, cmd: any) => { + // const wsId: WorkspaceId = { + // name: workspace + // } + // const token = generateToken(systemAccountEmail, wsId) + // const endpoint = await getTransactorEndpoint(token) + // const blobClient = new BlobClient(endpoint, token, wsId) + // const wrstream = createWriteStream(local) + // await blobClient.writeTo(toolCtx, remote, -1, { + // write: (buffer, cb) => { + // wrstream.write(buffer, cb) + // }, + // end: (cb) => { + // wrstream.end(cb) + // } + // }) + // }) + + // program + // .command('move-files') + // .option('-w, --workspace ', 'Selected workspace only', '') + // .option('-m, --move ', 'When set to true, the files will be moved, otherwise copied', 'false') + // .option('-bl, --blobLimit ', 'A blob size limit in megabytes (default 50mb)', '999999') + // .option('-c, --concurrency ', 'Number of files being processed concurrently', '10') + // .option('--disabled', 'Include disabled workspaces', false) + // .action( + // async (cmd: { workspace: string, move: string, blobLimit: string, concurrency: string, disabled: boolean }) => { + // const params = { + // concurrency: parseInt(cmd.concurrency), + // move: cmd.move === 'true' + // } + + // await withAccountDatabase(async (db) => { + // await withStorage(async (adapter) => { + // try { + // const exAdapter = adapter as StorageAdapterEx + // if (exAdapter.adapters === undefined || exAdapter.adapters.length < 2) { + // throw new Error('bad storage config, at least two storage providers are required') + // } + + // console.log('moving files to storage provider', exAdapter.adapters[0].name) + + // let index = 1 + // const workspaces = await listWorkspacesPure(db) + // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) + + // const rateLimit = new RateLimiter(10) + // for (const workspace of workspaces) { + // if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { + // continue + // } + // if (!isActiveMode(workspace.mode)) { + // console.log('ignore non active workspace', workspace.workspace, workspace.mode) + // continue + // } + // if (workspace.disabled === true && !cmd.disabled) { + // console.log('ignore disabled workspace', workspace.workspace) + // continue + // } + + // await rateLimit.exec(async () => { + // console.log('start', workspace.workspace, index, '/', workspaces.length) + // await moveFiles(toolCtx, getWorkspaceId(workspace.workspace), exAdapter, params) + // console.log('done', workspace.workspace) + // index += 1 + // }) + // } + // await rateLimit.waitProcessing() + // } catch (err: any) { + // console.error(err) + // } + // }) + // }) + // } + // ) + + // program + // .command('show-lost-files-mongo') + // .option('-w, --workspace ', 'Selected workspace only', '') + // .option('--disabled', 'Include disabled workspaces', false) + // .option('--all', 'Show all files', false) + // .action(async (cmd: { workspace: string, disabled: boolean, all: boolean }) => { + // await withAccountDatabase(async (db) => { + // await withStorage(async (adapter) => { + // const mongodbUri = getMongoDBUrl() + // const client = getMongoClient(mongodbUri) + // const _client = await client.getClient() + // try { + // let index = 1 + // const workspaces = await listWorkspacesPure(db) + // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) + + // for (const workspace of workspaces) { + // if (!isActiveMode(workspace.mode)) { + // console.log('ignore non active workspace', workspace.workspace, workspace.mode) + // continue + // } + // if (workspace.disabled === true && !cmd.disabled) { + // console.log('ignore disabled workspace', workspace.workspace) + // continue + // } + + // if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { + // continue + // } + + // try { + // console.log('start', workspace.workspace, index, '/', workspaces.length) + // const workspaceId = getWorkspaceId(workspace.workspace) + // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) + // await showLostFiles(toolCtx, workspaceId, wsDb, adapter, { showAll: cmd.all }) + // console.log('done', workspace.workspace) + // } catch (err) { + // console.error(err) + // } + + // index += 1 + // } + // } catch (err: any) { + // console.error(err) + // } finally { + // client.close() + // } + // }) + // }) + // }) + + // program.command('fix-bw-workspace ').action(async (workspace: string) => { + // await withStorage(async (adapter) => { + // await fixMinioBW(toolCtx, getWorkspaceId(workspace), adapter) + // }) + // }) + + // program + // .command('clean-removed-transactions ') + // .description('clean removed transactions') + // .action(async (workspace: string, cmd: any) => { + // const wsid = getWorkspaceId(workspace) + // const token = generateToken(systemAccountEmail, wsid) + // const endpoint = await getTransactorEndpoint(token) + // await cleanRemovedTransactions(wsid, endpoint) + // }) + + // program + // .command('clean-archived-spaces ') + // .description('clean archived spaces') + // .action(async (workspace: string, cmd: any) => { + // const wsid = getWorkspaceId(workspace) + // const token = generateToken(systemAccountEmail, wsid) + // const endpoint = await getTransactorEndpoint(token) + // await cleanArchivedSpaces(wsid, endpoint) + // }) + + // program + // .command('chunter-fix-comments ') + // .description('chunter-fix-comments') + // .action(async (workspace: string, cmd: any) => { + // const wsid = getWorkspaceId(workspace) + // const token = generateToken(systemAccountEmail, wsid) + // const endpoint = await getTransactorEndpoint(token) + // await fixCommentDoubleIdCreate(wsid, endpoint) + // }) + + // program + // .command('mixin-show-foreign-attributes ') + // .description('mixin-show-foreign-attributes') + // .option('--mixin ', 'Mixin class', '') + // .option('--property ', 'Property name', '') + // .option('--detail ', 'Show details', false) + // .action(async (workspace: string, cmd: { detail: boolean, mixin: string, property: string }) => { + // const wsid = getWorkspaceId(workspace) + // const token = generateToken(systemAccountEmail, wsid) + // const endpoint = await getTransactorEndpoint(token) + // await showMixinForeignAttributes(wsid, endpoint, cmd) + // }) + + // program + // .command('mixin-fix-foreign-attributes-mongo ') + // .description('mixin-fix-foreign-attributes') + // .option('--mixin ', 'Mixin class', '') + // .option('--property ', 'Property name', '') + // .action(async (workspace: string, cmd: { mixin: string, property: string }) => { + // const mongodbUri = getMongoDBUrl() + // const wsid = getWorkspaceId(workspace) + // const token = generateToken(systemAccountEmail, wsid) + // const endpoint = await getTransactorEndpoint(token) + // FIXME: add dataId + // await fixMixinForeignAttributes(mongodbUri, wsid, endpoint, cmd) + // }) program .command('configure ') @@ -1774,157 +1793,162 @@ export function devTool ( .option('--disable ', 'Disable plugin configuration', '') .option('--list', 'List plugin states', false) .action(async (workspace: string, cmd: { enable: string, disable: string, list: boolean }) => { - console.log(JSON.stringify(cmd)) - const wsid = getWorkspaceId(workspace) - const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - await changeConfiguration(wsid, endpoint, cmd) - }) - - program - .command('configure-all') - .description('configure all spaces') - .option('--enable ', 'Enable plugin configuration', '') - .option('--disable ', 'Disable plugin configuration', '') - .option('--list', 'List plugin states', false) - .action(async (cmd: { enable: string, disable: string, list: boolean }) => { await withAccountDatabase(async (db) => { - console.log('configure all workspaces') console.log(JSON.stringify(cmd)) - const workspaces = await listWorkspacesRaw(db) - for (const ws of workspaces) { - console.log('configure', ws.workspaceName ?? ws.workspace) - const wsid = getWorkspaceId(ws.workspace) - const token = generateToken(systemAccountEmail, wsid) - const endpoint = await getTransactorEndpoint(token) - await changeConfiguration(wsid, endpoint, cmd) + const ws = await getWorkspace(db, workspace) + if (ws === null) { + throw new Error(`workspace ${workspace} not found`) } + + await changeConfiguration(ws.uuid, await getWorkspaceTransactorEndpoint(ws.uuid), cmd) }) }) - program - .command('optimize-model ') - .description('optimize model') - .action(async (workspace: string, cmd: { enable: string, disable: string, list: boolean }) => { - console.log(JSON.stringify(cmd)) - const wsid = getWorkspaceId(workspace) - const token = generateToken(systemAccountEmail, wsid) - const endpoint = await getTransactorEndpoint(token) - await optimizeModel(wsid, endpoint) - }) + // program + // .command('configure-all') + // .description('configure all spaces') + // .option('--enable ', 'Enable plugin configuration', '') + // .option('--disable ', 'Disable plugin configuration', '') + // .option('--list', 'List plugin states', false) + // .action(async (cmd: { enable: string, disable: string, list: boolean }) => { + // await withAccountDatabase(async (db) => { + // console.log('configure all workspaces') + // console.log(JSON.stringify(cmd)) + // const workspaces = await listWorkspacesRaw(db) + // for (const ws of workspaces) { + // console.log('configure', ws.workspaceName ?? ws.workspace) + // const wsid = getWorkspaceId(ws.workspace) + // const token = generateToken(systemAccountEmail, wsid) + // const endpoint = await getTransactorEndpoint(token) + // await changeConfiguration(wsid, endpoint, cmd) + // } + // }) + // }) - program - .command('benchmark') - .description('benchmark') - .option('--from ', 'Min client count', '10') - .option('--steps ', 'Step with client count', '10') - .option('--sleep ', 'Random Delay max between operations', '0') - .option('--binary ', 'Use binary data transfer', false) - .option('--compression ', 'Use protocol compression', false) - .option('--write ', 'Perform write operations', false) - .option('--workspaces ', 'Workspaces to test on, comma separated', '') - .option('--mode ', 'A benchmark mode. Supported values: `find-all`, `connect-only` ', 'find-all') - .action( - async (cmd: { - from: string - steps: string - sleep: string - workspaces: string - binary: string - compression: string - write: string - mode: 'find-all' | 'connect-only' - }) => { - await withAccountDatabase(async (db) => { - console.log(JSON.stringify(cmd)) - if (!['find-all', 'connect-only'].includes(cmd.mode)) { - console.log('wrong mode') - return - } + // program + // .command('optimize-model ') + // .description('optimize model') + // .action(async (workspace: string, cmd: { enable: string, disable: string, list: boolean }) => { + // console.log(JSON.stringify(cmd)) + // const wsid = getWorkspaceId(workspace) + // const token = generateToken(systemAccountEmail, wsid) + // const endpoint = await getTransactorEndpoint(token) + // await optimizeModel(wsid, endpoint) + // }) - const allWorkspacesPure = Array.from(await listWorkspacesPure(db)) - const allWorkspaces = new Map(allWorkspacesPure.map((it) => [it.workspace, it])) + // program + // .command('benchmark') + // .description('benchmark') + // .option('--from ', 'Min client count', '10') + // .option('--steps ', 'Step with client count', '10') + // .option('--sleep ', 'Random Delay max between operations', '0') + // .option('--binary ', 'Use binary data transfer', false) + // .option('--compression ', 'Use protocol compression', false) + // .option('--write ', 'Perform write operations', false) + // .option('--workspaces ', 'Workspaces to test on, comma separated', '') + // .option('--mode ', 'A benchmark mode. Supported values: `find-all`, `connect-only` ', 'find-all') + // .action( + // async (cmd: { + // from: string + // steps: string + // sleep: string + // workspaces: string + // binary: string + // compression: string + // write: string + // mode: 'find-all' | 'connect-only' + // }) => { + // await withAccountDatabase(async (db) => { + // console.log(JSON.stringify(cmd)) + // if (!['find-all', 'connect-only'].includes(cmd.mode)) { + // console.log('wrong mode') + // return + // } - let workspaces = cmd.workspaces - .split(',') - .map((it) => it.trim()) - .filter((it) => it.length > 0) - .map((it) => getWorkspaceId(it)) + // const allWorkspacesPure = Array.from(await listWorkspacesPure(db)) + // const allWorkspaces = new Map(allWorkspacesPure.map((it) => [it.workspace, it])) - if (cmd.workspaces.length === 0) { - workspaces = allWorkspacesPure.map((it) => getWorkspaceId(it.workspace)) - } - const accounts = new Map(Array.from(await listAccounts(db)).map((it) => [it._id.toString(), it.email])) + // let workspaces = cmd.workspaces + // .split(',') + // .map((it) => it.trim()) + // .filter((it) => it.length > 0) + // .map((it) => getWorkspaceId(it)) - const accountWorkspaces = new Map() - for (const ws of workspaces) { - const wsInfo = allWorkspaces.get(ws.name) - if (wsInfo !== undefined) { - accountWorkspaces.set( - ws.name, - wsInfo.accounts.map((it) => accounts.get(it.toString()) as string) - ) - } - } - await benchmark(workspaces, accountWorkspaces, accountsUrl, { - steps: parseInt(cmd.steps), - from: parseInt(cmd.from), - sleep: parseInt(cmd.sleep), - binary: cmd.binary === 'true', - compression: cmd.compression === 'true', - write: cmd.write === 'true', - mode: cmd.mode - }) - }) - } - ) - program - .command('benchmarkWorker') - .description('benchmarkWorker') - .action(async (cmd: any) => { - console.log(JSON.stringify(cmd)) - benchmarkWorker() - }) + // if (cmd.workspaces.length === 0) { + // workspaces = allWorkspacesPure.map((it) => getWorkspaceId(it.workspace)) + // } + // const accounts = new Map(Array.from(await listAccounts(db)).map((it) => [it._id.toString(), it.email])) - program - .command('stress ') - .description('stress benchmark') - .option('--mode ', 'A benchmark mode. Supported values: `wrong`, `connect-disconnect` ', 'wrong') - .action(async (transactor: string, cmd: { mode: StressBenchmarkMode }) => { - await stressBenchmark(transactor, cmd.mode) - }) + // const accountWorkspaces = new Map() + // for (const ws of workspaces) { + // const wsInfo = allWorkspaces.get(ws.name) + // if (wsInfo !== undefined) { + // accountWorkspaces.set( + // ws.name, + // wsInfo.accounts.map((it) => accounts.get(it.toString()) as string) + // ) + // } + // } + // await benchmark(workspaces, accountWorkspaces, accountsUrl, { + // steps: parseInt(cmd.steps), + // from: parseInt(cmd.from), + // sleep: parseInt(cmd.sleep), + // binary: cmd.binary === 'true', + // compression: cmd.compression === 'true', + // write: cmd.write === 'true', + // mode: cmd.mode + // }) + // }) + // } + // ) + // program + // .command('benchmarkWorker') + // .description('benchmarkWorker') + // .action(async (cmd: any) => { + // console.log(JSON.stringify(cmd)) + // benchmarkWorker() + // }) - program - .command('fix-skills-mongo ') - .description('fix skills for workspace') - .action(async (workspace: string, step: string) => { - const mongodbUri = getMongoDBUrl() - const wsid = getWorkspaceId(workspace) - const token = generateToken(systemAccountEmail, wsid) - const endpoint = await getTransactorEndpoint(token) - await fixSkills(mongodbUri, wsid, endpoint, step) - }) + // program + // .command('stress ') + // .description('stress benchmark') + // .option('--mode ', 'A benchmark mode. Supported values: `wrong`, `connect-disconnect` ', 'wrong') + // .action(async (transactor: string, cmd: { mode: StressBenchmarkMode }) => { + // await stressBenchmark(transactor, cmd.mode) + // }) - program - .command('restore-ats-types-mongo ') - .description('Restore recruiting task types for workspace') - .action(async (workspace: string) => { - const mongodbUri = getMongoDBUrl() - console.log('Restoring recruiting task types in workspace ', workspace, '...') - const wsid = getWorkspaceId(workspace) - const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - await restoreRecruitingTaskTypes(mongodbUri, wsid, endpoint) - }) + // program + // .command('fix-skills-mongo ') + // .description('fix skills for workspace') + // .action(async (workspace: string, step: string) => { + // const mongodbUri = getMongoDBUrl() + // const wsid = getWorkspaceId(workspace) + // const token = generateToken(systemAccountEmail, wsid) + // const endpoint = await getTransactorEndpoint(token) + // await fixSkills(mongodbUri, wsid, endpoint, step) + // }) - program - .command('restore-ats-types-2-mongo ') - .description('Restore recruiting task types for workspace 2') - .action(async (workspace: string) => { - const mongodbUri = getMongoDBUrl() - console.log('Restoring recruiting task types in workspace ', workspace, '...') - const wsid = getWorkspaceId(workspace) - const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - await restoreHrTaskTypesFromUpdates(mongodbUri, wsid, endpoint) - }) + // program + // .command('restore-ats-types-mongo ') + // .description('Restore recruiting task types for workspace') + // .action(async (workspace: string) => { + // const mongodbUri = getMongoDBUrl() + // console.log('Restoring recruiting task types in workspace ', workspace, '...') + // const wsid = getWorkspaceId(workspace) + // const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') + // await restoreRecruitingTaskTypes(mongodbUri, wsid, endpoint) + // }) + + // program + // .command('restore-ats-types-2-mongo ') + // .description('Restore recruiting task types for workspace 2') + // .action(async (workspace: string) => { + // const mongodbUri = getMongoDBUrl() + // console.log('Restoring recruiting task types in workspace ', workspace, '...') + // const wsid = getWorkspaceId(workspace) + // const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') + // await restoreHrTaskTypesFromUpdates(mongodbUri, wsid, endpoint) + // }) program .command('change-field ') @@ -1939,9 +1963,13 @@ export function devTool ( workspace: string, cmd: { objectId: string, objectClass: string, type: string, attribute: string, value: string, domain: string } ) => { - const wsid = getWorkspaceId(workspace) - const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - await updateField(wsid, endpoint, cmd) + await withAccountDatabase(async (db) => { + const ws = await getWorkspace(db, workspace) + if (ws === null) { + throw new Error(`workspace ${workspace} not found`) + } + await updateField(ws.uuid, await getWorkspaceTransactorEndpoint(ws.uuid), cmd) + }) } ) @@ -1955,255 +1983,340 @@ export function devTool ( process.exit(1) } - const wsid = getWorkspaceId(workspace) - const token = generateToken(systemAccountEmail, wsid) - - console.log('reindex workspace', workspace) - await reindexWorkspace(toolCtx, fulltextUrl, token) - console.log('done', workspace) - }) - - program - .command('fulltext-reindex-all') - .description('reindex workspaces') - .action(async () => { - const fulltextUrl = process.env.FULLTEXT_URL - if (fulltextUrl === undefined) { - console.error('please provide FULLTEXT_URL') - process.exit(1) - } - await withAccountDatabase(async (db) => { - const workspaces = await listWorkspacesRaw(db) - workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - for (const workspace of workspaces) { - const wsid = getWorkspaceId(workspace.workspace) - const token = generateToken(systemAccountEmail, wsid) + const ws = await getWorkspace(db, workspace) - console.log('reindex workspace', workspace) - await reindexWorkspace(toolCtx, fulltextUrl, token) - console.log('done', workspace) - } - }) - }) - - program - .command('remove-duplicates-ids-mongo ') - .description('remove duplicates ids for futue migration') - .action(async (workspaces: string) => { - const mongodbUri = getMongoDBUrl() - await withStorage(async (adapter) => { - await removeDuplicateIds(toolCtx, mongodbUri, adapter, accountsUrl, workspaces) - }) - }) - - program.command('move-to-pg ').action(async (region: string) => { - const { dbUrl } = prepareTools() - const mongodbUri = getMongoDBUrl() - - await withAccountDatabase(async (db) => { - const workspaces = await listWorkspacesRaw(db) - workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - await moveFromMongoToPG( - db, - mongodbUri, - dbUrl, - workspaces.filter((p) => p.region !== region), - region - ) - }) - }) - - program - .command('move-workspace-to-pg ') - .option('-i, --include ', 'A list of ; separated domain names to include during backup', '*') - .option('-f|--force [force]', 'Force update', false) - .action( - async ( - workspace: string, - region: string, - cmd: { - include: string - force: boolean - } - ) => { - const { dbUrl } = prepareTools() - const mongodbUri = getMongoDBUrl() - - await withAccountDatabase(async (db) => { - const workspaceInfo = await getWorkspaceById(db, workspace) - if (workspaceInfo === null) { - throw new Error(`workspace ${workspace} not found`) - } - if (workspaceInfo.region === region && !cmd.force) { - throw new Error(`workspace ${workspace} is already migrated`) - } - await moveWorkspaceFromMongoToPG( - db, - mongodbUri, - dbUrl, - workspaceInfo, - region, - cmd.include === '*' ? undefined : new Set(cmd.include.split(';').map((it) => it.trim())), - cmd.force - ) - }) - } - ) - - program.command('move-account-db-to-pg').action(async () => { - const { dbUrl } = prepareTools() - const mongodbUri = getMongoDBUrl() - - if (mongodbUri === dbUrl) { - throw new Error('MONGO_URL and DB_URL are the same') - } - - await withAccountDatabase(async (pgDb) => { - await withAccountDatabase(async (mongoDb) => { - await moveAccountDbFromMongoToPG(toolCtx, mongoDb, pgDb) - }, mongodbUri) - }, dbUrl) - }) - - program - .command('perfomance') - .option('-p, --parallel', '', false) - .action(async (cmd: { parallel: boolean }) => { - const { txes, version, migrateOperations } = prepareTools() - await withAccountDatabase(async (db) => { - const email = generateId() - const ws = generateId() - const wsid = getWorkspaceId(ws) - const start = new Date() - const measureCtx = new MeasureMetricsContext('create-workspace', {}) - const wsInfo = await createWorkspaceRecord(measureCtx, db, null, email, ws, ws) - - // update the record so it's not taken by one of the workers for the next 60 seconds - await updateWorkspace(db, wsInfo, { - mode: 'creating', - progress: 0, - lastProcessingTime: Date.now() + 1000 * 60 - }) - - await createWorkspace(measureCtx, version, null, wsInfo, txes, migrateOperations, undefined, true) - - await updateWorkspace(db, wsInfo, { - mode: 'active', - progress: 100, - disabled: false, - version - }) - await createAcc(toolCtx, db, null, email, '1234', '', '', true) - await assignAccountToWs(toolCtx, db, null, email, ws, AccountRole.User) - console.log('Workspace created in', new Date().getTime() - start.getTime(), 'ms') - const token = generateToken(systemAccountEmail, wsid) - const endpoint = await getTransactorEndpoint(token, 'external') - await generateWorkspaceData(endpoint, ws, cmd.parallel, email) - await testFindAll(endpoint, ws, email) - await dropWorkspace(toolCtx, db, null, ws) - }) - }) - - program - .command('reset-ws-attempts ') - .description('Reset workspace creation/upgrade attempts counter') - .action(async (workspace) => { - await withAccountDatabase(async (db) => { - const info = await getWorkspaceById(db, workspace) - if (info === null) { + if (ws == null) { throw new Error(`workspace ${workspace} not found`) } - await updateWorkspace(db, info, { - attempts: 0 - }) - - console.log('Attempts counter for workspace', workspace, 'has been reset') + console.log('reindex workspace', workspace) + await reindexWorkspace(toolCtx, fulltextUrl, getToolToken(ws?.uuid)) + console.log('done', workspace) }) }) - program - .command('generate-uuid-workspaces') - .description('generate uuids for all workspaces which are missing it') - .option('-d, --dryrun', 'Dry run', false) - .action(async (cmd: { dryrun: boolean }) => { - await withAccountDatabase(async (db) => { - console.log('generate uuids for all workspaces which are missing it') - await generateUuidMissingWorkspaces(toolCtx, db, cmd.dryrun) - }) - }) + // program + // .command('fulltext-reindex-all') + // .description('reindex workspaces') + // .action(async () => { + // const fulltextUrl = process.env.FULLTEXT_URL + // if (fulltextUrl === undefined) { + // console.error('please provide FULLTEXT_URL') + // process.exit(1) + // } - program - .command('update-data-wsid-to-uuid') - .description('updates workspaceId in pg/cr to uuid') - .option('-d, --dryrun', 'Dry run', false) - .action(async (cmd: { dryrun: boolean }) => { - await withAccountDatabase(async (db) => { - console.log('updates workspaceId in pg/cr to uuid') - const { dbUrl } = prepareTools() - await updateDataWorkspaceIdToUuid(toolCtx, db, dbUrl, cmd.dryrun) - }) - }) + // await withAccountDatabase(async (db) => { + // const workspaces = await listWorkspacesRaw(db) + // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) + // for (const workspace of workspaces) { + // const wsid = getWorkspaceId(workspace.workspace) + // const token = generateToken(systemAccountEmail, wsid) - program - .command('add-controlled-doc-rank-mongo') - .description('add rank to controlled documents') - .option('-w, --workspace ', 'Selected workspace only', '') - .action(async (cmd: { workspace: string }) => { - const { version } = prepareTools() + // console.log('reindex workspace', workspace) + // await reindexWorkspace(toolCtx, fulltextUrl, token) + // console.log('done', workspace) + // } + // }) + // }) - let workspaces: Workspace[] = [] - await withAccountDatabase(async (db) => { - workspaces = await listWorkspacesPure(db) - workspaces = workspaces - .filter((p) => isActiveMode(p.mode)) - .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) - .sort((a, b) => b.lastVisit - a.lastVisit) - }) + // program + // .command('remove-duplicates-ids-mongo ') + // .description('remove duplicates ids for futue migration') + // .action(async (workspaces: string) => { + // const mongodbUri = getMongoDBUrl() + // await withStorage(async (adapter) => { + // await removeDuplicateIds(toolCtx, mongodbUri, adapter, accountsUrl, workspaces) + // }) + // }) - console.log('found workspaces', workspaces.length) + // program.command('move-to-pg ').action(async (region: string) => { + // const { dbUrl } = prepareTools() + // const mongodbUri = getMongoDBUrl() - const mongodbUri = getMongoDBUrl() - const client = getMongoClient(mongodbUri) - const _client = await client.getClient() + // await withAccountDatabase(async (db) => { + // const workspaces = await listWorkspacesRaw(db) + // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) + // await moveFromMongoToPG( + // db, + // mongodbUri, + // dbUrl, + // workspaces.filter((p) => p.region !== region), + // region + // ) + // }) + // }) - try { - const count = workspaces.length - let index = 0 - for (const workspace of workspaces) { - index++ + // program + // .command('move-workspace-to-pg ') + // .option('-i, --include ', 'A list of ; separated domain names to include during backup', '*') + // .option('-f|--force [force]', 'Force update', false) + // .action( + // async ( + // workspace: string, + // region: string, + // cmd: { + // include: string + // force: boolean + // } + // ) => { + // const { dbUrl } = prepareTools() + // const mongodbUri = getMongoDBUrl() - toolCtx.info('processing workspace', { - workspace: workspace.workspace, - version: workspace.version, - index, - count - }) + // await withAccountDatabase(async (db) => { + // const ws = await getWorkspace(db, workspace) + // if (ws === null) { + // throw new Error(`workspace ${workspace} not found`) + // } + // await updateField(ws.uuid, await getWorkspaceTransactorEndpoint(ws.uuid), cmd) + // }) + // } + // ) - if (workspace.version === undefined || !deepEqual(workspace.version, version)) { - console.log(`upgrade to ${versionToString(version)} is required`) - continue - } - const workspaceId = getWorkspaceId(workspace.workspace) - const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) + // program + // .command('recreate-elastic-indexes-mongo ') + // .description('reindex workspace to elastic') + // .action(async (workspace: string) => { + // const mongodbUri = getMongoDBUrl() + // const wsid = getWorkspaceId(workspace) + // await recreateElastic(mongodbUri, wsid) + // }) - await addControlledDocumentRank(toolCtx, wsDb, workspaceId) - } - } finally { - client.close() - } - }) + // program + // .command('recreate-all-elastic-indexes-mongo') + // .description('reindex elastic') + // .action(async () => { + // const { dbUrl } = prepareTools() + // const mongodbUri = getMongoDBUrl() - program - .command('fill-github-users') - .option('-t, --token ', 'Github token to increase the limit of requests to GitHub') - .description('adds github username info to all accounts') - .action(async (cmd: { token?: string }) => { - await withAccountDatabase(async (db) => { - await fillGithubUsers(toolCtx, db, cmd.token) - }) - }) + // await withAccountDatabase(async (db) => { + // const workspaces = await listWorkspacesRaw(db) + // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) + // for (const workspace of workspaces) { + // const wsid = getWorkspaceId(workspace.workspace) + // await recreateElastic(mongodbUri ?? dbUrl, wsid) + // } + // }) + // }) + + // program + // .command('remove-duplicates-ids-mongo ') + // .description('remove duplicates ids for futue migration') + // .action(async (workspaces: string) => { + // const mongodbUri = getMongoDBUrl() + // await withStorage(async (adapter) => { + // await removeDuplicateIds(toolCtx, mongodbUri, adapter, accountsUrl, workspaces) + // }) + // }) + + // program.command('move-to-pg ').action(async (region: string) => { + // const { dbUrl } = prepareTools() + // const mongodbUri = getMongoDBUrl() + + // await withAccountDatabase(async (db) => { + // const workspaces = await listWorkspacesRaw(db) + // workspaces.sort((a, b) => b.lastVisit - a.lastVisit) + // await moveFromMongoToPG( + // db, + // mongodbUri, + // dbUrl, + // workspaces.filter((p) => p.region !== region), + // region + // ) + // }) + // }) + + // program + // .command('move-workspace-to-pg ') + // .option('-i, --include ', 'A list of ; separated domain names to include during backup', '*') + // .option('-f|--force [force]', 'Force update', false) + // .action( + // async ( + // workspace: string, + // region: string, + // cmd: { + // include: string + // force: boolean + // } + // ) => { + // const { dbUrl } = prepareTools() + // const mongodbUri = getMongoDBUrl() + + // await withAccountDatabase(async (db) => { + // const workspaceInfo = await getWorkspaceById(db, workspace) + // if (workspaceInfo === null) { + // throw new Error(`workspace ${workspace} not found`) + // } + // if (workspaceInfo.region === region && !cmd.force) { + // throw new Error(`workspace ${workspace} is already migrated`) + // } + // await moveWorkspaceFromMongoToPG( + // db, + // mongodbUri, + // dbUrl, + // workspaceInfo, + // region, + // cmd.include === '*' ? undefined : new Set(cmd.include.split(';').map((it) => it.trim())), + // cmd.force + // ) + // }) + // } + // ) + + // program.command('move-account-db-to-pg').action(async () => { + // const { dbUrl } = prepareTools() + // const mongodbUri = getMongoDBUrl() + + // if (mongodbUri === dbUrl) { + // throw new Error('MONGO_URL and DB_URL are the same') + // } + + // await withAccountDatabase(async (pgDb) => { + // await withAccountDatabase(async (mongoDb) => { + // await moveAccountDbFromMongoToPG(toolCtx, mongoDb, pgDb) + // }, mongodbUri) + // }, dbUrl) + // }) + + // program + // .command('perfomance') + // .option('-p, --parallel', '', false) + // .action(async (cmd: { parallel: boolean }) => { + // const { txes, version, migrateOperations } = prepareTools() + // await withAccountDatabase(async (db) => { + // const email = generateId() + // const ws = generateId() + // const wsid = getWorkspaceId(ws) + // const start = new Date() + // const measureCtx = new MeasureMetricsContext('create-workspace', {}) + // const wsInfo = await createWorkspaceRecord(measureCtx, db, null, email, ws, ws) + + // // update the record so it's not taken by one of the workers for the next 60 seconds + // await updateWorkspace(db, wsInfo, { + // mode: 'creating', + // progress: 0, + // lastProcessingTime: Date.now() + 1000 * 60 + // }) + + // await createWorkspace(measureCtx, version, null, wsInfo, txes, migrateOperations, undefined, true) + + // await updateWorkspace(db, wsInfo, { + // mode: 'active', + // progress: 100, + // disabled: false, + // version + // }) + // await createAcc(toolCtx, db, null, email, '1234', '', '', true) + // await assignAccountToWs(toolCtx, db, null, email, ws, AccountRole.User) + // console.log('Workspace created in', new Date().getTime() - start.getTime(), 'ms') + // const token = generateToken(systemAccountEmail, wsid) + // const endpoint = await getTransactorEndpoint(token, 'external') + // await generateWorkspaceData(endpoint, ws, cmd.parallel, email) + // await testFindAll(endpoint, ws, email) + // await dropWorkspace(toolCtx, db, null, ws) + // }) + // }) + + // program + // .command('reset-ws-attempts ') + // .description('Reset workspace creation/upgrade attempts counter') + // .action(async (workspace) => { + // await withAccountDatabase(async (db) => { + // const info = await getWorkspaceById(db, workspace) + // if (info === null) { + // throw new Error(`workspace ${workspace} not found`) + // } + + // await updateWorkspace(db, info, { + // attempts: 0 + // }) + + // console.log('Attempts counter for workspace', workspace, 'has been reset') + // }) + // }) + + // program + // .command('generate-uuid-workspaces') + // .description('generate uuids for all workspaces which are missing it') + // .option('-d, --dryrun', 'Dry run', false) + // .action(async (cmd: { dryrun: boolean }) => { + // await withAccountDatabase(async (db) => { + // console.log('generate uuids for all workspaces which are missing it') + // await generateUuidMissingWorkspaces(toolCtx, db, cmd.dryrun) + // }) + // }) + + // program + // .command('update-data-wsid-to-uuid') + // .description('updates workspaceId in pg/cr to uuid') + // .option('-d, --dryrun', 'Dry run', false) + // .action(async (cmd: { dryrun: boolean }) => { + // await withAccountDatabase(async (db) => { + // console.log('updates workspaceId in pg/cr to uuid') + // const { dbUrl } = prepareTools() + // await updateDataWorkspaceIdToUuid(toolCtx, db, dbUrl, cmd.dryrun) + // }) + // }) + + // program + // .command('add-controlled-doc-rank-mongo') + // .description('add rank to controlled documents') + // .option('-w, --workspace ', 'Selected workspace only', '') + // .action(async (cmd: { workspace: string }) => { + // const { version } = prepareTools() + + // let workspaces: Workspace[] = [] + // await withAccountDatabase(async (db) => { + // workspaces = await listWorkspacesPure(db) + // workspaces = workspaces + // .filter((p) => isActiveMode(p.mode)) + // .filter((p) => cmd.workspace === '' || p.workspace === cmd.workspace) + // .sort((a, b) => b.lastVisit - a.lastVisit) + // }) + + // console.log('found workspaces', workspaces.length) + + // const mongodbUri = getMongoDBUrl() + // const client = getMongoClient(mongodbUri) + // const _client = await client.getClient() + + // try { + // const count = workspaces.length + // let index = 0 + // for (const workspace of workspaces) { + // index++ + + // toolCtx.info('processing workspace', { + // workspace: workspace.workspace, + // version: workspace.version, + // index, + // count + // }) + + // if (workspace.version === undefined || !deepEqual(workspace.version, version)) { + // console.log(`upgrade to ${versionToString(version)} is required`) + // continue + // } + // const workspaceId = getWorkspaceId(workspace.workspace) + // const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) + + // await addControlledDocumentRank(toolCtx, wsDb, workspaceId) + // } + // } finally { + // client.close() + // } + // }) + + // Not needed anymore? + // program + // .command('fill-github-users') + // .option('-t, --token ', 'Github token to increase the limit of requests to GitHub') + // .description('adds github username info to all accounts') + // .action(async (cmd: { token?: string }) => { + // await withAccountDatabase(async (db) => { + // await fillGithubUsers(toolCtx, db, cmd.token) + // }) + // }) extendProgram?.(program) diff --git a/dev/tool/src/markup.ts b/dev/tool/src/markup.ts index 776be29e79..25e918c445 100644 --- a/dev/tool/src/markup.ts +++ b/dev/tool/src/markup.ts @@ -28,7 +28,7 @@ import core, { type Ref, type TxCreateDoc, type TxUpdateDoc, - type WorkspaceId, + type WorkspaceDataId, DOMAIN_TX, SortingOrder, makeCollabId, @@ -49,7 +49,7 @@ export interface RestoreWikiContentParams { export async function restoreWikiContentMongo ( ctx: MeasureContext, db: Db, - workspaceId: WorkspaceId, + workspaceId: WorkspaceDataId, storageAdapter: StorageAdapter, params: RestoreWikiContentParams ): Promise { @@ -119,7 +119,7 @@ export async function restoreWikiContentMongo ( export async function findWikiDocYdocName ( ctx: MeasureContext, db: Db, - workspaceId: WorkspaceId, + workspaceId: WorkspaceDataId, doc: Ref ): Promise | undefined> { const updateContentTx = await db.collection>(DOMAIN_TX).findOne( @@ -198,7 +198,7 @@ export interface RestoreControlledDocContentParams { export async function restoreControlledDocContentMongo ( ctx: MeasureContext, db: Db, - workspaceId: WorkspaceId, + workspaceId: WorkspaceDataId, storageAdapter: StorageAdapter, params: RestoreWikiContentParams ): Promise { @@ -247,7 +247,7 @@ export async function restoreControlledDocContentMongo ( export async function restoreControlledDocContentForDoc ( ctx: MeasureContext, db: Db, - workspaceId: WorkspaceId, + workspaceId: WorkspaceDataId, storageAdapter: StorageAdapter, params: RestoreWikiContentParams, doc: Doc, @@ -302,7 +302,7 @@ export async function restoreControlledDocContentForDoc ( export async function restoreMarkupRefsMongo ( ctx: MeasureContext, db: Db, - workspaceId: WorkspaceId, + workspaceId: WorkspaceDataId, hierarchy: Hierarchy, storageAdapter: StorageAdapter ): Promise { diff --git a/dev/tool/src/mixin.ts b/dev/tool/src/mixin.ts index 1fb66f142c..3e346d74d7 100644 --- a/dev/tool/src/mixin.ts +++ b/dev/tool/src/mixin.ts @@ -25,7 +25,7 @@ import core, { type Obj, type Ref, SortingOrder, - type WorkspaceId + type WorkspaceUuid } from '@hcengineering/core' import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo' import { connect } from '@hcengineering/server-tool' @@ -44,7 +44,7 @@ interface ObjectPropertyInfo { } export async function showMixinForeignAttributes ( - workspaceId: WorkspaceId, + workspaceId: WorkspaceUuid, transactorUrl: string, cmd: { detail: boolean, mixin: string, property: string } ): Promise { @@ -81,7 +81,8 @@ export async function showMixinForeignAttributes ( export async function fixMixinForeignAttributes ( mongoUrl: string, - workspaceId: WorkspaceId, + workspaceId: WorkspaceUuid, + dataId: string | undefined, transactorUrl: string, cmd: { mixin: string, property: string } ): Promise { @@ -102,7 +103,7 @@ export async function fixMixinForeignAttributes ( const client = getMongoClient(mongoUrl) try { const _client = await client.getClient() - const db = getWorkspaceMongoDB(_client, workspaceId) + const db = getWorkspaceMongoDB(_client, dataId ?? workspaceId) for (const [mixin, objects] of result) { console.log('fixing', mixin) diff --git a/dev/tool/src/qms.ts b/dev/tool/src/qms.ts index 43f26b2a3e..c7b87cd67a 100644 --- a/dev/tool/src/qms.ts +++ b/dev/tool/src/qms.ts @@ -13,16 +13,20 @@ // limitations under the License. // -import { type MeasureContext, type Ref, type WorkspaceId } from '@hcengineering/core' +import { type WorkspaceUuid, type MeasureContext, type Ref } from '@hcengineering/core' import documents, { type DocumentMeta, type ProjectMeta } from '@hcengineering/controlled-documents' import { DOMAIN_DOCUMENTS } from '@hcengineering/model-controlled-documents' import { type Db } from 'mongodb' import { makeRank } from '@hcengineering/task' -export async function addControlledDocumentRank (ctx: MeasureContext, db: Db, workspaceId: WorkspaceId): Promise { +export async function addControlledDocumentRank ( + ctx: MeasureContext, + db: Db, + workspaceId: WorkspaceUuid +): Promise { const collections = await db.listCollections().toArray() if (collections.find((it) => it.name === DOMAIN_DOCUMENTS) === undefined) { - ctx.error('skipping migration, no collection found', { workspace: workspaceId.name }) + ctx.error('skipping migration, no collection found', { workspace: workspaceId }) return } diff --git a/dev/tool/src/storage.ts b/dev/tool/src/storage.ts index 9d550abdbc..04381ce1d8 100644 --- a/dev/tool/src/storage.ts +++ b/dev/tool/src/storage.ts @@ -18,9 +18,9 @@ import { type Blob, type MeasureContext, type Ref, - type WorkspaceId, concatLink, - RateLimiter + RateLimiter, + type WorkspaceDataId } from '@hcengineering/core' import { type DatalakeClient } from '@hcengineering/datalake' import { type UploadObjectParams } from '@hcengineering/datalake/types/client' @@ -42,7 +42,7 @@ export interface MoveFilesParams { export async function moveFiles ( ctx: MeasureContext, - workspaceId: WorkspaceId, + workspaceId: WorkspaceDataId, exAdapter: StorageAdapterEx, params: MoveFilesParams ): Promise { @@ -67,7 +67,7 @@ export async function moveFiles ( export async function showLostFiles ( ctx: MeasureContext, - workspaceId: WorkspaceId, + workspaceId: WorkspaceDataId, db: Db, storageAdapter: StorageAdapter, { showAll }: { showAll: boolean } @@ -95,7 +95,7 @@ async function processAdapter ( exAdapter: StorageAdapterEx, source: StorageAdapter, target: StorageAdapter, - workspaceId: WorkspaceId, + workspaceId: WorkspaceDataId, params: MoveFilesParams ): Promise { if (source === target) { @@ -222,7 +222,7 @@ async function processFile ( ctx: MeasureContext, source: Pick, target: Pick, - workspaceId: WorkspaceId, + workspaceId: WorkspaceDataId, blob: Blob ): Promise { const readable = await source.get(ctx, workspaceId, blob._id) @@ -267,7 +267,7 @@ export interface CopyDatalakeParams { export async function copyToDatalake ( ctx: MeasureContext, - workspaceId: WorkspaceId, + workspaceId: WorkspaceDataId, config: S3Config, adapter: S3Service, datalake: DatalakeClient, @@ -374,7 +374,7 @@ export async function copyToDatalake ( export async function copyBlobToDatalake ( ctx: MeasureContext, - workspaceId: WorkspaceId, + workspaceId: WorkspaceDataId, blob: ListBlobResult, config: S3Config, adapter: S3Service, @@ -417,7 +417,7 @@ export async function copyBlobToDatalake ( function uploadMultipart ( ctx: MeasureContext, datalake: DatalakeClient, - workspaceId: WorkspaceId, + workspaceId: WorkspaceDataId, objectName: string, stream: Readable, metadata: UploadObjectParams diff --git a/dev/tool/src/telegram.ts b/dev/tool/src/telegram.ts index b92b68872b..8b537ad0db 100644 --- a/dev/tool/src/telegram.ts +++ b/dev/tool/src/telegram.ts @@ -14,7 +14,7 @@ // limitations under the License. // -import { DOMAIN_TX, type MeasureContext, type Ref, type WorkspaceId } from '@hcengineering/core' +import { DOMAIN_TX, type WorkspaceDataId, type MeasureContext, type Ref } from '@hcengineering/core' import { DOMAIN_ATTACHMENT } from '@hcengineering/model-attachment' import contact, { DOMAIN_CHANNEL } from '@hcengineering/model-contact' import { DOMAIN_TELEGRAM } from '@hcengineering/model-telegram' @@ -31,7 +31,7 @@ const LastMessages = 'last-msgs' export async function clearTelegramHistory ( ctx: MeasureContext, mongoUrl: string, - workspaceId: WorkspaceId, + workspaceId: WorkspaceDataId, tgDb: string, storageAdapter: StorageAdapter ): Promise { diff --git a/dev/tool/src/utils.ts b/dev/tool/src/utils.ts index 5aae788722..ba297c9ed7 100644 --- a/dev/tool/src/utils.ts +++ b/dev/tool/src/utils.ts @@ -1,4 +1,7 @@ +import { getWorkspaceById, getWorkspaceByUrl, type AccountDB, type Workspace } from '@hcengineering/account' import { + systemAccountUuid, + type WorkspaceUuid, type AttachedData, type AttachedDoc, type Class, @@ -8,6 +11,8 @@ import { type Space, type TxOperations } from '@hcengineering/core' +import { getTransactorEndpoint } from '@hcengineering/server-client' +import { generateToken } from '@hcengineering/server-token' export async function findOrUpdateAttached ( client: TxOperations, @@ -42,3 +47,27 @@ export async function findOrUpdateAttached ( } return existingObj } + +export async function getWorkspace (db: AccountDB, workspace: string): Promise { + const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}$/i + let wsObj: Workspace | null + + if (uuidRegex.test(workspace)) { + wsObj = await getWorkspaceById(db, workspace as WorkspaceUuid) + } else { + wsObj = await getWorkspaceByUrl(db, workspace) + } + + return wsObj +} + +export function getToolToken (workspace?: WorkspaceUuid): string { + return generateToken(systemAccountUuid, workspace, { service: 'tool' }) +} + +export async function getWorkspaceTransactorEndpoint ( + workspace: WorkspaceUuid, + type: 'external' | 'internal' = 'external' +): Promise { + return await getTransactorEndpoint(getToolToken(workspace), type) +} diff --git a/dev/tool/src/workspace.ts b/dev/tool/src/workspace.ts index d704a8d309..9236e5a688 100644 --- a/dev/tool/src/workspace.ts +++ b/dev/tool/src/workspace.ts @@ -14,7 +14,6 @@ // limitations under the License. // -import contact from '@hcengineering/contact' import core, { type BackupClient, type Class, @@ -23,17 +22,17 @@ import core, { DOMAIN_TX, type Ref, type Tx, - type WorkspaceId + type WorkspaceUuid } from '@hcengineering/core' import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo' import { connect } from '@hcengineering/server-tool' import { generateModelDiff, printDiff } from './mdiff' -export async function diffWorkspace (mongoUrl: string, workspace: WorkspaceId, rawTxes: Tx[]): Promise { +export async function diffWorkspace (mongoUrl: string, dbName: string, rawTxes: Tx[]): Promise { const client = getMongoClient(mongoUrl) try { const _client = await client.getClient() - const db = getWorkspaceMongoDB(_client, workspace) + const db = getWorkspaceMongoDB(_client, dbName) console.log('diffing transactions...') @@ -42,7 +41,7 @@ export async function diffWorkspace (mongoUrl: string, workspace: WorkspaceId, r .find({ objectSpace: core.space.Model, modifiedBy: core.account.System, - objectClass: { $ne: contact.class.PersonAccount } + objectClass: { $ne: 'contact:class:PersonAccount' } // Note: we may keep these transactions in old workspaces for history purposes }) .toArray() @@ -50,7 +49,7 @@ export async function diffWorkspace (mongoUrl: string, workspace: WorkspaceId, r return ( tx.objectSpace === core.space.Model && tx.modifiedBy === core.account.System && - (tx as any).objectClass !== contact.class.PersonAccount + (tx as any).objectClass !== 'contact:class:PersonAccount' ) }) @@ -72,7 +71,7 @@ export async function diffWorkspace (mongoUrl: string, workspace: WorkspaceId, r } export async function updateField ( - workspaceId: WorkspaceId, + workspaceId: WorkspaceUuid, transactorUrl: string, cmd: { objectId: string, objectClass: string, type: string, attribute: string, value: string, domain: string } ): Promise { diff --git a/models/activity/src/index.ts b/models/activity/src/index.ts index 4d967097f8..0887b2c527 100644 --- a/models/activity/src/index.ts +++ b/models/activity/src/index.ts @@ -38,7 +38,7 @@ import contact, { type Person } from '@hcengineering/contact' import core, { DOMAIN_MODEL, IndexKind, - type Account, + type PersonId, type Class, type Doc, type DocumentQuery, @@ -62,6 +62,7 @@ import { TypeRef, TypeString, TypeTimestamp, + TypePersonId, UX, type Builder } from '@hcengineering/model' @@ -226,8 +227,8 @@ export class TReaction extends TAttachedDoc implements Reaction { @Prop(TypeString(), activity.string.Emoji) emoji!: string - @Prop(TypeRef(core.class.Account), view.string.Created) - createBy!: Ref + @Prop(TypePersonId(), view.string.Created) + createBy!: PersonId } @Model(activity.class.SavedMessage, preference.class.Preference) diff --git a/models/activity/src/migration.ts b/models/activity/src/migration.ts index 34c1ff8b34..a85af80aef 100644 --- a/models/activity/src/migration.ts +++ b/models/activity/src/migration.ts @@ -15,7 +15,15 @@ import { type ActivityMessage, type DocUpdateMessage, type Reaction } from '@hcengineering/activity' import contact from '@hcengineering/contact' -import core, { type Class, type Doc, type Domain, groupByArray, type Ref, type Space } from '@hcengineering/core' +import core, { + type Class, + type Doc, + type Domain, + groupByArray, + MeasureMetricsContext, + type Ref, + type Space +} from '@hcengineering/core' import { type MigrateOperation, type MigrateUpdate, @@ -26,6 +34,8 @@ import { tryMigrate } from '@hcengineering/model' import { htmlToMarkup } from '@hcengineering/text' +import { getSocialIdByOldAccount } from '@hcengineering/model-core' + import { activityId, DOMAIN_ACTIVITY, DOMAIN_REACTION, DOMAIN_USER_MENTION } from './index' import activity from './plugin' @@ -181,6 +191,50 @@ async function migrateActivityMarkup (client: MigrationClient): Promise { ) } +async function migrateAccountsToSocialIds (client: MigrationClient): Promise { + const ctx = new MeasureMetricsContext('activity migrateAccountsToSocialIds', {}) + const socialIdByAccount = await getSocialIdByOldAccount(client) + + ctx.info('processing activity reactions ', {}) + const iterator = await client.traverse(DOMAIN_ACTIVITY, { _class: activity.class.Reaction }) + + try { + let processed = 0 + while (true) { + const docs = await iterator.next(200) + if (docs === null || docs.length === 0) { + break + } + + const operations: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] + + for (const doc of docs) { + const reaction = doc as Reaction + const newCreateBy = socialIdByAccount[reaction.createBy] ?? reaction.createBy + + if (newCreateBy === reaction.createBy) continue + + operations.push({ + filter: { _id: doc._id }, + update: { + createBy: newCreateBy + } + }) + } + + if (operations.length > 0) { + await client.bulk(DOMAIN_ACTIVITY, operations) + } + + processed += docs.length + ctx.info('...processed', { count: processed }) + } + } finally { + await iterator.close() + } + ctx.info('finished processing activity reactions ', {}) +} + export const activityOperation: MigrateOperation = { async migrate (client: MigrationClient): Promise { await tryMigrate(client, activityId, [ @@ -223,6 +277,10 @@ export const activityOperation: MigrateOperation = { await client.move(DOMAIN_ACTIVITY, { _class: activity.class.Reaction }, DOMAIN_REACTION) await client.move(DOMAIN_ACTIVITY, { _class: activity.class.UserMentionInfo }, DOMAIN_USER_MENTION) } + }, + { + state: 'accounts-to-social-ids', + func: migrateAccountsToSocialIds } ]) }, diff --git a/models/ai-bot/src/migration.ts b/models/ai-bot/src/migration.ts index e54aca4b2d..502173e1a1 100644 --- a/models/ai-bot/src/migration.ts +++ b/models/ai-bot/src/migration.ts @@ -13,74 +13,17 @@ // limitations under the License. // -import contact, { type Channel, type Person, type PersonAccount } from '@hcengineering/contact' -import core, { - DOMAIN_MODEL_TX, - type TxCUD, - type TxCreateDoc, - type Ref, - type TxUpdateDoc, - TxProcessor, - type Domain -} from '@hcengineering/core' -import { DOMAIN_CHANNEL, DOMAIN_CONTACT } from '@hcengineering/model-contact' import { tryMigrate, type MigrateOperation, type MigrationClient, type MigrationUpgradeClient } from '@hcengineering/model' -import aiBot, { aiBotId } from '@hcengineering/ai-bot' - -const DOMAIN_ACTIVITY = 'activity' as Domain - -async function migrateAiExtraAccounts (client: MigrationClient): Promise { - const currentAccount = ( - await client.model.findAll(contact.class.PersonAccount, { _id: aiBot.account.AIBot as Ref }) - )[0] - if (currentAccount === undefined) return - - const txes = await client.find>(DOMAIN_MODEL_TX, { - _class: { $in: [core.class.TxCreateDoc, core.class.TxUpdateDoc] }, - objectClass: contact.class.PersonAccount, - objectId: aiBot.account.AIBot as Ref - }) - - const personsToDelete: Ref[] = [] - const txesToDelete: Ref>[] = [] - - for (const tx of txes) { - if (tx._class === core.class.TxCreateDoc) { - const acc = TxProcessor.createDoc2Doc(tx as TxCreateDoc) - if (acc.person !== currentAccount.person) { - personsToDelete.push(acc.person) - txesToDelete.push(tx._id) - } - } else if (tx._class === core.class.TxUpdateDoc) { - const person = (tx as TxUpdateDoc).operations.person - if (person !== undefined && person !== currentAccount.person) { - personsToDelete.push(person) - txesToDelete.push(tx._id) - } - } - } - - if (personsToDelete.length === 0) return - - await client.deleteMany(DOMAIN_MODEL_TX, { _id: { $in: txesToDelete } }) - await client.deleteMany(DOMAIN_ACTIVITY, { attachedTo: { $in: personsToDelete } }) - await client.deleteMany(DOMAIN_CHANNEL, { attachedTo: { $in: personsToDelete } }) - await client.deleteMany(DOMAIN_CONTACT, { _id: { $in: personsToDelete } }) -} +import { aiBotId } from '@hcengineering/ai-bot' export const aiBotOperation: MigrateOperation = { async migrate (client: MigrationClient): Promise { - await tryMigrate(client, aiBotId, [ - { - state: 'remove-ai-bot-extra-accounts-v100', - func: migrateAiExtraAccounts - } - ]) + await tryMigrate(client, aiBotId, []) }, async upgrade (state: Map>, client: () => Promise): Promise {} } diff --git a/models/analytics-collector/src/index.ts b/models/analytics-collector/src/index.ts index e8e0dc38cd..575601e327 100644 --- a/models/analytics-collector/src/index.ts +++ b/models/analytics-collector/src/index.ts @@ -18,7 +18,7 @@ import { type OnboardingChannel } from '@hcengineering/analytics-collector' import chunter from '@hcengineering/chunter' import { TChannel } from '@hcengineering/model-chunter' import activity, { type ActivityMessageControl } from '@hcengineering/activity' -import core from '@hcengineering/core' +import core, { type PersonId, type WorkspaceUuid } from '@hcengineering/core' import analyticsCollector from './plugin' @@ -40,9 +40,9 @@ export class TOnboardingChannel extends TChannel implements OnboardingChannel { @ReadOnly() userName!: string - @Prop(TypeString(), analyticsCollector.string.Email) + @Prop(TypeString(), analyticsCollector.string.SocialId) @ReadOnly() - email!: string + socialString!: PersonId @Prop(TypeString(), analyticsCollector.string.WorkspaceName) @ReadOnly() @@ -54,7 +54,7 @@ export class TOnboardingChannel extends TChannel implements OnboardingChannel { @Prop(TypeString(), analyticsCollector.string.WorkspaceId) @ReadOnly() - workspaceId!: string + workspaceId!: WorkspaceUuid @Prop(TypeBoolean(), analyticsCollector.string.DisableAIReplies) disableAIReplies!: boolean diff --git a/models/analytics-collector/src/migration.ts b/models/analytics-collector/src/migration.ts index aad168359a..3ce33df046 100644 --- a/models/analytics-collector/src/migration.ts +++ b/models/analytics-collector/src/migration.ts @@ -15,14 +15,17 @@ import { type MigrateOperation, + type MigrateUpdate, type MigrationClient, + type MigrationDocumentQuery, type MigrationUpgradeClient, tryMigrate } from '@hcengineering/model' -import { analyticsCollectorId } from '@hcengineering/analytics-collector' -import { DOMAIN_SPACE } from '@hcengineering/model-core' +import analyticsCollector, { analyticsCollectorId } from '@hcengineering/analytics-collector' +import { DOMAIN_SPACE, getSocialKeyByOldEmail } from '@hcengineering/model-core' import { DOMAIN_DOC_NOTIFY, DOMAIN_NOTIFICATION } from '@hcengineering/model-notification' import { DOMAIN_ACTIVITY } from '@hcengineering/model-activity' +import { buildSocialIdString, type Doc, MeasureMetricsContext } from '@hcengineering/core' async function removeOnboardingChannels (client: MigrationClient): Promise { const channels = await client.find(DOMAIN_SPACE, { 'analytics:mixin:AnalyticsChannel': { $exists: true } }) @@ -41,12 +44,58 @@ async function removeOnboardingChannels (client: MigrationClient): Promise await client.deleteMany(DOMAIN_SPACE, { _id: { $in: channelsIds } }) } +async function migrateAccountsToSocialIds (client: MigrationClient): Promise { + const ctx = new MeasureMetricsContext('analytics collector migrateAccountsToSocialIds', {}) + + ctx.info('processing analytics collector onboarding channels ', {}) + const iterator = await client.traverse(DOMAIN_SPACE, { _class: analyticsCollector.class.OnboardingChannel }) + + try { + let processed = 0 + while (true) { + const docs = await iterator.next(200) + if (docs === null || docs.length === 0) { + break + } + + const operations: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] + + for (const doc of docs) { + const email = (doc as any).email + if (email === undefined || email === '') continue + const socialString = buildSocialIdString(getSocialKeyByOldEmail(email)) + + operations.push({ + filter: { _id: doc._id }, + update: { + socialString + } + }) + } + + if (operations.length > 0) { + await client.bulk(DOMAIN_SPACE, operations) + } + + processed += docs.length + ctx.info('...processed', { count: processed }) + } + } finally { + await iterator.close() + } + ctx.info('finished processing analytics collector onboarding channels ', {}) +} + export const analyticsCollectorOperation: MigrateOperation = { async migrate (client: MigrationClient): Promise { await tryMigrate(client, analyticsCollectorId, [ { state: 'remove-analytics-channels-v3', func: removeOnboardingChannels + }, + { + state: 'accounts-to-social-ids', + func: migrateAccountsToSocialIds } ]) }, diff --git a/models/calendar/src/migration.ts b/models/calendar/src/migration.ts index 47b3d13706..2798d530c7 100644 --- a/models/calendar/src/migration.ts +++ b/models/calendar/src/migration.ts @@ -13,20 +13,109 @@ // limitations under the License. // -import { calendarId, type Event, type ReccuringEvent } from '@hcengineering/calendar' -import { type Ref, type Space } from '@hcengineering/core' +import { type Calendar, calendarId, type Event, type ReccuringEvent } from '@hcengineering/calendar' +import { type Doc, MeasureMetricsContext, type PersonId, type Ref, type Space } from '@hcengineering/core' import { createDefaultSpace, + type MigrateUpdate, + type MigrationDocumentQuery, tryMigrate, tryUpgrade, type MigrateOperation, type MigrationClient, type MigrationUpgradeClient } from '@hcengineering/model' -import { DOMAIN_SPACE } from '@hcengineering/model-core' +import { DOMAIN_SPACE, getSocialIdByOldAccount } from '@hcengineering/model-core' import { DOMAIN_CALENDAR, DOMAIN_EVENT } from '.' import calendar from './plugin' +function getCalendarId (socialString: PersonId): Ref { + return `${socialString}_calendar` as Ref +} + +async function migrateAccountsToSocialIds (client: MigrationClient): Promise { + const ctx = new MeasureMetricsContext('calendar migrateAccountsToSocialIds', {}) + const hierarchy = client.hierarchy + const socialIdByAccount = await getSocialIdByOldAccount(client) + const eventClasses = hierarchy.getDescendants(calendar.class.Event) + + const calendars = await client.find(DOMAIN_CALENDAR, { + _class: calendar.class.Calendar + }) + + ctx.info('processing internal calendars') + + for (const calendar of calendars) { + const id = calendar._id + if (!id.endsWith('_calendar')) { + ctx.warn('Wrong calendar id format', { calendar: calendar._id }) + continue + } + + const account = id.substring(0, id.length - 9) + const socialId = socialIdByAccount[account] + if (socialId === undefined) { + ctx.warn('no socialId for account', { account }) + continue + } + + await client.delete(DOMAIN_CALENDAR, calendar._id) + await client.create(DOMAIN_CALENDAR, { + ...calendar, + _id: getCalendarId(socialId) + }) + } + + let processedEvents = 0 + const eventsIterator = await client.traverse(DOMAIN_EVENT, { + _class: { $in: eventClasses } + }) + + try { + while (true) { + const events = await eventsIterator.next(200) + if (events === null || events.length === 0) { + break + } + + const operations: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] + + for (const event of events) { + const id = event.calendar + if (!id.endsWith('_calendar')) { + // Nothing to do, in external calendar + continue + } + + const account = id.substring(0, id.length - 9) + const socialId = socialIdByAccount[account] + if (socialId === undefined) { + ctx.warn('no socialId for account', { account }) + continue + } + + operations.push({ + filter: { _id: event._id }, + update: { + calendar: getCalendarId(socialId) + } + }) + } + + if (operations.length > 0) { + await client.bulk(DOMAIN_EVENT, operations) + } + + processedEvents += events.length + ctx.info('...processed events', { count: processedEvents }) + } + + ctx.info('finished processing events') + } finally { + await eventsIterator.close() + } +} + async function migrateCalendars (client: MigrationClient): Promise { await client.move( DOMAIN_SPACE, @@ -146,6 +235,10 @@ export const calendarOperation: MigrateOperation = { DOMAIN_EVENT ) } + }, + { + state: 'accounts-to-social-ids', + func: migrateAccountsToSocialIds } ]) }, diff --git a/models/chunter/src/migration.ts b/models/chunter/src/migration.ts index 70d2c2151c..979b597828 100644 --- a/models/chunter/src/migration.ts +++ b/models/chunter/src/migration.ts @@ -13,9 +13,9 @@ // limitations under the License. // -import { chunterId, type DirectMessage, type ThreadMessage } from '@hcengineering/chunter' +import { chunterId, type ThreadMessage } from '@hcengineering/chunter' import core, { - type Account, + type PersonId, TxOperations, type Class, type Doc, @@ -33,10 +33,14 @@ import { } from '@hcengineering/model' import activity, { migrateMessagesSpace, DOMAIN_ACTIVITY } from '@hcengineering/model-activity' import notification from '@hcengineering/notification' -import contactPlugin, { type Person, type PersonAccount } from '@hcengineering/contact' +import { + getAllEmployeesPrimarySocialStrings, + pickPrimarySocialId, + getSocialStringsByEmployee, + includesAny +} from '@hcengineering/contact' import { DOMAIN_DOC_NOTIFY, DOMAIN_NOTIFICATION } from '@hcengineering/model-notification' import { type DocUpdateMessage } from '@hcengineering/activity' -import { DOMAIN_SPACE } from '@hcengineering/model-core' import chunter from './plugin' import { DOMAIN_CHUNTER } from './index' @@ -50,20 +54,20 @@ export async function createDocNotifyContexts ( objectClass: Ref>, objectSpace: Ref ): Promise { - const users = await client.findAll(core.class.Account, {}) + const socialStringsByEmployee = getSocialStringsByEmployee(tx) + const allSocialStrings = Object.values(socialStringsByEmployee).flat() + const docNotifyContexts = await client.findAll(notification.class.DocNotifyContext, { - user: { $in: users.map((it) => it._id) }, + user: { $in: allSocialStrings }, objectId }) - for (const user of users) { - if (user._id === core.account.System) { - continue - } - const docNotifyContext = docNotifyContexts.find((it) => it.user === user._id) + + for (const userSocialStrings of Object.values(socialStringsByEmployee)) { + const docNotifyContext = docNotifyContexts.find((it) => userSocialStrings.includes(it.user)) if (docNotifyContext === undefined) { await tx.createDoc(notification.class.DocNotifyContext, core.space.Space, { - user: user._id, + user: pickPrimarySocialId(userSocialStrings), objectId, objectClass, objectSpace, @@ -98,7 +102,7 @@ export async function createGeneral (client: MigrationUpgradeClient, tx: TxOpera topic: 'General Channel', private: false, archived: false, - members: await getAllPersonAccounts(tx), + members: await getAllEmployeesPrimarySocialStrings(tx), autoJoin: true }, chunter.space.General @@ -109,22 +113,16 @@ export async function createGeneral (client: MigrationUpgradeClient, tx: TxOpera await createDocNotifyContexts(client, tx, chunter.space.General, chunter.class.Channel, core.space.Space) } -async function getAllPersonAccounts (tx: TxOperations): Promise[]> { - const employees = await tx.findAll(contactPlugin.mixin.Employee, { active: true }) - const accounts = await tx.findAll(contactPlugin.class.PersonAccount, { - person: { $in: employees.map((it) => it._id) } - }) - return accounts.map((it) => it._id) -} - async function joinEmployees (current: Space, tx: TxOperations): Promise { - const accs = await getAllPersonAccounts(tx) - const newMembers: Ref[] = [...current.members] - for (const acc of accs) { - if (!newMembers.includes(acc)) { - newMembers.push(acc) + const byEmployee = await getSocialStringsByEmployee(tx) + const newMembers: PersonId[] = [...current.members] + + for (const socialStrings of Object.values(byEmployee)) { + if (!includesAny(newMembers, socialStrings)) { + newMembers.push(pickPrimarySocialId(socialStrings)) } } + await tx.update(current, { members: newMembers }) @@ -154,7 +152,7 @@ export async function createRandom (client: MigrationUpgradeClient, tx: TxOperat topic: 'Random Talks', private: false, archived: false, - members: await getAllPersonAccounts(tx), + members: await getAllEmployeesPrimarySocialStrings(tx), autoJoin: true }, chunter.space.Random @@ -241,53 +239,6 @@ async function removeWrongActivity (client: MigrationClient): Promise { }) } -async function removeDuplicatedDirects (client: MigrationClient): Promise { - const directs = await client.find(DOMAIN_SPACE, { _class: chunter.class.DirectMessage }) - const personAccounts = await client.model.findAll(contactPlugin.class.PersonAccount, {}) - const personByAccount = new Map(personAccounts.map((it) => [it._id, it.person])) - - const accountsToPersons = (members: Ref[]): Ref[] => { - const personsSet = new Set( - members - .map((it) => personByAccount.get(it as Ref)) - .filter((it): it is Ref => it !== undefined) - ) - return Array.from(personsSet) - } - - const map: Map = new Map() - const toRemove: Ref[] = [] - - for (const direct of directs) { - const persons = accountsToPersons(direct.members) - - if (persons.length === 0) { - toRemove.push(direct._id) - continue - } - - const key = persons.sort().join(',') - - if (!map.has(key)) { - map.set(key, [direct]) - } else { - map.get(key)?.push(direct) - } - } - - for (const [, directs] of map) { - if (directs.length === 1) continue - const toSave = directs.reduce((acc, it) => ((it.messages ?? 0) > (acc.messages ?? 0) ? it : acc), directs[0]) - const rest = directs.filter((it) => it._id !== toSave._id) - toRemove.push(...rest.map((it) => it._id)) - } - - await client.deleteMany(DOMAIN_SPACE, { _id: { $in: toRemove } }) - await client.deleteMany(DOMAIN_ACTIVITY, { attachedTo: { $in: toRemove } }) - await client.deleteMany(DOMAIN_ACTIVITY, { objectId: { $in: toRemove } }) - await client.deleteMany(DOMAIN_DOC_NOTIFY, { objectId: { $in: toRemove } }) -} - export const chunterOperation: MigrateOperation = { async migrate (client: MigrationClient): Promise { await tryMigrate(client, chunterId, [ @@ -346,12 +297,6 @@ export const chunterOperation: MigrateOperation = { await client.deleteMany(DOMAIN_TX, { mixin: 'chunter:mixin:ChannelInfo' }) } }, - { - state: 'remove-duplicated-directs-v1', - func: async (client) => { - await removeDuplicatedDirects(client) - } - }, { state: 'remove-direct-members-messages', func: async (client) => { diff --git a/models/contact/src/index.ts b/models/contact/src/index.ts index 60e26b083d..93ca9c8582 100644 --- a/models/contact/src/index.ts +++ b/models/contact/src/index.ts @@ -19,6 +19,7 @@ import { AvatarType, contactId, type AvatarProvider, + type SocialIdentity, type Channel, type ChannelProvider, type Contact, @@ -28,7 +29,6 @@ import { type Member, type Organization, type Person, - type PersonAccount, type Status, type PersonSpace } from '@hcengineering/contact' @@ -37,15 +37,19 @@ import { DOMAIN_MODEL, DateRangeMode, IndexKind, + type Collection, type Blob, type Class, type MarkupBlobRef, type Domain, type Ref, - type Timestamp + type Timestamp, + type SocialIdType, + type PersonUuid, + type PersonId } from '@hcengineering/core' import { - Collection, + Collection as CollectionType, Hidden, Index, Mixin, @@ -56,6 +60,7 @@ import { TypeBoolean, TypeCollaborativeDoc, TypeDate, + TypePersonId, TypeRecord, TypeRef, TypeString, @@ -65,14 +70,14 @@ import { } from '@hcengineering/model' import attachment from '@hcengineering/model-attachment' import chunter from '@hcengineering/model-chunter' -import core, { TAccount, TAttachedDoc, TDoc, TSpace } from '@hcengineering/model-core' +import core, { TAttachedDoc, TDoc, TSpace } from '@hcengineering/model-core' import { createPublicLinkAction } from '@hcengineering/model-guest' import { generateClassNotificationTypes } from '@hcengineering/model-notification' import presentation from '@hcengineering/model-presentation' import view, { createAction, createAttributePresenter, type Viewlet } from '@hcengineering/model-view' import workbench from '@hcengineering/model-workbench' import notification from '@hcengineering/notification' -import type { Asset, IntlString, Resource } from '@hcengineering/platform' +import { getEmbeddedLabel, type Asset, type IntlString, type Resource } from '@hcengineering/platform' import setting from '@hcengineering/setting' import templates from '@hcengineering/templates' import { type AnyComponent } from '@hcengineering/ui/src/types' @@ -125,18 +130,20 @@ export class TContact extends TDoc implements Contact { url?: string } - @Prop(Collection(contact.class.Channel), contact.string.ContactInfo) + @Prop(CollectionType(contact.class.Channel), contact.string.ContactInfo) channels?: number - @Prop(Collection(attachment.class.Attachment), attachment.string.Attachments, { shortLabel: attachment.string.Files }) + @Prop(CollectionType(attachment.class.Attachment), attachment.string.Attachments, { + shortLabel: attachment.string.Files + }) attachments?: number - @Prop(Collection(chunter.class.ChatMessage), chunter.string.Comments) + @Prop(CollectionType(chunter.class.ChatMessage), chunter.string.Comments) comments?: number @Prop(TypeString(), contact.string.Location) @Index(IndexKind.FullText) - city!: string + city?: string } @Model(contact.class.Channel, core.class.AttachedDoc, DOMAIN_CHANNEL) @@ -156,11 +163,40 @@ export class TChannel extends TAttachedDoc implements Channel { lastMessage?: Timestamp } +@Model(contact.class.SocialIdentity, core.class.AttachedDoc, DOMAIN_CHANNEL) +@UX(contact.string.SocialId) +export class TSocialIdentity extends TAttachedDoc implements SocialIdentity { + declare attachedTo: Ref + declare attachedToClass: Ref> + + @Prop(TypePersonId(), getEmbeddedLabel('Key')) + @Hidden() + key!: PersonId + + @Prop(TypeString(), contact.string.Type) + type!: SocialIdType + + @Prop(TypeString(), contact.string.Value) + @Index(IndexKind.FullText) + value!: string + + @Prop(TypeBoolean(), contact.string.Confirmed) + @ReadOnly() + confirmed!: boolean +} + @Model(contact.class.Person, contact.class.Contact) @UX(contact.string.Person, contact.icon.Person, 'PRSN', 'name', undefined, contact.string.Persons) export class TPerson extends TContact implements Person { + @Prop(TypeString(), getEmbeddedLabel('UUID')) + @Hidden() + personUuid?: PersonUuid + @Prop(TypeDate(DateRangeMode.DATE, false), contact.string.Birthday) birthday?: Timestamp + + @Prop(CollectionType(contact.class.SocialIdentity), contact.string.SocialIds) + socialIds?: Collection } @Model(contact.class.Member, core.class.AttachedDoc, DOMAIN_CONTACT) @@ -177,7 +213,7 @@ export class TOrganization extends TContact implements Organization { @Index(IndexKind.FullText) description!: MarkupBlobRef | null - @Prop(Collection(contact.class.Member), contact.string.Members) + @Prop(CollectionType(contact.class.Member), contact.string.Members) members!: number } @@ -198,7 +234,7 @@ export class TEmployee extends TPerson implements Employee { @Hidden() active!: boolean - @Prop(Collection(contact.class.Status), contact.string.Status) + @Prop(CollectionType(contact.class.Status), contact.string.Status) @Hidden() statuses?: number @@ -207,12 +243,6 @@ export class TEmployee extends TPerson implements Employee { position?: string | null } -@Model(contact.class.PersonAccount, core.class.Account) -export class TPersonAccount extends TAccount implements PersonAccount { - @Prop(TypeRef(contact.class.Person), contact.string.Person) - person!: Ref -} - @Model(contact.class.ContactsTab, core.class.Doc, DOMAIN_MODEL) export class TContactsTab extends TDoc implements ContactsTab { label!: IntlString @@ -233,9 +263,9 @@ export function createModel (builder: Builder): void { TChannelProvider, TContact, TPerson, + TSocialIdentity, TOrganization, TEmployee, - TPersonAccount, TChannel, TStatus, TMember, @@ -508,15 +538,15 @@ export function createModel (builder: Builder): void { pinned: true }) - builder.mixin(core.class.Account, core.class.Class, view.mixin.Aggregation, { - createAggregationManager: contact.aggregation.CreatePersonAggregationManager, - setStoreFunc: contact.function.SetPersonStore, - filterFunc: contact.function.PersonFilterFunction - }) + // builder.mixin(core.class.Account, core.class.Class, view.mixin.Aggregation, { + // createAggregationManager: contact.aggregation.CreatePersonAggregationManager, + // setStoreFunc: contact.function.SetPersonStore, + // filterFunc: contact.function.PersonFilterFunction + // }) - builder.mixin(core.class.Account, core.class.Class, view.mixin.Groupping, { - grouppingManager: contact.aggregation.GrouppingPersonManager - }) + // builder.mixin(core.class.Account, core.class.Class, view.mixin.Groupping, { + // grouppingManager: contact.aggregation.GrouppingPersonManager + // }) builder.mixin(contact.class.Organization, core.class.Class, view.mixin.ObjectEditor, { editor: contact.component.EditOrganization, @@ -583,8 +613,8 @@ export function createModel (builder: Builder): void { presenter: contact.component.EmployeeFilterValuePresenter }) - builder.mixin(core.class.Account, core.class.Class, view.mixin.AttributeFilterPresenter, { - presenter: contact.component.PersonAccountFilterValuePresenter + builder.mixin(contact.class.Person, core.class.Class, view.mixin.AttributeFilterPresenter, { + presenter: contact.component.PersonFilterValuePresenter }) builder.mixin(contact.mixin.Employee, core.class.Class, view.mixin.AttributeFilter, { @@ -751,22 +781,10 @@ export function createModel (builder: Builder): void { presenter: contact.component.PersonPresenter }) - builder.mixin(core.class.Account, core.class.Class, view.mixin.ArrayEditor, { + builder.mixin(core.class.TypePersonId, core.class.Class, view.mixin.ArrayEditor, { inlineEditor: contact.component.AccountArrayEditor }) - builder.mixin(contact.class.PersonAccount, core.class.Class, view.mixin.ArrayEditor, { - inlineEditor: contact.component.AccountArrayEditor - }) - - builder.mixin(core.class.Account, core.class.Class, view.mixin.ObjectPresenter, { - presenter: contact.component.PersonAccountPresenter - }) - builder.mixin(core.class.Account, core.class.Class, view.mixin.AttributePresenter, { - presenter: contact.component.PersonAccountRefPresenter, - arrayPresenter: contact.component.AccountArrayEditor - }) - builder.mixin(contact.class.Organization, core.class.Class, view.mixin.ObjectPresenter, { presenter: contact.component.OrganizationPresenter }) diff --git a/models/contact/src/migration.ts b/models/contact/src/migration.ts index d78df3c00e..2efb5bfb49 100644 --- a/models/contact/src/migration.ts +++ b/models/contact/src/migration.ts @@ -1,15 +1,19 @@ // -import { AvatarType, type Contact, type Person, type PersonSpace } from '@hcengineering/contact' +import { AvatarType, type SocialIdentity, type Contact } from '@hcengineering/contact' import { + type AccountRole, + buildSocialIdString, type Class, type Doc, type Domain, + DOMAIN_MODEL_TX, DOMAIN_TX, generateId, + MeasureMetricsContext, type Ref, type Space, - TxOperations + type TxCUD } from '@hcengineering/core' import { createDefaultSpace, @@ -23,42 +27,10 @@ import { tryUpgrade } from '@hcengineering/model' import activity, { DOMAIN_ACTIVITY } from '@hcengineering/model-activity' -import core, { DOMAIN_SPACE } from '@hcengineering/model-core' +import core, { getAccountsFromTxes, getSocialKeyByOldEmail } from '@hcengineering/model-core' import { DOMAIN_VIEW } from '@hcengineering/model-view' -import contact, { contactId, DOMAIN_CONTACT } from './index' - -async function createEmployeeEmail (client: TxOperations): Promise { - const employees = await client.findAll(contact.mixin.Employee, {}) - const channels = ( - await client.findAll(contact.class.Channel, { - attachedTo: { $in: employees.map((p) => p._id) } - }) - ).filter((it) => it.provider === contact.channelProvider.Email) - const channelsMap = new Map(channels.map((p) => [p.attachedTo, p])) - for (const employee of employees) { - const acc = client.getModel().getAccountByPersonId(employee._id) - if (acc.length === 0) continue - const current = channelsMap.get(employee._id) - if (current === undefined) { - await client.addCollection( - contact.class.Channel, - contact.space.Contacts, - employee._id, - contact.mixin.Employee, - 'channels', - { - provider: contact.channelProvider.Email, - value: acc[0].email.trim() - }, - undefined, - employee.modifiedOn - ) - } else if (current.value !== acc[0].email.trim()) { - await client.update(current, { value: acc[0].email.trim() }, false, current.modifiedOn) - } - } -} +import contact, { contactId, DOMAIN_CHANNEL, DOMAIN_CONTACT } from './index' const colorPrefix = 'color://' const gravatarPrefix = 'gravatar://' @@ -113,47 +85,75 @@ async function migrateAvatars (client: MigrationClient): Promise { ) } -async function createPersonSpaces (client: MigrationClient): Promise { - const spaces = await client.find(DOMAIN_SPACE, { _class: contact.class.PersonSpace }) +async function getOldPersonAccounts ( + client: MigrationClient +): Promise> { + const accountsTxes: TxCUD[] = await client.find>(DOMAIN_MODEL_TX, { + objectClass: 'contact:class:PersonAccount' as Ref> + }) - if (spaces.length > 0) { - return - } + return getAccountsFromTxes(accountsTxes) +} - const accounts = await client.model.findAll(contact.class.PersonAccount, {}) - const employees = await client.find(DOMAIN_CONTACT, { [contact.mixin.Employee]: { $exists: true } }) - - const newSpaces = new Map, PersonSpace>() - const now = Date.now() - - for (const account of accounts) { - const employee = employees.find(({ _id }) => _id === account.person) - if (employee === undefined) continue - - const space = newSpaces.get(account.person) - - if (space !== undefined) { - space.members.push(account._id) - } else { - newSpaces.set(account.person, { - _id: generateId(), - _class: contact.class.PersonSpace, - space: core.space.Space, - name: 'Personal space', - description: '', - private: true, - archived: false, - members: [account._id], - person: account.person, - modifiedBy: core.account.System, - createdBy: core.account.System, - modifiedOn: now, - createdOn: now - }) +async function assignWorkspaceRoles (client: MigrationClient): Promise { + const ctx = new MeasureMetricsContext('contact assignWorkspaceRoles', {}) + ctx.info('assigning workspace roles...') + const oldPersonAccounts = await getOldPersonAccounts(client) + for (const { person, email, role } of oldPersonAccounts) { + // check it's an active employee + const personObj = (await client.find(DOMAIN_CONTACT, { _id: person, _class: contact.class.Person }))[0] + if (personObj === undefined) { + continue + } + const employee = client.hierarchy.as(personObj, contact.mixin.Employee) + if (employee === undefined || !employee.active) { + continue + } + const socialKey = getSocialKeyByOldEmail(email) + try { + await client.accountClient.updateWorkspaceRoleBySocialId(buildSocialIdString(socialKey), role) + } catch (err: any) { + ctx.error('Failed to update workspace role', { email, ...socialKey, role, err }) } } - await client.create(DOMAIN_SPACE, Array.from(newSpaces.values())) + ctx.info('finished assigning workspace roles', { users: oldPersonAccounts.length }) +} + +async function createSocialIdentities (client: MigrationClient): Promise { + const ctx = new MeasureMetricsContext('createSocialIdentities', {}) + ctx.info('processing person accounts ', {}) + + const personAccountsTxes: any[] = await client.find>(DOMAIN_MODEL_TX, { + objectClass: 'contact:class:PersonAccount' as Ref> + }) + const personAccounts = getAccountsFromTxes(personAccountsTxes) + + for (const pAcc of personAccounts) { + const email: string = pAcc.email ?? '' + if (email === '') continue + + const socialIdKey = getSocialKeyByOldEmail(email) + const socialId: SocialIdentity = { + _id: generateId(), + _class: contact.class.SocialIdentity, + space: contact.space.Contacts, + ...socialIdKey, + key: buildSocialIdString(socialIdKey), + confirmed: false, + + attachedTo: pAcc.person, + attachedToClass: contact.class.Person, + collection: 'socialIds', + + modifiedOn: Date.now(), + createdBy: core.account.ConfigUser, + createdOn: Date.now(), + modifiedBy: core.account.ConfigUser + } + + await client.create(DOMAIN_CHANNEL, socialId) + } } export const contactOperation: MigrateOperation = { @@ -298,8 +298,12 @@ export const contactOperation: MigrateOperation = { } }, { - state: 'create-person-spaces-v1', - func: createPersonSpaces + state: 'create-social-identities', + func: createSocialIdentities + }, + { + state: 'assign-workspace-roles', + func: assignWorkspaceRoles } ]) }, @@ -310,13 +314,6 @@ export const contactOperation: MigrateOperation = { func: async (client) => { await createDefaultSpace(client, contact.space.Contacts, { name: 'Contacts', description: 'Contacts' }) } - }, - { - state: 'createEmails', - func: async (client) => { - const tx = new TxOperations(client, core.account.System) - await createEmployeeEmail(tx) - } } ]) } diff --git a/models/contact/src/plugin.ts b/models/contact/src/plugin.ts index cbb7e1b4a8..8fe7488339 100644 --- a/models/contact/src/plugin.ts +++ b/models/contact/src/plugin.ts @@ -39,8 +39,6 @@ export default mergeIds(contactId, contact, { OrganizationPresenter: '' as AnyComponent, Contacts: '' as AnyComponent, ContactsTabs: '' as AnyComponent, - PersonAccountPresenter: '' as AnyComponent, - PersonAccountRefPresenter: '' as AnyComponent, OrganizationEditor: '' as AnyComponent, EmployeePresenter: '' as AnyComponent, EmployeeRefPresenter: '' as AnyComponent, @@ -59,7 +57,6 @@ export default mergeIds(contactId, contact, { ActivityChannelPresenter: '' as AnyComponent, EmployeeFilter: '' as AnyComponent, EmployeeFilterValuePresenter: '' as AnyComponent, - PersonAccountFilterValuePresenter: '' as AnyComponent, ChannelIcon: '' as AnyComponent }, string: { diff --git a/models/controlled-documents/src/migration.ts b/models/controlled-documents/src/migration.ts index a69e9e2f36..6900e6c998 100644 --- a/models/controlled-documents/src/migration.ts +++ b/models/controlled-documents/src/migration.ts @@ -34,7 +34,8 @@ import { type Ref, SortingOrder, toIdMap, - TxOperations + TxOperations, + type WorkspaceDataId } from '@hcengineering/core' import { createDefaultSpace, @@ -300,7 +301,8 @@ async function migrateDocSections (client: MigrationClient): Promise { // Migrate sections headers + content try { const collabId = makeDocCollabId(document, 'content') - const ydoc = await loadCollabYdoc(ctx, storage, client.workspaceId, collabId) + const dataId = client.wsIds.dataId ?? (client.wsIds.uuid as unknown as WorkspaceDataId) + const ydoc = await loadCollabYdoc(ctx, storage, dataId, collabId) if (ydoc === undefined) { // no content, ignore continue @@ -346,7 +348,7 @@ async function migrateDocSections (client: MigrationClient): Promise { } }) - await saveCollabYdoc(ctx, storage, client.workspaceId, collabId, ydoc) + await saveCollabYdoc(ctx, storage, dataId, collabId, ydoc) } catch (err) { ctx.error('error collaborative document content migration', { error: err, document: document.title }) } diff --git a/models/controlled-documents/src/types.ts b/models/controlled-documents/src/types.ts index 30f809588e..c450cf625d 100644 --- a/models/controlled-documents/src/types.ts +++ b/models/controlled-documents/src/types.ts @@ -55,9 +55,9 @@ import { type CollectionSize, type Role, type TypedSpace, - type Account, type RolesAssignment, - type Rank + type Rank, + type PersonId } from '@hcengineering/core' import { ArrOf, @@ -472,7 +472,7 @@ export class TDocumentApprovalRequest extends TDocumentRequest implements Docume @Mixin(documents.mixin.DocumentSpaceTypeData, documents.class.DocumentSpace) @UX(getEmbeddedLabel('Default Documents'), documents.icon.Document) export class TDocumentSpaceTypeData extends TDocumentSpace implements RolesAssignment { - [key: Ref]: Ref[] + [key: Ref]: PersonId[] } /** diff --git a/models/core/src/core.ts b/models/core/src/core.ts index 67ddc55d0b..1abcddced5 100644 --- a/models/core/src/core.ts +++ b/models/core/src/core.ts @@ -14,7 +14,7 @@ // import { - type Account, + type PersonId, type AnyAttribute, type ArrOf, type Association, @@ -71,6 +71,7 @@ import { TypeRef, TypeString, TypeTimestamp, + TypePersonId, UX } from '@hcengineering/model' import { getEmbeddedLabel, type IntlString, type Plugin } from '@hcengineering/platform' @@ -102,13 +103,13 @@ export class TDoc extends TObj implements Doc { @Index(IndexKind.Indexed) modifiedOn!: Timestamp - @Prop(TypeRef(core.class.Account), core.string.ModifiedBy) + @Prop(TypePersonId(), core.string.ModifiedBy) @Index(IndexKind.Indexed) - modifiedBy!: Ref + modifiedBy!: PersonId - @Prop(TypeRef(core.class.Account), core.string.CreatedBy) + @Prop(TypePersonId(), core.string.CreatedBy) @Index(IndexKind.Indexed) - createdBy!: Ref + createdBy!: PersonId @Prop(TypeTimestamp(), core.string.CreatedDate) @ReadOnly() @@ -260,6 +261,10 @@ export class TTypeFileSize extends TType {} @Model(core.class.TypeMarkup, core.class.Type) export class TTypeMarkup extends TType {} +@UX(core.string.PersonId) +@Model(core.class.TypePersonId, core.class.Type) +export class TTypePersonId extends TType {} + @UX(core.string.Ref) @Model(core.class.RefTo, core.class.Type) export class TRefTo extends TType implements RefTo { diff --git a/models/core/src/index.ts b/models/core/src/index.ts index f86265dca6..ee94511df3 100644 --- a/models/core/src/index.ts +++ b/models/core/src/index.ts @@ -14,7 +14,6 @@ // import { - AccountRole, DOMAIN_BENCHMARK, DOMAIN_BLOB, DOMAIN_CONFIGURATION, @@ -23,8 +22,7 @@ import { DOMAIN_SPACE, DOMAIN_STATUS, DOMAIN_TRANSIENT, - DOMAIN_TX, - systemAccountEmail + DOMAIN_TX } from '@hcengineering/core' import { type Builder } from '@hcengineering/model' import { TBenchmarkDoc } from './benchmark' @@ -64,6 +62,7 @@ import { TTypeHyperlink, TTypeIntlString, TTypeMarkup, + TTypePersonId, TTypeNumber, TTypeRank, TTypeRecord, @@ -74,16 +73,7 @@ import { TSequence } from './core' import { definePermissions } from './permissions' -import { - TAccount, - TPermission, - TRole, - TSpace, - TSpaceType, - TSpaceTypeDescriptor, - TSystemSpace, - TTypedSpace -} from './security' +import { TPermission, TRole, TSpace, TSpaceType, TSpaceTypeDescriptor, TSystemSpace, TTypedSpace } from './security' import { defineSpaceType } from './spaceType' import { TDomainStatusPlaceholder, TStatus, TStatusCategory } from './status' import { TUserStatus } from './transient' @@ -91,7 +81,7 @@ import { TTx, TTxApplyIf, TTxCreateDoc, TTxCUD, TTxMixin, TTxRemoveDoc, TTxUpdat export { coreId, DOMAIN_SPACE } from '@hcengineering/core' export * from './core' -export { coreOperation } from './migration' +export { coreOperation, getSocialIdByOldAccount, getAccountsFromTxes, getSocialKeyByOldEmail } from './migration' export * from './security' export * from './status' export * from './tx' @@ -120,11 +110,11 @@ export function createModel (builder: Builder): void { TSpaceTypeDescriptor, TRole, TPermission, - TAccount, TAttribute, TType, TEnumOf, TTypeMarkup, + TTypePersonId, TTypeCollaborativeDoc, TArrOf, TRefTo, @@ -164,16 +154,6 @@ export function createModel (builder: Builder): void { TTransientConfiguration ) - builder.createDoc( - core.class.Account, - core.space.Model, - { - email: systemAccountEmail, - role: AccountRole.Owner - }, - core.account.System - ) - builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, { domain: DOMAIN_TX, disabled: [ diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts index 5ece09f075..287b029ded 100644 --- a/models/core/src/migration.ts +++ b/models/core/src/migration.ts @@ -15,6 +15,7 @@ import { saveCollabJson } from '@hcengineering/collaboration' import core, { + buildSocialIdString, coreId, DOMAIN_MODEL_TX, DOMAIN_SPACE, @@ -26,6 +27,8 @@ import core, { makeDocCollabId, MeasureMetricsContext, RateLimiter, + SocialIdType, + type PersonId, type AnyAttribute, type Blob, type Class, @@ -36,7 +39,15 @@ import core, { type Space, type Status, type TxCreateDoc, - type TxCUD + type TxCUD, + type SpaceType, + type TxUpdateDoc, + type Role, + toIdMap, + type TypedSpace, + TxProcessor, + type SocialKey, + type WorkspaceDataId } from '@hcengineering/core' import { createDefaultSpace, @@ -226,7 +237,14 @@ async function processMigrateContentFor ( if (value != null && value.startsWith('{')) { try { const buffer = Buffer.from(value) - await storageAdapter.put(ctx, client.workspaceId, blobId, buffer, 'application/json', buffer.length) + await storageAdapter.put( + ctx, + client.wsIds.dataId ?? (client.wsIds.uuid as unknown as WorkspaceDataId), + blobId, + buffer, + 'application/json', + buffer.length + ) } catch (err) { ctx.error('failed to process document', { _class: doc._class, _id: doc._id, err }) } @@ -284,6 +302,221 @@ async function migrateCollaborativeDocsToJson (client: MigrationClient): Promise } } +export function getAccountsFromTxes (accTxes: TxCUD[]): any { + const byAccounts = accTxes.reduce[]>>((acc, tx) => { + if (acc[tx.objectId] === undefined) { + acc[tx.objectId] = [] + } + + acc[tx.objectId].push(tx) + return acc + }, {}) + + return Object.values(byAccounts) + .map((txes) => TxProcessor.buildDoc2Doc(txes)) + .filter((it) => it !== undefined) +} + +export async function getSocialIdByOldAccount (client: MigrationClient): Promise> { + const systemAccounts = [core.account.System, core.account.ConfigUser] + const accountsTxes: TxCUD[] = await client.find>(DOMAIN_MODEL_TX, { + objectClass: { $in: ['core:class:Account', 'contact:class:PersonAccount'] as Ref>[] } + }) + const accounts = getAccountsFromTxes(accountsTxes) + + const socialIdByAccount: Record = {} + for (const account of accounts) { + if (account.email === undefined) { + continue + } + + if (systemAccounts.includes(account._id)) { + socialIdByAccount[account._id] = account._id + } else { + socialIdByAccount[account._id] = buildSocialIdString(getSocialKeyByOldEmail(account.email)) + } + } + + return socialIdByAccount +} + +export function getSocialKeyByOldEmail (rawEmail: string): SocialKey { + const email = rawEmail.toLowerCase() + let type: SocialIdType + let value: string + if (email.startsWith('github:')) { + type = SocialIdType.GITHUB + value = email.slice(7) + } else if (email.startsWith('openid:')) { + type = SocialIdType.OIDC + value = email.slice(7) + } else { + type = SocialIdType.EMAIL + value = email + } + + return { + type, + value + } +} + +async function migrateAccountsToSocialIds (client: MigrationClient): Promise { + const ctx = new MeasureMetricsContext('core migrateAccountsToSocialIds', {}) + const hierarchy = client.hierarchy + const socialIdByAccount = await getSocialIdByOldAccount(client) + + ctx.info('migrating createdBy and modifiedBy') + function chunkArray (array: T[], chunkSize: number): T[][] { + const chunks: T[][] = [] + for (let i = 0; i < array.length; i += chunkSize) { + chunks.push(array.slice(i, i + chunkSize)) + } + return chunks + } + + const operations: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] + for (const [accId, socialId] of Object.entries(socialIdByAccount)) { + if (accId === socialId) continue + operations.push({ + filter: { createdBy: accId as any }, + update: { + createdBy: socialId + } + }) + operations.push({ + filter: { modifiedBy: accId as any }, + update: { + modifiedBy: socialId + } + }) + } + + if (operations.length > 0) { + const operationsChunks = chunkArray(operations, 40) + for (const domain of client.hierarchy.domains()) { + let processed = 0 + ctx.info('processing domain ', { domain }) + for (const operationsChunk of operationsChunks) { + if (operationsChunk.length === 0) continue + + await client.bulk(domain, operationsChunk) + processed++ + if (operationsChunks.length > 1) { + ctx.info('processed chunk', { processed, of: operationsChunks.length }) + } + } + } + } else { + ctx.info('no user accounts to migrate') + } + ctx.info('finished migrating createdBy and modifiedBy') + + const spaceTypes = client.model.findAllSync(core.class.SpaceType, {}) + const spaceTypesById = toIdMap(spaceTypes) + const roles = client.model.findAllSync(core.class.Role, {}) + const rolesBySpaceType = new Map, Role[]>() + for (const role of roles) { + const spaceType = role.attachedTo + if (spaceType === undefined) continue + if (rolesBySpaceType.has(spaceType)) { + rolesBySpaceType.get(spaceType)?.push(role) + } else { + rolesBySpaceType.set(spaceType, [role]) + } + } + + ctx.info('processing spaces members, owners and roles assignment', {}) + let processedSpaces = 0 + const spacesIterator = await client.traverse(DOMAIN_SPACE, {}) + + try { + while (true) { + const spaces = await spacesIterator.next(200) + if (spaces === null || spaces.length === 0) { + break + } + + const operations: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] + + for (const s of spaces) { + if (!hierarchy.isDerived(s._class, core.class.Space)) continue + const space = s as Space + + const newMembers = space.members.map((m) => socialIdByAccount[m] ?? m) + const newOwners = space.owners?.map((m) => socialIdByAccount[m] ?? m) + const update: MigrateUpdate = { + members: newMembers, + owners: newOwners + } + + const type = spaceTypesById.get((space as TypedSpace).type) + + if (type !== undefined) { + const mixin = hierarchy.as(space, type.targetClass) + if (mixin !== undefined) { + const roles = rolesBySpaceType.get(type._id) + + for (const role of roles ?? []) { + const oldAssignees: string[] | undefined = (mixin as any)[role._id] + if (oldAssignees != null && oldAssignees.length > 0) { + const newAssignees = oldAssignees.map((a) => socialIdByAccount[a]) + + update[`${type.targetClass}`] = { + [role._id]: newAssignees + } + } + } + } + } + + operations.push({ + filter: { _id: space._id }, + update + }) + } + + if (operations.length > 0) { + await client.bulk(DOMAIN_SPACE, operations) + } + + processedSpaces += spaces.length + ctx.info('...spaces processed', { count: processedSpaces }) + } + + ctx.info('finished processing spaces members, owners and roles assignment', { processedSpaces }) + } finally { + await spacesIterator.close() + } + + ctx.info('processing space types members', {}) + let updatedSpaceTypes = 0 + for (const spaceType of spaceTypes) { + if (spaceType.members === undefined || spaceType.members.length === 0) continue + + const newMembers = spaceType.members.map((m) => socialIdByAccount[m] ?? m) + const tx: TxUpdateDoc = { + _id: generateId(), + _class: core.class.TxUpdateDoc, + space: core.space.Tx, + objectId: spaceType._id, + objectClass: spaceType._class, + objectSpace: spaceType.space, + operations: { + members: newMembers + }, + modifiedOn: Date.now(), + createdBy: core.account.ConfigUser, + createdOn: Date.now(), + modifiedBy: core.account.ConfigUser + } + + await client.create(DOMAIN_MODEL_TX, tx) + updatedSpaceTypes++ + } + ctx.info('finished processing space types members', { totalSpaceTypes: spaceTypes.length, updatedSpaceTypes }) +} + async function processMigrateJsonForDomain ( ctx: MeasureContext, domain: Domain, @@ -331,7 +564,7 @@ async function processMigrateJsonForDoc ( client: MigrationClient, storageAdapter: StorageAdapter ): Promise> { - const { hierarchy, workspaceId } = client + const { hierarchy, wsIds } = client const update: MigrateUpdate = {} @@ -349,12 +582,13 @@ async function processMigrateJsonForDoc ( : attribute.name const collabId = makeDocCollabId(doc, attribute.name) - + const dataId = wsIds.dataId ?? (wsIds.uuid as unknown as WorkspaceDataId) if (value.startsWith('{')) { // For some reason we have documents that are already markups const jsonId = await retry(5, async () => { - return await saveCollabJson(ctx, storageAdapter, workspaceId, collabId, value) + return await saveCollabJson(ctx, storageAdapter, dataId, collabId, value) }) + update[attributeName] = jsonId continue } @@ -374,17 +608,17 @@ async function processMigrateJsonForDoc ( const ydocId = makeCollabYdocId(collabId) if (ydocId !== currentYdocId) { await retry(5, async () => { - const stat = await storageAdapter.stat(ctx, workspaceId, currentYdocId) + const stat = await storageAdapter.stat(ctx, dataId, currentYdocId) if (stat !== undefined) { - const data = await storageAdapter.read(ctx, workspaceId, currentYdocId) + const data = await storageAdapter.read(ctx, dataId, currentYdocId) const buffer = Buffer.concat(data as any) - await storageAdapter.put(ctx, workspaceId, ydocId, buffer, 'application/ydoc', buffer.length) + await storageAdapter.put(ctx, dataId, ydocId, buffer, 'application/ydoc', buffer.length) } }) } } catch (err) { const error = err instanceof Error ? err.message : String(err) - ctx.warn('failed to process collaborative doc', { workspaceId, collabId, currentYdocId, error }) + ctx.warn('failed to process collaborative doc', { dataId, collabId, currentYdocId, error }) } const unset = update.$unset ?? {} @@ -484,6 +718,10 @@ export const coreOperation: MigrateOperation = { { state: 'collaborative-docs-to-json', func: migrateCollaborativeDocsToJson + }, + { + state: 'accounts-to-social-ids', + func: migrateAccountsToSocialIds } ]) }, diff --git a/models/core/src/security.ts b/models/core/src/security.ts index ef0585d61d..c1fd68e16e 100644 --- a/models/core/src/security.ts +++ b/models/core/src/security.ts @@ -17,8 +17,7 @@ import { DOMAIN_MODEL, DOMAIN_SPACE, IndexKind, - type Account, - type AccountRole, + type PersonId, type Arr, type Class, type CollectionSize, @@ -42,6 +41,7 @@ import { TypeBoolean, TypeRef, TypeString, + TypePersonId, UX } from '@hcengineering/model' import { getEmbeddedLabel, type Asset, type IntlString } from '@hcengineering/platform' @@ -68,12 +68,12 @@ export class TSpace extends TDoc implements Space { @Index(IndexKind.Indexed) archived!: boolean - @Prop(ArrOf(TypeRef(core.class.Account)), core.string.Members) + @Prop(ArrOf(TypePersonId()), core.string.Members) @Index(IndexKind.Indexed) - members!: Arr> + members!: Arr - @Prop(ArrOf(TypeRef(core.class.Account)), core.string.Owners) - owners?: Ref[] + @Prop(ArrOf(TypePersonId()), core.string.Owners) + owners?: PersonId[] @Prop(TypeBoolean(), core.string.AutoJoin) autoJoin?: boolean @@ -119,8 +119,8 @@ export class TSpaceType extends TDoc implements SpaceType { @Prop(Collection(core.class.Role), core.string.Roles) roles!: CollectionSize - @Prop(ArrOf(TypeRef(core.class.Account)), core.string.Members) - members!: Arr> + @Prop(ArrOf(TypePersonId()), core.string.Members) + members!: Arr @Prop(TypeBoolean(), core.string.AutoJoin) autoJoin?: boolean @@ -162,12 +162,5 @@ export class TPermission extends TDoc implements Permission { @Mixin(core.mixin.SpacesTypeData, core.class.Space) @UX(getEmbeddedLabel("All spaces' type")) // TODO: add icon? export class TSpacesTypeData extends TSpace implements RolesAssignment { - [key: Ref]: Ref[] -} - -@Model(core.class.Account, core.class.Doc, DOMAIN_MODEL) -@UX(core.string.Account, undefined, undefined, 'name') -export class TAccount extends TDoc implements Account { - email!: string - role!: AccountRole + [key: Ref]: PersonId[] } diff --git a/models/core/src/spaceType.ts b/models/core/src/spaceType.ts index bfc5f7d5db..b1330fc702 100644 --- a/models/core/src/spaceType.ts +++ b/models/core/src/spaceType.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import { ArrOf, Prop, TypeRef, type Builder } from '@hcengineering/model' +import { ArrOf, Prop, TypeString, type Builder } from '@hcengineering/model' import { type Asset } from '@hcengineering/platform' import { getRoleAttributeLabel } from '@hcengineering/core' @@ -31,7 +31,7 @@ const roles = [ export function defineSpaceType (builder: Builder): void { for (const role of roles) { const label = getRoleAttributeLabel(role.name) - const roleAssgtType = ArrOf(TypeRef(core.class.Account)) + const roleAssgtType = ArrOf(TypeString()) Prop(roleAssgtType, label)(TSpacesTypeData.prototype, role._id) } diff --git a/models/core/src/transient.ts b/models/core/src/transient.ts index c653b3ac6d..69639dab27 100644 --- a/models/core/src/transient.ts +++ b/models/core/src/transient.ts @@ -13,13 +13,13 @@ // limitations under the License. // -import { DOMAIN_TRANSIENT, type Account, type Ref, type UserStatus } from '@hcengineering/core' +import { DOMAIN_TRANSIENT, type PersonUuid, type UserStatus } from '@hcengineering/core' import { Model } from '@hcengineering/model' import core from './component' import { TDoc } from './core' @Model(core.class.UserStatus, core.class.Doc, DOMAIN_TRANSIENT) export class TUserStatus extends TDoc implements UserStatus { - user!: Ref + user!: PersonUuid online!: boolean } diff --git a/models/document/src/index.ts b/models/document/src/index.ts index ab173e6bda..0adc234d81 100644 --- a/models/document/src/index.ts +++ b/models/document/src/index.ts @@ -14,8 +14,8 @@ // import activity from '@hcengineering/activity' -import type { CollectionSize, Domain, MarkupBlobRef, Rank, Role, RolesAssignment } from '@hcengineering/core' -import { Account, AccountRole, IndexKind, Ref } from '@hcengineering/core' +import type { CollectionSize, MarkupBlobRef, Domain, Rank, Ref, Role, RolesAssignment } from '@hcengineering/core' +import { PersonId, AccountRole, IndexKind } from '@hcengineering/core' import { type Document, type DocumentSnapshot, @@ -36,6 +36,7 @@ import { TypeNumber, TypeRef, TypeString, + TypePersonId, UX } from '@hcengineering/model' import attachment from '@hcengineering/model-attachment' @@ -79,9 +80,9 @@ export class TDocument extends TDoc implements Document, Todoable { @Hidden() declare space: Ref - @Prop(TypeRef(core.class.Account), document.string.LockedBy) + @Prop(TypePersonId(), document.string.LockedBy) @Hidden() - lockedBy?: Ref + lockedBy?: PersonId @Prop(Collection(attachment.class.Embedding), attachment.string.Embeddings) embeddings?: number @@ -150,7 +151,7 @@ export class TTeamspace extends TTypedSpace implements Teamspace {} @Mixin(document.mixin.DefaultTeamspaceTypeData, document.class.Teamspace) @UX(getEmbeddedLabel('Default teamspace type'), document.icon.Document) export class TDefaultTeamspaceTypeData extends TTeamspace implements RolesAssignment { - [key: Ref]: Ref[] + [key: Ref]: PersonId[] } function defineTeamspace (builder: Builder): void { diff --git a/models/document/src/migration.ts b/models/document/src/migration.ts index dffaeb7a44..0cc53d19b2 100644 --- a/models/document/src/migration.ts +++ b/models/document/src/migration.ts @@ -18,11 +18,12 @@ import { DOMAIN_TX, makeDocCollabId, MeasureMetricsContext, + type Ref, SortingOrder, type Class, type CollaborativeDoc, type Doc, - type Ref + type WorkspaceDataId } from '@hcengineering/core' import { type Document, type DocumentSnapshot, type Teamspace } from '@hcengineering/document' import { @@ -35,7 +36,7 @@ import { type MigrationUpgradeClient } from '@hcengineering/model' import { DOMAIN_ACTIVITY } from '@hcengineering/model-activity' -import core, { DOMAIN_SPACE } from '@hcengineering/model-core' +import core, { DOMAIN_SPACE, getSocialIdByOldAccount } from '@hcengineering/model-core' import { DOMAIN_NOTIFICATION } from '@hcengineering/notification' import { type Asset } from '@hcengineering/platform' import { makeRank } from '@hcengineering/rank' @@ -212,8 +213,8 @@ async function renameFieldsRevert (client: MigrationClient): Promise { try { const collabId = makeDocCollabId(document, 'content') - - const ydoc = await loadCollabYdoc(ctx, storage, client.workspaceId, collabId) + const dataId = client.wsIds.dataId ?? (client.wsIds.uuid as unknown as WorkspaceDataId) + const ydoc = await loadCollabYdoc(ctx, storage, dataId, collabId) if (ydoc === undefined) { continue } @@ -224,7 +225,7 @@ async function renameFieldsRevert (client: MigrationClient): Promise { yDocCopyXmlField(ydoc, 'description', 'content') - await saveCollabYdoc(ctx, storage, client.workspaceId, collabId, ydoc) + await saveCollabYdoc(ctx, storage, dataId, collabId, ydoc) } catch (err) { ctx.error('error document content migration', { error: err, document: document.title }) } @@ -260,8 +261,8 @@ async function restoreContentField (client: MigrationClient): Promise { for (const document of documents) { try { const collabId = makeDocCollabId(document, 'content') - - const ydoc = await loadCollabYdoc(ctx, storage, client.workspaceId, collabId) + const dataId = client.wsIds.dataId ?? (client.wsIds.uuid as unknown as WorkspaceDataId) + const ydoc = await loadCollabYdoc(ctx, storage, dataId, collabId) if (ydoc === undefined) { ctx.error('document content not found', { document: document.title }) continue @@ -275,7 +276,7 @@ async function restoreContentField (client: MigrationClient): Promise { if (ydoc.share.has('')) { yDocCopyXmlField(ydoc, '', 'content') if (ydoc.share.has('content')) { - await saveCollabYdoc(ctx, storage, client.workspaceId, collabId, ydoc) + await saveCollabYdoc(ctx, storage, dataId, collabId, ydoc) } else { ctx.error('document content still not found', { document: document.title }) } @@ -296,6 +297,51 @@ async function migrateRanks (client: MigrationClient): Promise { } } +async function migrateAccountsToSocialIds (client: MigrationClient): Promise { + const ctx = new MeasureMetricsContext('document migrateAccountsToSocialIds', {}) + const socialIdByAccount = await getSocialIdByOldAccount(client) + + ctx.info('processing document lockedBy ', {}) + const iterator = await client.traverse(DOMAIN_DOCUMENT, { _class: document.class.Document }) + + try { + let processed = 0 + while (true) { + const docs = await iterator.next(200) + if (docs === null || docs.length === 0) { + break + } + + const operations: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] + + for (const doc of docs) { + const document = doc as Document + const newLockedBy = + document.lockedBy != null ? socialIdByAccount[document.lockedBy] ?? document.lockedBy : document.lockedBy + + if (newLockedBy === document.lockedBy) continue + + operations.push({ + filter: { _id: document._id }, + update: { + lockedBy: newLockedBy + } + }) + } + + if (operations.length > 0) { + await client.bulk(DOMAIN_DOCUMENT, operations) + } + + processed += docs.length + ctx.info('...processed', { count: processed }) + } + } finally { + await iterator.close() + } + ctx.info('finished processing document lockedBy ', {}) +} + async function removeOldClasses (client: MigrationClient): Promise { const classes = [ 'document:class:DocumentContent', @@ -355,6 +401,10 @@ export const documentOperation: MigrateOperation = { { state: 'migrateEmbeddings', func: migrateEmbeddings + }, + { + state: 'accounts-to-social-ids', + func: migrateAccountsToSocialIds } ]) }, diff --git a/models/drive/src/index.ts b/models/drive/src/index.ts index d6d5bd7e4f..338eaa7bb7 100644 --- a/models/drive/src/index.ts +++ b/models/drive/src/index.ts @@ -24,7 +24,7 @@ import core, { type Ref, type Role, type RolesAssignment, - Account, + PersonId, AccountRole, IndexKind, SortingOrder @@ -81,7 +81,7 @@ export class TDrive extends TTypedSpace implements Drive {} @Mixin(drive.mixin.DefaultDriveTypeData, drive.class.Drive) @UX(getEmbeddedLabel('Default drive type')) export class TDefaultDriveTypeData extends TDrive implements RolesAssignment { - [key: Ref]: Ref[] + [key: Ref]: PersonId[] } @Model(drive.class.Resource, core.class.Doc, DOMAIN_DRIVE) diff --git a/models/guest/src/index.ts b/models/guest/src/index.ts index bf9fe3e495..5c6e3ff038 100644 --- a/models/guest/src/index.ts +++ b/models/guest/src/index.ts @@ -1,12 +1,5 @@ -import { - AccountRole, - type Class, - type IndexingConfiguration, - type Doc, - type Domain, - type Ref -} from '@hcengineering/core' -import { type PublicLink, type Restrictions, guestAccountEmail } from '@hcengineering/guest' +import { type Class, type IndexingConfiguration, type Doc, type Domain, type Ref } from '@hcengineering/core' +import { type PublicLink, type Restrictions } from '@hcengineering/guest' import { type Builder, Model } from '@hcengineering/model' import core, { TDoc } from '@hcengineering/model-core' import { type Location } from '@hcengineering/ui' @@ -26,15 +19,6 @@ export class TPublicLink extends TDoc implements PublicLink { export function createModel (builder: Builder): void { builder.createModel(TPublicLink) - builder.createDoc( - core.class.Account, - core.space.Model, - { - email: guestAccountEmail, - role: AccountRole.DocGuest - }, - guest.account.Guest - ) builder.createDoc(core.class.DomainIndexConfiguration, core.space.Model, { domain: GUEST_DOMAIN, disabled: [ diff --git a/models/guest/src/migration.ts b/models/guest/src/migration.ts index 17e8e31380..1a4251c858 100644 --- a/models/guest/src/migration.ts +++ b/models/guest/src/migration.ts @@ -1,78 +1,15 @@ -import { - AccountRole, - DOMAIN_MODEL_TX, - type Account, - type Ref, - type Space, - type TxCreateDoc, - type TxUpdateDoc -} from '@hcengineering/core' import { guestId } from '@hcengineering/guest' import { - migrateSpace, tryMigrate, type MigrateOperation, type MigrationClient, type MigrationUpgradeClient, type ModelLogger } from '@hcengineering/model' -import core from '@hcengineering/model-core' -import { GUEST_DOMAIN } from '.' export const guestOperation: MigrateOperation = { async migrate (client: MigrationClient, logger: ModelLogger): Promise { - await tryMigrate(client, guestId, [ - { - state: 'migrateRoles', - func: async (client) => { - const stateMap = { - 0: AccountRole.User, - 1: AccountRole.Maintainer, - 2: AccountRole.Owner - } - const createTxes = await client.find>(DOMAIN_MODEL_TX, { - _class: core.class.TxCreateDoc, - 'attributes.role': { $in: [0, 1, 2] } - }) - for (const tx of createTxes) { - await client.update( - DOMAIN_MODEL_TX, - { - _id: tx._id - }, - { - $set: { - 'attributes.role': (stateMap as any)[tx.attributes.role] - } - } - ) - } - const updateTxes = await client.find>(DOMAIN_MODEL_TX, { - _class: core.class.TxUpdateDoc, - 'operations.role': { $in: [0, 1, 2] } - }) - for (const tx of updateTxes) { - await client.update( - DOMAIN_MODEL_TX, - { - _id: tx._id - }, - { - $set: { - 'operations.role': (stateMap as any)[(tx.operations as any).role] - } - } - ) - } - } - }, - { - state: 'removeDeprecatedSpace', - func: async (client: MigrationClient) => { - await migrateSpace(client, 'guest:space:Links' as Ref, core.space.Workspace, [GUEST_DOMAIN]) - } - } - ]) + await tryMigrate(client, guestId, []) }, async upgrade (state: Map>, client: () => Promise): Promise {} } diff --git a/models/guest/src/plugin.ts b/models/guest/src/plugin.ts index deff851c67..855d30cd84 100644 --- a/models/guest/src/plugin.ts +++ b/models/guest/src/plugin.ts @@ -1,4 +1,4 @@ -import { type Account, type Doc, type Ref } from '@hcengineering/core' +import { type Doc, type Ref } from '@hcengineering/core' import { guestId } from '@hcengineering/guest' import guest from '@hcengineering/guest-resources/src/plugin' import { mergeIds } from '@hcengineering/platform' @@ -6,9 +6,6 @@ import { type AnyComponent } from '@hcengineering/ui' import { type Action, type ActionCategory } from '@hcengineering/view' export default mergeIds(guestId, guest, { - account: { - Guest: '' as Ref - }, action: { CreatePublicLink: '' as Ref> }, diff --git a/models/hr/src/index.ts b/models/hr/src/index.ts index 019405aceb..3f87a0afd1 100644 --- a/models/hr/src/index.ts +++ b/models/hr/src/index.ts @@ -28,7 +28,6 @@ import { import { hrId, type Department, - type DepartmentMember, type PublicHoliday, type Request, type RequestType, @@ -53,7 +52,7 @@ import { import attachment from '@hcengineering/model-attachment' import calendar from '@hcengineering/model-calendar' import chunter from '@hcengineering/model-chunter' -import contact, { TEmployee, TPersonAccount } from '@hcengineering/model-contact' +import contact, { TEmployee } from '@hcengineering/model-contact' import core, { TAttachedDoc, TDoc, TType } from '@hcengineering/model-core' import view, { classPresenter, createAction } from '@hcengineering/model-view' import workbench from '@hcengineering/model-workbench' @@ -97,8 +96,8 @@ export class TDepartment extends TDoc implements Department { @Prop(TypeRef(contact.mixin.Employee), hr.string.TeamLead) teamLead!: Ref | null - @Prop(ArrOf(TypeRef(hr.class.DepartmentMember)), contact.string.Members) - members!: Arr> + @Prop(ArrOf(TypeRef(contact.mixin.Employee)), contact.string.Members) + members!: Arr> @Prop(ArrOf(TypeRef(contact.class.Contact)), hr.string.Subscribers) subscribers?: Arr> @@ -107,10 +106,6 @@ export class TDepartment extends TDoc implements Department { managers!: Arr> } -@Model(hr.class.DepartmentMember, contact.class.PersonAccount) -@UX(contact.string.Employee, hr.icon.HR) -export class TDepartmentMember extends TPersonAccount implements DepartmentMember {} - @Mixin(hr.mixin.Staff, contact.mixin.Employee) @UX(hr.string.Staff, hr.icon.HR, 'STFF', 'name') export class TStaff extends TEmployee implements Staff { @@ -191,7 +186,7 @@ export class TPublicHoliday extends TDoc implements PublicHoliday { } export function createModel (builder: Builder): void { - builder.createModel(TDepartment, TDepartmentMember, TRequest, TRequestType, TPublicHoliday, TStaff, TTzDate) + builder.createModel(TDepartment, TRequest, TRequestType, TPublicHoliday, TStaff, TTzDate) builder.createDoc( workbench.class.Application, @@ -227,10 +222,6 @@ export function createModel (builder: Builder): void { editor: hr.component.EditRequest }) - builder.mixin(hr.class.DepartmentMember, core.class.Class, view.mixin.ArrayEditor, { - editor: hr.component.DepartmentStaff - }) - classPresenter(builder, hr.class.TzDate, hr.component.TzDatePresenter, hr.component.TzDateEditor) builder.createDoc( diff --git a/models/hr/src/migration.ts b/models/hr/src/migration.ts index 29f53575ac..6ad85ac5e9 100644 --- a/models/hr/src/migration.ts +++ b/models/hr/src/migration.ts @@ -13,7 +13,15 @@ // limitations under the License. // -import { type Space, TxOperations, type Ref } from '@hcengineering/core' +import { + type Space, + TxOperations, + type Ref, + type Class, + type Doc, + DOMAIN_MODEL_TX, + type TxCUD +} from '@hcengineering/core' import { type Department } from '@hcengineering/hr' import { migrateSpace, @@ -23,7 +31,8 @@ import { type MigrationClient, type MigrationUpgradeClient } from '@hcengineering/model' -import core, { DOMAIN_SPACE } from '@hcengineering/model-core' +import core, { DOMAIN_SPACE, getAccountsFromTxes } from '@hcengineering/model-core' + import hr, { DOMAIN_HR, hrId } from './index' async function createDepartment (tx: TxOperations): Promise { @@ -77,6 +86,23 @@ async function migrateDepartments (client: MigrationClient): Promise { ) } +async function migrateDepartmentMembersToEmployee (client: MigrationClient): Promise { + const departments = await client.find(DOMAIN_HR, { _class: hr.class.Department }) + + for (const department of departments) { + const accounts = department.members + if (accounts === undefined || accounts.length === 0) continue + + const personAccountsTxes: any[] = await client.find>(DOMAIN_MODEL_TX, { + objectClass: 'contact:class:PersonAccount' as Ref>, + objectId: { $in: accounts } + }) + const personAccounts = getAccountsFromTxes(personAccountsTxes) + + await client.update(DOMAIN_HR, { _id: department._id }, { members: personAccounts.map((pAcc: any) => pAcc.person) }) + } +} + export const hrOperation: MigrateOperation = { async migrate (client: MigrationClient): Promise { await tryMigrate(client, hrId, [ @@ -89,6 +115,10 @@ export const hrOperation: MigrateOperation = { func: async (client: MigrationClient) => { await migrateSpace(client, 'hr:space:HR' as Ref, core.space.Workspace, [DOMAIN_HR]) } + }, + { + state: 'migrateDepartmentMembersToEmployee', + func: migrateDepartmentMembersToEmployee } ]) }, diff --git a/models/lead/src/migration.ts b/models/lead/src/migration.ts index df581ebabf..4d13347cd5 100644 --- a/models/lead/src/migration.ts +++ b/models/lead/src/migration.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import { AccountRole, DOMAIN_MODEL_TX, TxOperations, type Ref, type Status } from '@hcengineering/core' +import { DOMAIN_MODEL_TX, TxOperations, type Ref, type Status } from '@hcengineering/core' import { leadId, type Lead } from '@hcengineering/lead' import { tryMigrate, @@ -25,7 +25,7 @@ import { } from '@hcengineering/model' import core, { DOMAIN_SPACE } from '@hcengineering/model-core' -import contact, { DOMAIN_CONTACT } from '@hcengineering/model-contact' +import { DOMAIN_CONTACT } from '@hcengineering/model-contact' import task, { createSequence, DOMAIN_TASK, migrateDefaultStatusesBase } from '@hcengineering/model-task' import lead from './plugin' @@ -149,24 +149,6 @@ async function migrateDefaultTypeMixins (client: MigrationClient): Promise ) } -async function migrateDefaultProjectOwners (client: MigrationClient): Promise { - const workspaceOwners = await client.model.findAll(contact.class.PersonAccount, { - role: AccountRole.Owner - }) - - await client.update( - DOMAIN_SPACE, - { - _id: lead.space.DefaultFunnel - }, - { - $set: { - owners: workspaceOwners.map((it) => it._id) - } - } - ) -} - export const leadOperation: MigrateOperation = { async preMigrate (client: MigrationClient, logger: ModelLogger): Promise { await tryMigrate(client, leadId, [ @@ -188,10 +170,6 @@ export const leadOperation: MigrateOperation = { await migrateDefaultTypeMixins(client) } }, - { - state: 'migrateDefaultProjectOwners', - func: migrateDefaultProjectOwners - }, { state: 'migrate-customer-description', func: async (client) => { diff --git a/models/lead/src/types.ts b/models/lead/src/types.ts index 25b807ec0e..92cc09a750 100644 --- a/models/lead/src/types.ts +++ b/models/lead/src/types.ts @@ -15,7 +15,7 @@ import type { Employee } from '@hcengineering/contact' import { - Account, + PersonId, IndexKind, type MarkupBlobRef, type Role, @@ -103,7 +103,7 @@ export class TCustomer extends TContact implements Customer { @Mixin(lead.mixin.DefaultFunnelTypeData, lead.class.Funnel) @UX(getEmbeddedLabel('Default funnel'), lead.icon.Funnel) export class TDefaultFunnelTypeData extends TFunnel implements RolesAssignment { - [key: Ref]: Ref[] + [key: Ref]: PersonId[] } @Mixin(lead.mixin.LeadTypeData, lead.class.Lead) diff --git a/models/notification/src/index.ts b/models/notification/src/index.ts index a4199c3ee2..f9eb7de0e8 100644 --- a/models/notification/src/index.ts +++ b/models/notification/src/index.ts @@ -20,7 +20,7 @@ import { AccountRole, DOMAIN_MODEL, IndexKind, - type Account, + type PersonId, type AttachedDoc, type Class, type Collection, @@ -47,6 +47,7 @@ import { TypeIntlString, TypeMarkup, TypeRef, + TypePersonId, UX, type Builder } from '@hcengineering/model' @@ -94,12 +95,12 @@ export { notification as default } @Model(notification.class.BrowserNotification, core.class.Doc, DOMAIN_TRANSIENT) export class TBrowserNotification extends TDoc implements BrowserNotification { - senderId?: Ref | undefined + senderId?: PersonId | undefined tag!: Ref> title!: string body!: string onClickLocation?: Location | undefined - user!: Ref + user!: PersonId messageId?: Ref messageClass?: Ref> objectId!: Ref @@ -108,7 +109,7 @@ export class TBrowserNotification extends TDoc implements BrowserNotification { @Model(notification.class.PushSubscription, core.class.Doc, DOMAIN_USER_NOTIFY) export class TPushSubscription extends TDoc implements PushSubscription { - user!: Ref + user!: PersonId endpoint!: string keys!: PushSubscriptionKeys } @@ -169,9 +170,9 @@ export class TClassCollaborators extends TClass { @Mixin(notification.mixin.Collaborators, core.class.Doc) @UX(notification.string.Collaborators) export class TCollaborators extends TDoc { - @Prop(ArrOf(TypeRef(core.class.Account)), notification.string.Collaborators) + @Prop(ArrOf(TypePersonId()), notification.string.Collaborators) @Index(IndexKind.Indexed) - collaborators!: Ref[] + collaborators!: PersonId[] } @Mixin(notification.mixin.NotificationObjectPresenter, core.class.Class) @@ -191,9 +192,9 @@ export class TNotificationContextPresenter extends TClass implements Notificatio @Model(notification.class.DocNotifyContext, core.class.Doc, DOMAIN_DOC_NOTIFY) export class TDocNotifyContext extends TDoc implements DocNotifyContext { - @Prop(TypeRef(core.class.Account), core.string.Account) + @Prop(TypePersonId(), core.string.Account) @Index(IndexKind.Indexed) - user!: Ref + user!: PersonId @Prop(TypeRef(core.class.Doc), core.string.Object) @Index(IndexKind.Indexed) @@ -228,9 +229,9 @@ export class TInboxNotification extends TDoc implements InboxNotification { @Index(IndexKind.Indexed) docNotifyContext!: Ref - @Prop(TypeRef(core.class.Account), core.string.Account) + @Prop(TypePersonId(), core.string.Account) @Index(IndexKind.Indexed) - user!: Ref + user!: PersonId @Prop(TypeBoolean(), core.string.Boolean) // @Index(IndexKind.Indexed) diff --git a/models/notification/src/migration.ts b/models/notification/src/migration.ts index 7b97e501c6..7ec4df8513 100644 --- a/models/notification/src/migration.ts +++ b/models/notification/src/migration.ts @@ -15,9 +15,19 @@ import chunter from '@hcengineering/chunter' import contact, { type PersonSpace } from '@hcengineering/contact' -import core, { DOMAIN_TX, type Class, type Doc, type DocumentQuery, type Ref, type Space } from '@hcengineering/core' +import core, { + DOMAIN_TX, + MeasureMetricsContext, + type Class, + type Doc, + type DocumentQuery, + type Ref, + type Space +} from '@hcengineering/core' import { migrateSpace, + type MigrateUpdate, + type MigrationDocumentQuery, tryMigrate, type MigrateOperation, type MigrationClient, @@ -31,7 +41,7 @@ import notification, { } from '@hcengineering/notification' import { DOMAIN_PREFERENCE } from '@hcengineering/preference' -import { DOMAIN_SPACE } from '@hcengineering/model-core' +import { DOMAIN_SPACE, getSocialIdByOldAccount } from '@hcengineering/model-core' import { DOMAIN_DOC_NOTIFY, DOMAIN_NOTIFICATION, DOMAIN_USER_NOTIFY } from './index' export async function removeNotifications ( @@ -221,6 +231,162 @@ export async function migrateDuplicateContexts (client: MigrationClient): Promis } } +async function migrateAccountsToSocialIds (client: MigrationClient): Promise { + const ctx = new MeasureMetricsContext('notification migrateAccountsToSocialIds', {}) + const hierarchy = client.hierarchy + const socialIdByAccount = await getSocialIdByOldAccount(client) + + ctx.info('processing collaborators ', {}) + for (const domain of client.hierarchy.domains()) { + ctx.info('processing domain ', { domain }) + let processed = 0 + const iterator = await client.traverse(domain, {}) + + try { + while (true) { + const docs = await iterator.next(200) + if (docs === null || docs.length === 0) { + break + } + + const operations: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] + + for (const doc of docs) { + const mixin = hierarchy.as(doc, notification.mixin.Collaborators) + const oldCollaborators = mixin.collaborators + + if (oldCollaborators === undefined || oldCollaborators.length === 0) continue + + const newCollaborators = oldCollaborators.map((c) => socialIdByAccount[c] ?? c) + + operations.push({ + filter: { _id: doc._id }, + update: { + [`${notification.mixin.Collaborators}`]: { + collaborators: newCollaborators + } + } + }) + } + + if (operations.length > 0) { + await client.bulk(domain, operations) + } + + processed += docs.length + ctx.info('...processed', { count: processed }) + } + + ctx.info('finished processing domain ', { domain, processed }) + } finally { + await iterator.close() + } + } + ctx.info('finished processing collaborators ', {}) + + ctx.info('processing notifications fields ', {}) + function chunkArray (array: T[], chunkSize: number): T[][] { + const chunks: T[][] = [] + for (let i = 0; i < array.length; i += chunkSize) { + chunks.push(array.slice(i, i + chunkSize)) + } + return chunks + } + + const operations: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] + for (const [accId, socialId] of Object.entries(socialIdByAccount)) { + if (accId === socialId) continue + operations.push({ + filter: { + user: accId, + _class: { + $in: [ + notification.class.DocNotifyContext, + notification.class.BrowserNotification, + notification.class.PushSubscription, + notification.class.InboxNotification, + notification.class.ActivityInboxNotification, + notification.class.CommonInboxNotification + ] + } + }, + update: { + user: socialId + } + }) + operations.push({ + filter: { + senderId: accId, + _class: notification.class.BrowserNotification + }, + update: { + senderId: socialId + } + }) + } + + if (operations.length > 0) { + const operationsChunks = chunkArray(operations, 40) + let processed = 0 + for (const operationsChunk of operationsChunks) { + if (operationsChunk.length === 0) continue + + await client.bulk(DOMAIN_NOTIFICATION, operationsChunk) + processed++ + if (operationsChunks.length > 1) { + ctx.info('processed chunk', { processed, of: operationsChunks.length }) + } + } + } else { + ctx.info('no user accounts to migrate') + } + + ctx.info('finished processing notifications fields ', {}) + + ctx.info('processing doc notify contexts ', {}) + const dncIterator = await client.traverse(DOMAIN_DOC_NOTIFY, { + _class: notification.class.DocNotifyContext + }) + try { + let processed = 0 + while (true) { + const docs = await dncIterator.next(200) + if (docs === null || docs.length === 0) { + break + } + + const operations: { + filter: MigrationDocumentQuery + update: MigrateUpdate + }[] = [] + + for (const doc of docs) { + const oldUser = doc.user + const newUser = socialIdByAccount[oldUser] ?? oldUser + + if (newUser !== oldUser) { + operations.push({ + filter: { _id: doc._id }, + update: { + user: newUser + } + }) + } + } + + if (operations.length > 0) { + await client.bulk(DOMAIN_DOC_NOTIFY, operations) + } + + processed += docs.length + ctx.info('...processed', { count: processed }) + } + } finally { + await dncIterator.close() + } + ctx.info('finished processing doc notify contexts ', {}) +} + export async function migrateSettings (client: MigrationClient): Promise { await client.update( DOMAIN_PREFERENCE, @@ -467,9 +633,13 @@ export const notificationOperation: MigrateOperation = { await client.update(DOMAIN_DOC_NOTIFY, { space: core.space.Space }, { space: core.space.Workspace }) } }, + // { + // state: 'migrate-notifications-object', + // func: migrateNotificationsObject + // }, { - state: 'migrate-notifications-object', - func: migrateNotificationsObject + state: 'accounts-to-social-ids', + func: migrateAccountsToSocialIds } ]) }, diff --git a/models/products/src/index.ts b/models/products/src/index.ts index 941c871b0e..e14c533f35 100644 --- a/models/products/src/index.ts +++ b/models/products/src/index.ts @@ -22,8 +22,8 @@ import { type Attachment } from '@hcengineering/attachment' import contact from '@hcengineering/contact' import chunter from '@hcengineering/chunter' import { getRoleAttributeProps } from '@hcengineering/setting' -import type { Type, CollectionSize, Markup, Arr, RolesAssignment, Permission, Role } from '@hcengineering/core' -import { IndexKind, Ref, Account } from '@hcengineering/core' +import type { Type, Ref, CollectionSize, Markup, Arr, RolesAssignment, Permission, Role } from '@hcengineering/core' +import { IndexKind, PersonId } from '@hcengineering/core' import { type Builder, Model, @@ -39,6 +39,7 @@ import { ArrOf, TypeAny, ReadOnly, + TypePersonId, Mixin } from '@hcengineering/model' import attachment from '@hcengineering/model-attachment' @@ -78,8 +79,8 @@ export class TTypeProductVersionState extends TType {} @Model(products.class.Product, documents.class.ExternalSpace) @UX(products.string.Product, products.icon.Product, 'Product', 'name', undefined, products.string.Products) export class TProduct extends TExternalSpace implements Product { - @Prop(ArrOf(TypeRef(core.class.Account)), core.string.Members) - declare members: Arr> + @Prop(ArrOf(TypePersonId()), core.string.Members) + declare members: Arr @Prop(TypeMarkup(), products.string.Description) @Index(IndexKind.FullText) @@ -145,7 +146,7 @@ export class TProductVersion extends TProject implements ProductVersion { @Mixin(products.mixin.ProductTypeData, products.class.Product) @UX(getEmbeddedLabel('Default Products'), products.icon.ProductVersion) export class TProductTypeData extends TProduct implements RolesAssignment { - [key: Ref]: Ref[] + [key: Ref]: PersonId[] } function defineProduct (builder: Builder): void { diff --git a/models/recruit/src/types.ts b/models/recruit/src/types.ts index 8d1c097744..850d634780 100644 --- a/models/recruit/src/types.ts +++ b/models/recruit/src/types.ts @@ -15,7 +15,7 @@ import type { Employee, Organization } from '@hcengineering/contact' import { - Account, + PersonId, IndexKind, type Collection, type MarkupBlobRef, @@ -251,7 +251,7 @@ export class TOpinion extends TAttachedDoc implements Opinion { @Mixin(recruit.mixin.DefaultVacancyTypeData, recruit.class.Vacancy) @UX(getEmbeddedLabel('Default vacancy'), recruit.icon.Vacancy) export class TDefaultVacancyTypeData extends TVacancy implements RolesAssignment { - [key: Ref]: Ref[] + [key: Ref]: PersonId[] } @Mixin(recruit.mixin.ApplicantTypeData, recruit.class.Applicant) diff --git a/models/request/src/migration.ts b/models/request/src/migration.ts index 7ff603449a..1b9e7248bc 100644 --- a/models/request/src/migration.ts +++ b/models/request/src/migration.ts @@ -12,88 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. // -import contact, { type Person, type PersonAccount } from '@hcengineering/contact' -import core, { DOMAIN_MODEL_TX, type Ref, type TxCreateDoc } from '@hcengineering/core' +import { requestId } from '@hcengineering/request' import { tryMigrate, type MigrateOperation, - type MigrateUpdate, type MigrationClient, - type MigrationDocumentQuery, type MigrationUpgradeClient, type ModelLogger } from '@hcengineering/model' -import request, { requestId, type Request } from '@hcengineering/request' - -import { DOMAIN_REQUEST } from '.' - -async function migrateRequestPersonAccounts (client: MigrationClient): Promise { - const descendants = client.hierarchy.getDescendants(request.class.Request) - const requests = await client.find(DOMAIN_REQUEST, { - _class: { $in: descendants } - }) - const personAccountsCreateTxes = await client.find(DOMAIN_MODEL_TX, { - _class: core.class.TxCreateDoc, - objectClass: contact.class.PersonAccount - }) - const personAccountToPersonMap = personAccountsCreateTxes.reduce, Ref>>( - (map, tx) => { - const ctx = tx as TxCreateDoc - - map[ctx.objectId] = ctx.attributes.person - - return map - }, - {} - ) - const operations: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] - for (const request of requests) { - const newRequestedPersons = request.requested - .map((paId) => personAccountToPersonMap[paId as unknown as Ref]) - .filter((p) => p != null) - const newApprovedPersons = request.approved - .map((paId) => personAccountToPersonMap[paId as unknown as Ref]) - .filter((p) => p != null) - const newRejectedPerson = - request.rejected != null ? personAccountToPersonMap[request.rejected as unknown as Ref] : undefined - - if (newRequestedPersons.length > 0) { - operations.push({ - filter: { - _id: request._id - }, - update: { - requested: newRequestedPersons, - approved: newApprovedPersons - } - }) - } - - if (newRejectedPerson !== undefined) { - operations.push({ - filter: { - _id: request._id - }, - update: { - rejected: newRejectedPerson - } - }) - } - } - - if (operations.length > 0) { - await client.bulk(DOMAIN_REQUEST, operations) - } -} export const requestOperation: MigrateOperation = { async migrate (client: MigrationClient, logger: ModelLogger): Promise { - await tryMigrate(client, requestId, [ - { - state: 'migrateRequestPersonAccounts', - func: migrateRequestPersonAccounts - } - ]) + await tryMigrate(client, requestId, []) }, async upgrade (state: Map>, client: () => Promise): Promise {} } diff --git a/models/server-activity/src/migration.ts b/models/server-activity/src/migration.ts index 24f1b44ac4..d4e626aa26 100644 --- a/models/server-activity/src/migration.ts +++ b/models/server-activity/src/migration.ts @@ -54,7 +54,7 @@ function getActivityControl (client: MigrationClient): ActivityControl { findAll: async (ctx, _class, query, options) => toFindResult(await client.find(client.hierarchy.getDomain(_class), query, options)), storageAdapter: client.storageAdapter, - workspace: client.workspaceId + workspace: client.wsIds } } diff --git a/models/server-ai-bot/src/index.ts b/models/server-ai-bot/src/index.ts index 7fd057182d..e9e2940862 100644 --- a/models/server-ai-bot/src/index.ts +++ b/models/server-ai-bot/src/index.ts @@ -25,7 +25,6 @@ import { TChatMessage } from '@hcengineering/model-chunter' export { serverAiBotId } from '@hcengineering/server-ai-bot' export const DOMAIN_AI_BOT = 'ai_bot' as Domain - @Mixin(aiBot.mixin.TransferredMessage, chunter.class.ChatMessage) export class TTransferredMessage extends TChatMessage implements TransferredMessage { messageId!: Ref diff --git a/models/server-calendar/src/index.ts b/models/server-calendar/src/index.ts index 06567046db..814283bb25 100644 --- a/models/server-calendar/src/index.ts +++ b/models/server-calendar/src/index.ts @@ -34,10 +34,18 @@ export function createModel (builder: Builder): void { }) builder.createDoc(serverCore.class.Trigger, core.space.Model, { - trigger: serverCalendar.trigger.OnPersonAccountCreate, + trigger: serverCalendar.trigger.OnSocialIdentityCreate, txMatch: { _class: core.class.TxCreateDoc, - objectClass: contact.class.PersonAccount + objectClass: contact.class.SocialIdentity + } + }) + + builder.createDoc(serverCore.class.Trigger, core.space.Model, { + trigger: serverCalendar.trigger.OnEmployee, + txMatch: { + _class: core.class.TxMixin, + mixin: contact.mixin.Employee } }) diff --git a/models/server-contact/src/index.ts b/models/server-contact/src/index.ts index 8476613c83..381587053d 100644 --- a/models/server-contact/src/index.ts +++ b/models/server-contact/src/index.ts @@ -71,6 +71,14 @@ export function createModel (builder: Builder): void { } }) + builder.createDoc(serverCore.class.Trigger, core.space.Model, { + trigger: serverContact.trigger.OnSocialIdentityCreate, + txMatch: { + _class: core.class.TxCreateDoc, + objectClass: contact.class.SocialIdentity + } + }) + builder.createDoc(serverCore.class.Trigger, core.space.Model, { trigger: serverContact.trigger.OnEmployeeCreate, txMatch: { @@ -81,14 +89,6 @@ export function createModel (builder: Builder): void { } }) - builder.createDoc(serverCore.class.Trigger, core.space.Model, { - trigger: serverContact.trigger.OnPersonAccountCreate, - txMatch: { - objectClass: contact.class.PersonAccount, - _class: core.class.TxCreateDoc - } - }) - builder.createDoc(serverCore.class.Trigger, core.space.Model, { trigger: serverContact.trigger.OnSpaceTypeMembers, txMatch: { diff --git a/models/server-controlled-documents/src/index.ts b/models/server-controlled-documents/src/index.ts index 23fa2dbc80..756e923188 100644 --- a/models/server-controlled-documents/src/index.ts +++ b/models/server-controlled-documents/src/index.ts @@ -14,6 +14,14 @@ import notification from '@hcengineering/notification' export { serverDocumentsId } from '@hcengineering/server-controlled-documents/src/index' export function createModel (builder: Builder): void { + builder.createDoc(serverCore.class.Trigger, core.space.Model, { + trigger: serverDocuments.trigger.OnSocialIdentityCreate, + txMatch: { + _class: core.class.TxCreateDoc, + objectClass: contact.class.SocialIdentity + } + }) + builder.createDoc(serverCore.class.Trigger, core.space.Model, { trigger: serverDocuments.trigger.OnDocDeleted, txMatch: { @@ -50,13 +58,6 @@ export function createModel (builder: Builder): void { } }) - builder.createDoc(serverCore.class.Trigger, core.space.Model, { - trigger: serverDocuments.trigger.OnWorkspaceOwnerAdded, - txMatch: { - objectClass: contact.class.PersonAccount - } - }) - builder.mixin(documents.class.DocumentMeta, core.class.Class, serverCore.mixin.SearchPresenter, { iconConfig: { component: documents.component.DocumentIcon diff --git a/models/server-lead/src/index.ts b/models/server-lead/src/index.ts index 443dc9e69b..298142fc76 100644 --- a/models/server-lead/src/index.ts +++ b/models/server-lead/src/index.ts @@ -44,9 +44,10 @@ export function createModel (builder: Builder): void { ) builder.createDoc(serverCore.class.Trigger, core.space.Model, { - trigger: serverLead.trigger.OnWorkspaceOwnerAdded, + trigger: serverLead.trigger.OnSocialIdentityCreate, txMatch: { - objectClass: contact.class.PersonAccount + _class: core.class.TxCreateDoc, + objectClass: contact.class.SocialIdentity } }) } diff --git a/models/server-tracker/src/index.ts b/models/server-tracker/src/index.ts index e41a90d5f6..c78e8c693d 100644 --- a/models/server-tracker/src/index.ts +++ b/models/server-tracker/src/index.ts @@ -51,6 +51,14 @@ export function createModel (builder: Builder): void { title: [['title']] }) + builder.createDoc(serverCore.class.Trigger, core.space.Model, { + trigger: serverTracker.trigger.OnSocialIdentityCreate, + txMatch: { + _class: core.class.TxCreateDoc, + objectClass: contact.class.SocialIdentity + } + }) + builder.createDoc(serverCore.class.Trigger, core.space.Model, { trigger: serverTracker.trigger.OnIssueUpdate, txMatch: { @@ -66,13 +74,6 @@ export function createModel (builder: Builder): void { } }) - builder.createDoc(serverCore.class.Trigger, core.space.Model, { - trigger: serverTracker.trigger.OnWorkspaceOwnerAdded, - txMatch: { - objectClass: contact.class.PersonAccount - } - }) - builder.mixin( tracker.ids.AssigneeNotification, notification.class.NotificationType, diff --git a/models/setting/src/index.ts b/models/setting/src/index.ts index 5afe7b274b..a28457be42 100644 --- a/models/setting/src/index.ts +++ b/models/setting/src/index.ts @@ -15,8 +15,8 @@ import activity from '@hcengineering/activity' import contact from '@hcengineering/contact' -import { AccountRole, DOMAIN_MODEL, type Account, type Blob, type Domain, type Ref } from '@hcengineering/core' -import { Mixin, Model, UX, type Builder } from '@hcengineering/model' +import { AccountRole, DOMAIN_MODEL, type PersonId, type Blob, type Domain, type Ref } from '@hcengineering/core' +import { Mixin, Model, type Builder, UX } from '@hcengineering/model' import core, { TClass, TConfiguration, TDoc } from '@hcengineering/model-core' import view, { createAction } from '@hcengineering/model-view' import notification from '@hcengineering/notification' @@ -53,7 +53,7 @@ export class TIntegration extends TDoc implements Integration { type!: Ref disabled!: boolean value!: string - shared!: Ref[] + shared!: PersonId[] error?: IntlString | null } @Model(setting.class.SettingsCategory, core.class.Doc, DOMAIN_MODEL) diff --git a/models/setting/src/migration.ts b/models/setting/src/migration.ts index 062ca77dcf..da8ac6d128 100644 --- a/models/setting/src/migration.ts +++ b/models/setting/src/migration.ts @@ -13,17 +13,66 @@ // limitations under the License. // -import core, { type Ref, type Space } from '@hcengineering/core' +import core, { MeasureMetricsContext, type Ref, type Space } from '@hcengineering/core' import { migrateSpace, + type MigrateUpdate, + type MigrationDocumentQuery, tryMigrate, type MigrateOperation, type MigrationClient, type MigrationUpgradeClient } from '@hcengineering/model' -import { settingId } from '@hcengineering/setting' +import setting, { type Integration, settingId } from '@hcengineering/setting' +import { getSocialIdByOldAccount } from '@hcengineering/model-core' + import { DOMAIN_SETTING } from '.' +async function migrateAccountsToSocialIds (client: MigrationClient): Promise { + const ctx = new MeasureMetricsContext('setting migrateAccountsToSocialIds', {}) + const socialIdByAccount = await getSocialIdByOldAccount(client) + + ctx.info('processing setting integration shared ', {}) + const iterator = await client.traverse(DOMAIN_SETTING, { _class: setting.class.Integration }) + + try { + let processed = 0 + while (true) { + const docs = await iterator.next(200) + if (docs === null || docs.length === 0) { + break + } + + const operations: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] + + for (const doc of docs) { + const integration = doc as Integration + + if (integration.shared === undefined || integration.shared.length === 0) continue + + const newShared = integration.shared.map((s) => socialIdByAccount[s] ?? s) + + operations.push({ + filter: { _id: integration._id }, + update: { + shared: newShared + } + }) + } + + if (operations.length > 0) { + await client.bulk(DOMAIN_SETTING, operations) + } + + processed += docs.length + ctx.info('...processed', { count: processed }) + } + } finally { + await iterator.close() + } + ctx.info('finished processing setting integration shared ', {}) +} + export const settingOperation: MigrateOperation = { async migrate (client: MigrationClient): Promise { await tryMigrate(client, settingId, [ @@ -32,6 +81,10 @@ export const settingOperation: MigrateOperation = { func: async (client: MigrationClient) => { await migrateSpace(client, 'setting:space:Setting' as Ref, core.space.Workspace, [DOMAIN_SETTING]) } + }, + { + state: 'accounts-to-social-ids', + func: migrateAccountsToSocialIds } ]) }, diff --git a/models/task/src/index.ts b/models/task/src/index.ts index ccc078d3d2..7f7e2e40f7 100644 --- a/models/task/src/index.ts +++ b/models/task/src/index.ts @@ -371,23 +371,6 @@ export function createModel (builder: Builder): void { task.action.Move ) - createAction( - builder, - { - label: task.string.Export, - action: task.actionImpl.ExportTasks, - icon: view.icon.Move, - input: 'selection', - category: view.category.General, - target: task.class.Task, - context: { - mode: ['context', 'browser'], - group: 'tools' - } - }, - task.action.ExportTasks - ) - builder.createDoc( core.class.StatusCategory, core.space.Model, diff --git a/models/task/src/plugin.ts b/models/task/src/plugin.ts index 94f863de8e..3e6d880bdc 100644 --- a/models/task/src/plugin.ts +++ b/models/task/src/plugin.ts @@ -27,13 +27,11 @@ export default mergeIds(taskId, task, { action: { EditStatuses: '' as Ref, ArchiveState: '' as Ref, - PublicLink: '' as Ref>, - ExportTasks: '' as Ref + PublicLink: '' as Ref> }, actionImpl: { EditStatuses: '' as ViewAction, - SelectStatus: '' as ViewAction, - ExportTasks: '' as ViewAction + SelectStatus: '' as ViewAction }, category: { Task: '' as Ref, diff --git a/models/test-management/src/types.ts b/models/test-management/src/types.ts index 195bd7b10b..8742ecd660 100644 --- a/models/test-management/src/types.ts +++ b/models/test-management/src/types.ts @@ -32,18 +32,18 @@ import contact from '@hcengineering/contact' import chunter from '@hcengineering/chunter' import { getEmbeddedLabel } from '@hcengineering/platform' import { - Account, DateRangeMode, IndexKind, type RolesAssignment, type Role, - Ref, + type Ref, type Domain, type Timestamp, type Type, type CollectionSize, type MarkupBlobRef, - type Class + type Class, + type PersonId } from '@hcengineering/core' import { Mixin, @@ -107,7 +107,7 @@ export class TTestProject extends TTypedSpace implements TestProject { @Mixin(testManagement.mixin.DefaultProjectTypeData, testManagement.class.TestProject) @UX(getEmbeddedLabel('Default project'), testManagement.icon.TestProject) export class TDefaultProjectTypeData extends TTestProject implements RolesAssignment { - [key: Ref]: Ref[] + [key: Ref]: PersonId[] } /** diff --git a/models/time/src/migration.ts b/models/time/src/migration.ts index 91e282515c..9065ef2ca0 100644 --- a/models/time/src/migration.ts +++ b/models/time/src/migration.ts @@ -13,8 +13,7 @@ // limitations under the License. // -import { type PersonAccount } from '@hcengineering/contact' -import { type Account, type Doc, type Ref, SortingOrder, TxOperations } from '@hcengineering/core' +import { TxOperations } from '@hcengineering/core' import { type MigrateOperation, type MigrationClient, @@ -24,132 +23,12 @@ import { tryUpgrade, createDefaultSpace } from '@hcengineering/model' -import { makeRank } from '@hcengineering/rank' import core from '@hcengineering/model-core' -import task from '@hcengineering/task' import tags from '@hcengineering/tags' -import { timeId, type ToDo, ToDoPriority } from '@hcengineering/time' +import { timeId, ToDoPriority } from '@hcengineering/time' import { DOMAIN_TIME } from '.' import time from './plugin' -export async function migrateWorkSlots (client: TxOperations): Promise { - const h = client.getHierarchy() - const desc = h.getDescendants(task.class.Task) - const oldWorkSlots = await client.findAll(time.class.WorkSlot, { - attachedToClass: { $in: desc } - }) - const now = Date.now() - const todos = new Map, Ref>() - const count = new Map, number>() - let rank = makeRank(undefined, undefined) - for (const oldWorkSlot of oldWorkSlots) { - const todo = todos.get(oldWorkSlot.attachedTo) - if (todo === undefined) { - const acc = oldWorkSlot.space.replace('_calendar', '') as Ref - const account = (await client.findOne(core.class.Account, { _id: acc })) as PersonAccount - if (account.person !== undefined) { - rank = makeRank(undefined, rank) - const todo = await client.addCollection( - time.class.ProjectToDo, - time.space.ToDos, - oldWorkSlot.attachedTo, - oldWorkSlot.attachedToClass, - 'todos', - { - attachedSpace: (oldWorkSlot as any).attachedSpace, - title: oldWorkSlot.title, - description: '', - doneOn: oldWorkSlot.dueDate > now ? null : oldWorkSlot.dueDate, - workslots: 0, - priority: ToDoPriority.NoPriority, - user: account.person, - visibility: 'public', - rank - } - ) - await client.update(oldWorkSlot, { - attachedTo: todo, - attachedToClass: time.class.ProjectToDo, - collection: 'workslots' - }) - todos.set(oldWorkSlot.attachedTo, todo) - count.set(todo, 1) - } - } else { - await client.update(oldWorkSlot, { - attachedTo: todo, - attachedToClass: time.class.ProjectToDo, - collection: 'workslots' - }) - const c = count.get(todo) ?? 1 - count.set(todo, c + 1) - } - } - for (const [todoId, c] of count.entries()) { - const todo = await client.findOne(time.class.ToDo, { _id: todoId }) - if (todo === undefined) continue - const tx = client.txFactory.createTxUpdateDoc(time.class.ToDo, todo.space, todo._id, { - workslots: c - }) - tx.space = core.space.DerivedTx - await client.tx(tx) - } -} - -async function migrateTodosSpace (client: TxOperations): Promise { - const oldTodos = await client.findAll(time.class.ToDo, { - space: { $ne: time.space.ToDos } - }) - for (const oldTodo of oldTodos) { - const account = (await client.findOne(core.class.Account, { - _id: oldTodo.space as string as Ref - })) as PersonAccount - if (account.person === undefined) continue - await client.update(oldTodo, { - user: account.person, - space: time.space.ToDos - }) - } -} - -async function migrateTodosRanks (client: TxOperations): Promise { - const doneTodos = await client.findAll( - time.class.ToDo, - { - rank: { $exists: false }, - doneOn: null - }, - { - sort: { modifiedOn: SortingOrder.Ascending } - } - ) - let doneTodoRank = makeRank(undefined, undefined) - for (const todo of doneTodos) { - await client.update(todo, { - rank: doneTodoRank - }) - doneTodoRank = makeRank(undefined, doneTodoRank) - } - - const undoneTodos = await client.findAll( - time.class.ToDo, - { - rank: { $exists: false }, - doneOn: { $ne: null } - }, - { - sort: { doneOn: SortingOrder.Ascending } - } - ) - let undoneTodoRank = makeRank(undefined, undefined) - for (const todo of undoneTodos) { - await client.update(todo, { - rank: undoneTodoRank - }) - undoneTodoRank = makeRank(undefined, undoneTodoRank) - } -} - async function fillProps (client: MigrationClient): Promise { await client.update( DOMAIN_TIME, @@ -200,9 +79,6 @@ export const timeOperation: MigrateOperation = { }, time.category.Other ) - await migrateWorkSlots(tx) - await migrateTodosSpace(tx) - await migrateTodosRanks(tx) } } ]) diff --git a/models/tracker/src/migration.ts b/models/tracker/src/migration.ts index cf9da14fc6..b6386fae04 100644 --- a/models/tracker/src/migration.ts +++ b/models/tracker/src/migration.ts @@ -15,7 +15,6 @@ import activity, { type DocUpdateMessage } from '@hcengineering/activity' import core, { - AccountRole, DOMAIN_MODEL_TX, DOMAIN_STATUS, type Ref, @@ -47,7 +46,6 @@ import tracker, { trackerId } from '@hcengineering/tracker' -import contact from '@hcengineering/model-contact' import { classicIssueTaskStatuses } from '.' async function createDefaultProject (tx: TxOperations): Promise { @@ -335,24 +333,6 @@ async function migrateDefaultTypeMixins (client: MigrationClient): Promise ) } -async function migrateDefaultProjectOwners (client: MigrationClient): Promise { - const workspaceOwners = await client.model.findAll(contact.class.PersonAccount, { - role: AccountRole.Owner - }) - - await client.update( - DOMAIN_SPACE, - { - _id: tracker.project.DefaultProject - }, - { - $set: { - owners: workspaceOwners.map((it) => it._id) - } - } - ) -} - async function migrateIssueStatuses (client: MigrationClient): Promise { await client.update( DOMAIN_MODEL_TX, @@ -424,10 +404,6 @@ export const trackerOperation: MigrateOperation = { { state: 'migrateDefaultTypeMixins', func: migrateDefaultTypeMixins - }, - { - state: 'migrateDefaultProjectOwners', - func: migrateDefaultProjectOwners } ]) }, diff --git a/models/tracker/src/types.ts b/models/tracker/src/types.ts index 92d8ba2d13..e90298eef6 100644 --- a/models/tracker/src/types.ts +++ b/models/tracker/src/types.ts @@ -29,7 +29,7 @@ import { type RolesAssignment, type Role, type CollectionSize, - Account + PersonId } from '@hcengineering/core' import { ArrOf, @@ -422,7 +422,7 @@ export class TProjectTargetPreference extends TPreference implements ProjectTarg @Mixin(tracker.mixin.ClassicProjectTypeData, tracker.class.Project) @UX(getEmbeddedLabel('Classic project'), tracker.icon.Issues) export class TClassicProjectTypeData extends TProject implements RolesAssignment { - [key: Ref]: Ref[] + [key: Ref]: PersonId[] } @Mixin(tracker.mixin.IssueTypeData, tracker.class.Issue) diff --git a/models/training/src/types.ts b/models/training/src/types.ts index 21a595e2b9..052ef8e895 100644 --- a/models/training/src/types.ts +++ b/models/training/src/types.ts @@ -37,7 +37,7 @@ import core, { type TypedSpace, RolesAssignment, Role, - Account + PersonId } from '@hcengineering/core' import { ArrOf, @@ -262,5 +262,5 @@ export class TTrainingAttempt extends TAttachedDoc implements TrainingAttempt { @Mixin(training.mixin.TrainingsTypeData, core.class.TypedSpace) @UX(getEmbeddedLabel('Default Trainings'), training.icon.TrainingApplication) export class TTrainingsTypeData extends TTypedSpace implements RolesAssignment { - [key: Ref]: Ref[] + [key: Ref]: PersonId[] } diff --git a/models/view/src/index.ts b/models/view/src/index.ts index e83609c617..762d608c05 100644 --- a/models/view/src/index.ts +++ b/models/view/src/index.ts @@ -14,7 +14,7 @@ // import { - type Account, + type PersonId, type Class, type Client, DOMAIN_MODEL, @@ -116,7 +116,7 @@ export class TFilteredView extends TDoc implements FilteredView { viewOptions?: ViewOptions filterClass?: Ref> viewletId?: Ref | null - users!: Ref[] + users!: PersonId[] attachedTo!: string sharable?: boolean } @@ -936,6 +936,10 @@ export function createModel (builder: Builder): void { group: 'bottom' }) + builder.mixin(core.class.TypePersonId, core.class.Class, view.mixin.AttributeFilter, { + component: view.component.ValueFilter + }) + builder.createDoc( view.class.FilterMode, core.space.Model, @@ -1172,6 +1176,10 @@ export function createModel (builder: Builder): void { presenter: view.component.StringFilterPresenter }) + builder.mixin(core.class.TypePersonId, core.class.Class, view.mixin.AttributeFilterPresenter, { + presenter: view.component.StringFilterPresenter + }) + classPresenter(builder, core.class.EnumOf, view.component.EnumPresenter, view.component.EnumEditor) createAction( @@ -1216,6 +1224,14 @@ export function createModel (builder: Builder): void { indexes: [], searchDisabled: true }) + + builder.mixin(core.class.TypePersonId, core.class.Class, view.mixin.AttributePresenter, { + presenter: view.component.PersonIdPresenter + }) + + builder.mixin(core.class.TypePersonId, core.class.Class, view.mixin.AttributeFilterPresenter, { + presenter: view.component.PersonIdFilterValuePresenter + }) } export default view diff --git a/models/view/src/migration.ts b/models/view/src/migration.ts index f5587c0547..7c7f9769ed 100644 --- a/models/view/src/migration.ts +++ b/models/view/src/migration.ts @@ -15,13 +15,18 @@ import { type MigrateOperation, + type MigrateUpdate, type MigrationClient, + type MigrationDocumentQuery, type MigrationUpgradeClient, tryMigrate } from '@hcengineering/model' import { DOMAIN_PREFERENCE } from '@hcengineering/preference' import view, { type Filter, type FilteredView, type ViewletPreference, viewId } from '@hcengineering/view' +import { getSocialIdByOldAccount } from '@hcengineering/model-core' + import { DOMAIN_VIEW } from '.' +import { MeasureMetricsContext } from '@hcengineering/core' async function removeDoneStatePref (client: MigrationClient): Promise { const prefs = await client.find(DOMAIN_PREFERENCE, { @@ -76,6 +81,51 @@ async function removeDoneStateFilter (client: MigrationClient): Promise { } } +async function migrateAccountsToSocialIds (client: MigrationClient): Promise { + const ctx = new MeasureMetricsContext('view migrateAccountsToSocialIds', {}) + const socialIdByAccount = await getSocialIdByOldAccount(client) + + ctx.info('processing view filtered view users ', {}) + const iterator = await client.traverse(DOMAIN_VIEW, { _class: view.class.FilteredView }) + + try { + let processed = 0 + while (true) { + const docs = await iterator.next(200) + if (docs === null || docs.length === 0) { + break + } + + const operations: { filter: MigrationDocumentQuery, update: MigrateUpdate }[] = [] + + for (const doc of docs) { + const filteredView = doc as FilteredView + + if (filteredView.users === undefined || filteredView.users.length === 0) continue + + const newUsers = filteredView.users.map((u) => socialIdByAccount[u] ?? u) + + operations.push({ + filter: { _id: filteredView._id }, + update: { + users: newUsers + } + }) + } + + if (operations.length > 0) { + await client.bulk(DOMAIN_VIEW, operations) + } + + processed += docs.length + ctx.info('...processed', { count: processed }) + } + } finally { + await iterator.close() + } + ctx.info('finished processing view filtered view users ', {}) +} + export const viewOperation: MigrateOperation = { async migrate (client: MigrationClient): Promise { await tryMigrate(client, viewId, [ @@ -86,6 +136,10 @@ export const viewOperation: MigrateOperation = { { state: 'remove-done-state-filter', func: removeDoneStateFilter + }, + { + state: 'accounts-to-social-ids', + func: migrateAccountsToSocialIds } ]) }, diff --git a/models/view/src/plugin.ts b/models/view/src/plugin.ts index e70704a668..9d0ffea67b 100644 --- a/models/view/src/plugin.ts +++ b/models/view/src/plugin.ts @@ -89,6 +89,7 @@ export default mergeIds(viewId, view, { EnumPresenter: '' as AnyComponent, StatusPresenter: '' as AnyComponent, StatusRefPresenter: '' as AnyComponent, + PersonIdFilterValuePresenter: '' as AnyComponent, DateFilterPresenter: '' as AnyComponent, StringFilterPresenter: '' as AnyComponent, AudioViewer: '' as AnyComponent, diff --git a/packages/account-client/.eslintrc.js b/packages/account-client/.eslintrc.js new file mode 100644 index 0000000000..72235dc283 --- /dev/null +++ b/packages/account-client/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.json' + } +} diff --git a/packages/account-client/.npmignore b/packages/account-client/.npmignore new file mode 100644 index 0000000000..e3ec093c38 --- /dev/null +++ b/packages/account-client/.npmignore @@ -0,0 +1,4 @@ +* +!/lib/** +!CHANGELOG.md +/lib/**/__tests__/ diff --git a/packages/account-client/config/rig.json b/packages/account-client/config/rig.json new file mode 100644 index 0000000000..0110930f55 --- /dev/null +++ b/packages/account-client/config/rig.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + "rigPackageName": "@hcengineering/platform-rig" +} diff --git a/packages/account-client/jest.config.js b/packages/account-client/jest.config.js new file mode 100644 index 0000000000..2cfd408b67 --- /dev/null +++ b/packages/account-client/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'], + roots: ["./src"], + coverageReporters: ["text-summary", "html"] +} diff --git a/packages/account-client/package.json b/packages/account-client/package.json new file mode 100644 index 0000000000..1b57faa8af --- /dev/null +++ b/packages/account-client/package.json @@ -0,0 +1,50 @@ +{ + "name": "@hcengineering/account-client", + "version": "0.6.0", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", + "files": [ + "lib/**/*", + "types/**/*", + "tsconfig.json" + ], + "author": "Hardcore Engineering Inc.", + "license": "EPL-2.0", + "scripts": { + "build": "compile", + "build:watch": "compile", + "format": "format src", + "test": "jest --passWithNoTests --silent", + "_phase:build": "compile transpile src", + "_phase:test": "jest --passWithNoTests --silent", + "_phase:format": "format src", + "_phase:validate": "compile validate" + }, + "devDependencies": { + "cross-env": "~7.0.3", + "@hcengineering/platform-rig": "^0.6.0", + "@types/node": "~20.11.16", + "@typescript-eslint/eslint-plugin": "^6.11.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-promise": "^6.1.1", + "eslint-plugin-n": "^15.4.0", + "eslint": "^8.54.0", + "esbuild": "^0.24.2", + "@typescript-eslint/parser": "^6.11.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "prettier": "^3.1.0", + "typescript": "^5.3.3", + "jest": "^29.7.0", + "ts-jest": "^29.1.1", + "@types/jest": "^29.5.5" + }, + "dependencies": { + "@hcengineering/core": "^0.6.32", + "@hcengineering/platform": "^0.6.11" + }, + "repository": "https://github.com/hcengineering/platform", + "publishConfig": { + "registry": "https://npm.pkg.github.com" + } +} diff --git a/packages/account-client/src/client.ts b/packages/account-client/src/client.ts new file mode 100644 index 0000000000..563cf03064 --- /dev/null +++ b/packages/account-client/src/client.ts @@ -0,0 +1,562 @@ +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { + type AccountRole, + BackupStatus, + Data, + type Person, + PersonUuid, + SocialId, + Version, + type WorkspaceInfoWithStatus, + type WorkspaceMemberInfo, + WorkspaceMode, + concatLink +} from '@hcengineering/core' +import platform, { PlatformError, Severity, Status } from '@hcengineering/platform' +import type { LoginInfo, OtpInfo, WorkspaceLoginInfo, RegionInfo, WorkspaceOperation } from './types' + +/** @public */ +export interface AccountClient { + // Static methods + getProviders: () => Promise + + // RPC + getUserWorkspaces: () => Promise + selectWorkspace: ( + workspaceUrl: string, + kind?: 'external' | 'internal' | 'byregion', + externalRegions?: string[] + ) => Promise + validateOtp: (email: string, code: string) => Promise + loginOtp: (email: string) => Promise + getLoginInfoByToken: () => Promise + restorePassword: (password: string) => Promise + confirm: () => Promise + requestPasswordReset: (email: string) => Promise + sendInvite: (email: string, role?: AccountRole) => Promise + leaveWorkspace: (account: string) => Promise + changeUsername: (first: string, last: string) => Promise + changePassword: (oldPassword: string, newPassword: string) => Promise + signUpJoin: ( + email: string, + password: string, + first: string, + last: string, + inviteId: string + ) => Promise + join: (email: string, password: string, inviteId: string) => Promise + createInviteLink: ( + exp: number, + emailMask: string, + limit: number, + role: AccountRole, + personId?: any + ) => Promise + checkJoin: (inviteId: string) => Promise + getWorkspaceInfo: (updateLastVisit?: boolean) => Promise + getRegionInfo: () => Promise + createWorkspace: (name: string, region?: string) => Promise + signUpOtp: (email: string, first: string, last: string) => Promise + signUp: (email: string, password: string, first: string, last: string) => Promise + login: (email: string, password: string) => Promise + getPerson: () => Promise + getSocialIds: () => Promise + getWorkspaceMembers: () => Promise + updateWorkspaceRole: (account: string, role: AccountRole) => Promise + updateWorkspaceName: (name: string) => Promise + deleteWorkspace: () => Promise + findPerson: (socialString: string) => Promise + + // Service methods + workerHandshake: (region: string, version: Data, operation: WorkspaceOperation) => Promise + getPendingWorkspace: ( + region: string, + version: Data, + operation: WorkspaceOperation + ) => Promise + updateWorkspaceInfo: ( + wsUuid: string, + event: string, + version: Data, + progress: number, + message?: string + ) => Promise + listWorkspaces: (region?: string | null, mode?: WorkspaceMode | null) => Promise + performWorkspaceOperation: ( + workspaceId: string | string[], + event: 'archive' | 'migrate-to' | 'unarchive' | 'delete', + ...params: any + ) => Promise + assignWorkspace: (email: string, workspaceUuid: string, role: AccountRole) => Promise + updateBackupInfo: (info: BackupStatus) => Promise + updateWorkspaceRoleBySocialId: (socialKey: string, targetRole: AccountRole) => Promise +} + +/** @public */ +export function getClient (accountsUrl?: string, token?: string): AccountClient { + if (accountsUrl === undefined) { + throw new Error('Accounts url not specified') + } + + return new AccountClientImpl(accountsUrl, token) +} + +interface Request { + method: string + params: any[] +} + +class AccountClientImpl implements AccountClient { + constructor ( + private readonly url: string, + private readonly token?: string + ) { + if (url === '') { + throw new Error('Accounts url not specified') + } + } + + async getProviders (): Promise { + return await retry(5, async () => { + const response = await fetch(concatLink(this.url, '/providers')) + + return await response.json() + }) + } + + private async rpc(request: Request): Promise { + const response = await fetch(this.url, { + method: 'POST', + keepalive: true, + headers: { + ...(this.token === undefined + ? {} + : { + Authorization: 'Bearer ' + this.token + }), + 'Content-Type': 'application/json' + }, + body: JSON.stringify(request) + }) + + const result = await response.json() + if (result.error != null) { + throw new PlatformError(result.error) + } + + return result.result + } + + private flattenStatus (ws: any): WorkspaceInfoWithStatus { + if (ws === undefined) { + throw new PlatformError(new Status(Severity.ERROR, platform.status.WorkspaceNotFound, {})) + } + + const status = ws.status + if (status === undefined) { + return ws + } + + const result = { ...ws, ...status } + delete result.status + + return result + } + + async getUserWorkspaces (): Promise { + const request = { + method: 'getUserWorkspaces' as const, + params: [] + } + + return (await this.rpc(request)).map((ws) => this.flattenStatus(ws)) + } + + async selectWorkspace ( + workspaceUrl: string, + kind: 'external' | 'internal' | 'byregion' = 'external', + externalRegions: string[] = [] + ): Promise { + const request = { + method: 'selectWorkspace' as const, + params: [workspaceUrl, kind, externalRegions] + } + + return await this.rpc(request) + } + + async validateOtp (email: string, code: string): Promise { + const request = { + method: 'validateOtp' as const, + params: [email, code] + } + + return await this.rpc(request) + } + + async loginOtp (email: string): Promise { + const request = { + method: 'loginOtp' as const, + params: [email] + } + + return await this.rpc(request) + } + + async getLoginInfoByToken (): Promise { + const request = { + method: 'getLoginInfoByToken' as const, + params: [] + } + + return await this.rpc(request) + } + + async restorePassword (password: string): Promise { + const request = { + method: 'restorePassword' as const, + params: [password] + } + + return await this.rpc(request) + } + + async confirm (): Promise { + const request = { + method: 'confirm' as const, + params: [] + } + + return await this.rpc(request) + } + + async requestPasswordReset (email: string): Promise { + const request = { + method: 'requestPasswordReset' as const, + params: [email] + } + + await this.rpc(request) + } + + async sendInvite (email: string, role?: AccountRole): Promise { + const request = { + method: 'sendInvite' as const, + params: [email, role] + } + + await this.rpc(request) + } + + async leaveWorkspace (account: string): Promise { + const request = { + method: 'leaveWorkspace' as const, + params: [account] + } + + return await this.rpc(request) + } + + async changeUsername (first: string, last: string): Promise { + const request = { + method: 'changeUsername' as const, + params: [first, last] + } + + await this.rpc(request) + } + + async changePassword (oldPassword: string, newPassword: string): Promise { + const request = { + method: 'changePassword' as const, + params: [oldPassword, newPassword] + } + + await this.rpc(request) + } + + async signUpJoin ( + email: string, + password: string, + first: string, + last: string, + inviteId: string + ): Promise { + const request = { + method: 'signUpJoin' as const, + params: [email, password, first, last, inviteId] + } + + return await this.rpc(request) + } + + async join (email: string, password: string, inviteId: string): Promise { + const request = { + method: 'join' as const, + params: [email, password, inviteId] + } + + return await this.rpc(request) + } + + async createInviteLink ( + exp: number, + emailMask: string, + limit: number, + role: AccountRole, + personId?: any + ): Promise { + const request = { + method: 'createInviteLink' as const, + params: [exp, emailMask, limit, role, personId] + } + + return await this.rpc(request) + } + + async checkJoin (inviteId: string): Promise { + const request = { + method: 'checkJoin' as const, + params: [inviteId] + } + + return await this.rpc(request) + } + + async getWorkspaceInfo (updateLastVisit: boolean = false): Promise { + const request = { + method: 'getWorkspaceInfo' as const, + params: updateLastVisit ? [true] : [] + } + + return this.flattenStatus(await this.rpc(request)) + } + + async getRegionInfo (): Promise { + const request = { + method: 'getRegionInfo' as const, + params: [] + } + + return await this.rpc(request) + } + + async createWorkspace (name: string, region?: string): Promise { + const request = { + method: 'createWorkspace' as const, + params: [name, region] + } + + return await this.rpc(request) + } + + async signUpOtp (email: string, first: string, last: string): Promise { + const request = { + method: 'signUpOtp' as const, + params: [email, first, last] + } + + return await this.rpc(request) + } + + async signUp (email: string, password: string, first: string, last: string): Promise { + const request = { + method: 'signUp' as const, + params: [email, password, first, last] + } + + return await this.rpc(request) + } + + async login (email: string, password: string): Promise { + const request = { + method: 'login' as const, + params: [email, password] + } + + return await this.rpc(request) + } + + async getPerson (): Promise { + const request = { + method: 'getPerson' as const, + params: [] + } + + return await this.rpc(request) + } + + async getSocialIds (): Promise { + const request = { + method: 'getSocialIds' as const, + params: [] + } + + return await this.rpc(request) + } + + async workerHandshake (region: string, version: Data, operation: WorkspaceOperation): Promise { + const request = { + method: 'workerHandshake' as const, + params: [region, version, operation] + } + + await this.rpc(request) + } + + async getPendingWorkspace ( + region: string, + version: Data, + operation: WorkspaceOperation + ): Promise { + const request = { + method: 'getPendingWorkspace' as const, + params: [region, version, operation] + } + + const result = await this.rpc(request) + if (result == null) { + return null + } + + return this.flattenStatus(result) + } + + async updateWorkspaceInfo ( + wsUuid: string, + event: string, + version: Data, + progress: number, + message?: string + ): Promise { + const request = { + method: 'updateWorkspaceInfo' as const, + params: [wsUuid, event, version, progress, message] + } + + await this.rpc(request) + } + + async getWorkspaceMembers (): Promise { + const request = { + method: 'getWorkspaceMembers' as const, + params: [] + } + + return await this.rpc(request) + } + + async updateWorkspaceRole (account: string, role: AccountRole): Promise { + const request = { + method: 'updateWorkspaceRole' as const, + params: [account, role] + } + + await this.rpc(request) + } + + async updateWorkspaceName (name: string): Promise { + const request = { + method: 'updateWorkspaceName' as const, + params: [name] + } + + await this.rpc(request) + } + + async deleteWorkspace (): Promise { + const request = { + method: 'deleteWorkspace' as const, + params: [] + } + + await this.rpc(request) + } + + async findPerson (socialString: string): Promise { + const request = { + method: 'findPerson' as const, + params: [socialString] + } + + return await this.rpc(request) + } + + async listWorkspaces (region?: string | null, mode: WorkspaceMode | null = null): Promise { + const request = { + method: 'listWorkspaces' as const, + params: [region, mode] + } + + return ((await this.rpc(request)) ?? []).map((ws) => this.flattenStatus(ws)) + } + + async performWorkspaceOperation ( + workspaceId: string | string[], + event: 'archive' | 'migrate-to' | 'unarchive' | 'delete', + ...params: any + ): Promise { + const request = { + method: 'performWorkspaceOperation' as const, + params: [workspaceId, event, ...params] + } + + return await this.rpc(request) + } + + async updateBackupInfo (info: BackupStatus): Promise { + const request = { + method: 'updateBackupInfo' as const, + params: [info] + } + + await this.rpc(request) + } + + async assignWorkspace (email: string, workspaceUuid: string, role: AccountRole): Promise { + const request = { + method: 'assignWorkspace' as const, + params: [email, workspaceUuid, role] + } + + await this.rpc(request) + } + + async updateWorkspaceRoleBySocialId (socialKey: string, targetRole: AccountRole): Promise { + const request = { + method: 'updateWorkspaceRoleBySocialId' as const, + params: [socialKey, targetRole] + } + + await this.rpc(request) + } +} + +async function retry (retries: number, op: () => Promise, delay: number = 100): Promise { + let error: any + while (retries > 0) { + retries-- + try { + return await op() + } catch (err: any) { + error = err + if (retries !== 0) { + await new Promise((resolve) => setTimeout(resolve, delay)) + } + } + } + throw error +} diff --git a/packages/account-client/src/index.ts b/packages/account-client/src/index.ts new file mode 100644 index 0000000000..b487cf8270 --- /dev/null +++ b/packages/account-client/src/index.ts @@ -0,0 +1,18 @@ +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +export * from './client' +export * from './types' +export * from './utils' diff --git a/packages/account-client/src/types.ts b/packages/account-client/src/types.ts new file mode 100644 index 0000000000..4dd45b92ad --- /dev/null +++ b/packages/account-client/src/types.ts @@ -0,0 +1,30 @@ +import { WorkspaceDataId, WorkspaceUuid, type AccountRole, type PersonUuid, type Timestamp } from '@hcengineering/core' + +export interface LoginInfo { + account: PersonUuid + token?: string +} + +/** + * @public + */ +export interface WorkspaceLoginInfo extends LoginInfo { + workspace: WorkspaceUuid // worspace uuid + workspaceDataId?: WorkspaceDataId + workspaceUrl: string + endpoint: string + token: string + role: AccountRole +} + +export interface OtpInfo { + sent: boolean + retryOn: Timestamp +} + +export interface RegionInfo { + region: string + name: string +} + +export type WorkspaceOperation = 'create' | 'upgrade' | 'all' | 'all+backup' diff --git a/services/analytics-collector/pod-analytics-collector/src/account.ts b/packages/account-client/src/utils.ts similarity index 50% rename from services/analytics-collector/pod-analytics-collector/src/account.ts rename to packages/account-client/src/utils.ts index 52877bd892..cbca030bcb 100644 --- a/services/analytics-collector/pod-analytics-collector/src/account.ts +++ b/packages/account-client/src/utils.ts @@ -13,26 +13,8 @@ // limitations under the License. // -import { WorkspaceInfo } from '@hcengineering/account' +import type { LoginInfo, WorkspaceLoginInfo } from './types' -import config from './config' - -export async function getWorkspaceInfo (token: string): Promise { - const accountsUrl = config.AccountsUrl - const workspaceInfo = await ( - await fetch(accountsUrl, { - method: 'POST', - keepalive: true, - headers: { - Authorization: 'Bearer ' + token, - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - method: 'getWorkspaceInfo', - params: [] - }) - }) - ).json() - - return workspaceInfo.result as WorkspaceInfo +export function isWorkspaceLoginInfo (loginInfo: LoginInfo | WorkspaceLoginInfo): loginInfo is WorkspaceLoginInfo { + return (loginInfo as WorkspaceLoginInfo).workspace != null } diff --git a/packages/account-client/tsconfig.json b/packages/account-client/tsconfig.json new file mode 100644 index 0000000000..59e4fd4297 --- /dev/null +++ b/packages/account-client/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json", + + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "declarationDir": "./types", + "tsBuildInfoFile": ".build/build.tsbuildinfo" + } +} \ No newline at end of file diff --git a/packages/api-client/package.json b/packages/api-client/package.json index 72f64b4422..52baa57bd7 100644 --- a/packages/api-client/package.json +++ b/packages/api-client/package.json @@ -46,6 +46,7 @@ "@hcengineering/client": "^0.6.18", "@hcengineering/client-resources": "^0.6.27", "@hcengineering/collaborator-client": "^0.6.4", + "@hcengineering/account-client": "^0.6.0", "@hcengineering/platform": "^0.6.11", "@hcengineering/text": "^0.6.5" }, diff --git a/packages/api-client/src/account.ts b/packages/api-client/src/account.ts deleted file mode 100644 index d32364eb6a..0000000000 --- a/packages/api-client/src/account.ts +++ /dev/null @@ -1,64 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -/** @public */ -export interface LoginInfo { - token: string - endpoint: string - confirmed: boolean - email: string -} - -/** @public */ -export interface WorkspaceLoginInfo extends LoginInfo { - workspace: string - workspaceId: string -} - -export async function login (accountsUrl: string, user: string, password: string, workspace: string): Promise { - const response = await fetch(accountsUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - method: 'login', - params: [user, password, workspace] - }) - }) - - const result = await response.json() - return result.result?.token -} - -export async function selectWorkspace ( - accountsUrl: string, - token: string, - workspace: string -): Promise { - const response = await fetch(accountsUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer ' + token - }, - body: JSON.stringify({ - method: 'selectWorkspace', - params: [workspace, 'external'] - }) - }) - const result = await response.json() - return result.result as WorkspaceLoginInfo -} diff --git a/packages/api-client/src/client.ts b/packages/api-client/src/client.ts index 5ad7fe45b9..2a0d969752 100644 --- a/packages/api-client/src/client.ts +++ b/packages/api-client/src/client.ts @@ -12,9 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // - import { - type Account, type Class, type Client, type Data, @@ -35,12 +33,15 @@ import { Mixin, MixinUpdate, MixinData, - generateId + generateId, + PersonId, + buildSocialIdString, + WorkspaceUuid } from '@hcengineering/core' import client, { clientId } from '@hcengineering/client' import { addLocation, getResource } from '@hcengineering/platform' +import { getClient as getAccountClient } from '@hcengineering/account-client' -import { login, selectWorkspace } from './account' import { type ServerConfig, loadServerConfig } from './config' import { type MarkupFormat, @@ -58,28 +59,42 @@ export async function connect (url: string, options: ConnectOptions): Promise buildSocialIdString(si)) + + if (socialStrings.length === 0) { + throw new Error('No social ids found for the logged in user') + } + + const wsLoginInfo = await accountClient.selectWorkspace(options.workspace) + + if (wsLoginInfo === undefined) { + throw new Error(`Workspace ${options.workspace} not found`) + } + + return await createClient(url, endpoint, token, wsLoginInfo.workspace, socialStrings[0], config, options) } async function createClient ( url: string, endpoint: string, token: string, + workspaceUuid: WorkspaceUuid, + user: PersonId, config: ServerConfig, options: ConnectOptions ): Promise { addLocation(clientId, () => import(/* webpackChunkName: "client" */ '@hcengineering/client-resources')) - const { workspace, socketFactory, connectionTimeout } = options + const { socketFactory, connectionTimeout } = options const clientFactory = await getResource(client.function.GetClient) const connection = await clientFactory(token, endpoint, { socketFactory, connectionTimeout }) - const account = await connection.getAccount() - return new PlatformClientImpl(url, workspace, token, config, connection, account) + return new PlatformClientImpl(url, workspaceUuid, token, config, connection, user) } class PlatformClientImpl implements PlatformClient { @@ -88,13 +103,13 @@ class PlatformClientImpl implements PlatformClient { constructor ( private readonly url: string, - private readonly workspace: string, + private readonly workspace: WorkspaceUuid, private readonly token: string, private readonly config: ServerConfig, private readonly connection: Client, - private readonly account: Account + private readonly user: PersonId ) { - this.client = new TxOperations(connection, account._id) + this.client = new TxOperations(connection, user) this.markup = createMarkupOperations(url, workspace, token, config) } @@ -280,20 +295,21 @@ async function getWorkspaceToken ( ): Promise<{ endpoint: string, token: string }> { config ??= await loadServerConfig(url) - let token: string + let token: string | undefined if ('token' in options) { token = options.token } else { - const { email, password, workspace } = options - token = await login(config.ACCOUNTS_URL, email, password, workspace) + const { email, password } = options + const loginInfo = await getAccountClient(config.ACCOUNTS_URL).login(email, password) + token = loginInfo.token } if (token === undefined) { throw new Error('Login failed') } - const ws = await selectWorkspace(config.ACCOUNTS_URL, token, options.workspace) + const ws = await getAccountClient(config.ACCOUNTS_URL, token).selectWorkspace(options.workspace) if (ws === undefined) { throw new Error('Workspace not found') } diff --git a/packages/api-client/src/markup/client.ts b/packages/api-client/src/markup/client.ts index 180e6fdfdf..a999429bc3 100644 --- a/packages/api-client/src/markup/client.ts +++ b/packages/api-client/src/markup/client.ts @@ -13,7 +13,15 @@ // limitations under the License. // -import { type Class, type Doc, type Markup, type Ref, concatLink, makeCollabId } from '@hcengineering/core' +import { + type Class, + type Doc, + type Markup, + type Ref, + WorkspaceUuid, + concatLink, + makeCollabId +} from '@hcengineering/core' import { type CollaboratorClient, getClient } from '@hcengineering/collaborator-client' import { parseMessageMarkdown, jsonToMarkup, markupToHTML, markupToMarkdown, htmlToMarkup } from '@hcengineering/text' @@ -22,7 +30,7 @@ import { type MarkupOperations, type MarkupFormat, type MarkupRef } from './type export function createMarkupOperations ( url: string, - workspace: string, + workspace: WorkspaceUuid, token: string, config: ServerConfig ): MarkupOperations { @@ -36,13 +44,13 @@ class MarkupOperationsImpl implements MarkupOperations { constructor ( private readonly url: string, - private readonly workspace: string, + private readonly workspace: WorkspaceUuid, private readonly token: string, private readonly config: ServerConfig ) { this.refUrl = concatLink(this.url, `/browse?workspace=${workspace}`) this.imageUrl = concatLink(this.url, `/files?workspace=${workspace}&file=`) - this.collaborator = getClient({ name: workspace }, token, config.COLLABORATOR_URL) + this.collaborator = getClient(workspace, token, config.COLLABORATOR_URL) } async fetchMarkup ( diff --git a/packages/collaborator-client/src/client.ts b/packages/collaborator-client/src/client.ts index b7d34156a4..8cabaf1285 100644 --- a/packages/collaborator-client/src/client.ts +++ b/packages/collaborator-client/src/client.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import { Blob, CollaborativeDoc, Markup, MarkupBlobRef, Ref, WorkspaceId, concatLink } from '@hcengineering/core' +import { Blob, CollaborativeDoc, Markup, MarkupBlobRef, Ref, WorkspaceUuid, concatLink } from '@hcengineering/core' import { encodeDocumentId } from './utils' /** @public */ @@ -54,20 +54,20 @@ export interface CollaboratorClient { } /** @public */ -export function getClient (workspaceId: WorkspaceId, token: string, collaboratorUrl: string): CollaboratorClient { +export function getClient (workspaceId: WorkspaceUuid, token: string, collaboratorUrl: string): CollaboratorClient { const url = collaboratorUrl.replaceAll('wss://', 'https://').replace('ws://', 'http://') return new CollaboratorClientImpl(workspaceId, token, url) } class CollaboratorClientImpl implements CollaboratorClient { constructor ( - private readonly workspace: WorkspaceId, + private readonly workspace: WorkspaceUuid, private readonly token: string, private readonly collaboratorUrl: string ) {} private async rpc(document: CollaborativeDoc, method: string, payload: P): Promise { - const workspace = this.workspace.name + const workspace = this.workspace const documentId = encodeDocumentId(workspace, document) const url = concatLink(this.collaboratorUrl, `/rpc/${encodeURIComponent(documentId)}`) diff --git a/packages/core/lang/cs.json b/packages/core/lang/cs.json index 7f31b6c05c..53f70d1301 100644 --- a/packages/core/lang/cs.json +++ b/packages/core/lang/cs.json @@ -65,6 +65,7 @@ "BlobContentType": "Typ obsahu", "Relation": "Vztah", "Relations": "Vztahy", - "AddRelation": "Přidat vztah" + "AddRelation": "Přidat vztah", + "PersonId": "Osoba" } } \ No newline at end of file diff --git a/packages/core/lang/en.json b/packages/core/lang/en.json index e188d8b834..d9cd134a8b 100644 --- a/packages/core/lang/en.json +++ b/packages/core/lang/en.json @@ -65,6 +65,7 @@ "BlobContentType": "Content type", "Relation": "Relation", "Relations": "Relations", - "AddRelation": "Add relation" + "AddRelation": "Add relation", + "PersonId": "Person" } } diff --git a/packages/core/lang/es.json b/packages/core/lang/es.json index f0e938b4eb..bdb702591d 100644 --- a/packages/core/lang/es.json +++ b/packages/core/lang/es.json @@ -58,6 +58,7 @@ "BlobContentType": "Tipo de contenido", "Relation": "Relación", "Relations": "Relaciones", - "AddRelation": "Añadir relación" + "AddRelation": "Añadir relación", + "PersonId": "Id. de persona" } } diff --git a/packages/core/lang/fr.json b/packages/core/lang/fr.json index 75e46f4827..51502b8bee 100644 --- a/packages/core/lang/fr.json +++ b/packages/core/lang/fr.json @@ -65,6 +65,7 @@ "BlobContentType": "Type de contenu", "Relation": "Relation", "Relations": "Relations", - "AddRelation": "Ajouter une relation" + "AddRelation": "Ajouter une relation", + "PersonId": "Id de personne" } } \ No newline at end of file diff --git a/packages/core/lang/it.json b/packages/core/lang/it.json index 98cfbd387d..a977dc4aa8 100644 --- a/packages/core/lang/it.json +++ b/packages/core/lang/it.json @@ -65,6 +65,7 @@ "BlobContentType": "Tipo di contenuto", "Relation": "Relazione", "Relations": "Relazioni", - "AddRelation": "Aggiungi relazione" + "AddRelation": "Aggiungi relazione", + "PersonId": "ID persona" } } diff --git a/packages/core/lang/pt.json b/packages/core/lang/pt.json index 6ec4e23b32..eb7d5613d8 100644 --- a/packages/core/lang/pt.json +++ b/packages/core/lang/pt.json @@ -58,6 +58,7 @@ "BlobContentType": "Tipo de conteúdo", "Relation": "Relação", "Relations": "Relações", - "AddRelation": "Adicionar relação" + "AddRelation": "Adicionar relação", + "PersonId": "ID de pessoa" } } diff --git a/packages/core/lang/ru.json b/packages/core/lang/ru.json index 5cc81db71f..dc4042229a 100644 --- a/packages/core/lang/ru.json +++ b/packages/core/lang/ru.json @@ -65,6 +65,7 @@ "BlobContentType": "Тип контента", "Relation": "Связь", "Relations": "Связи", - "AddRelation": "Добавить связь" + "AddRelation": "Добавить связь", + "PersonId": "Персона" } } diff --git a/packages/core/lang/zh.json b/packages/core/lang/zh.json index 76a332f8b7..2c3c6856f4 100644 --- a/packages/core/lang/zh.json +++ b/packages/core/lang/zh.json @@ -65,6 +65,7 @@ "BlobContentType": "內容類型", "Relation": "关系", "Relations": "关系", - "AddRelation": "添加关系" + "AddRelation": "添加关系", + "PersonId": "人员 ID" } } diff --git a/packages/core/src/__tests__/client.test.ts b/packages/core/src/__tests__/client.test.ts index dde583377a..a906d4aaef 100644 --- a/packages/core/src/__tests__/client.test.ts +++ b/packages/core/src/__tests__/client.test.ts @@ -15,7 +15,7 @@ // import { IntlString, Plugin } from '@hcengineering/platform' import { ClientConnectEvent, DocChunk } from '..' -import type { Account, Class, Data, Doc, Domain, PluginConfiguration, Ref, Timestamp } from '../classes' +import type { Class, Data, Doc, Domain, PluginConfiguration, Ref, Timestamp } from '../classes' import { ClassifierKind, DOMAIN_MODEL, Space } from '../classes' import { ClientConnection, createClient } from '../client' import core from '../component' @@ -153,10 +153,6 @@ describe('client', () => { return txes } - async getAccount (): Promise { - return null as unknown as Account - } - async sendForceClose (): Promise {} })() } diff --git a/packages/core/src/__tests__/connection.ts b/packages/core/src/__tests__/connection.ts index 84570e22cf..ceb91c17ec 100644 --- a/packages/core/src/__tests__/connection.ts +++ b/packages/core/src/__tests__/connection.ts @@ -14,7 +14,7 @@ // import { ClientConnectEvent, DocChunk } from '..' -import type { Account, Class, Doc, Domain, Ref, Timestamp } from '../classes' +import type { Class, Doc, Domain, Ref, Timestamp } from '../classes' import { ClientConnection } from '../client' import core from '../component' import { Hierarchy } from '../hierarchy' @@ -93,10 +93,6 @@ export async function connect (handler: (tx: Tx) => void): Promise { - return null as unknown as Account - } - async sendForceClose (): Promise {} } diff --git a/packages/core/src/__tests__/memdb.test.ts b/packages/core/src/__tests__/memdb.test.ts index 4e8bed31e4..7a31810d62 100644 --- a/packages/core/src/__tests__/memdb.test.ts +++ b/packages/core/src/__tests__/memdb.test.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import { AccountRole, Client } from '..' +import { Client } from '..' import type { Class, Doc, Obj, Ref } from '../classes' import core from '../component' import { Hierarchy } from '../hierarchy' @@ -29,7 +29,7 @@ import { SearchResult } from '../storage' import { Tx } from '../tx' -import { createDoc, deleteDoc, genMinModel, test, TestMixin, updateDoc } from './minmodel' +import { genMinModel, test, TestMixin } from './minmodel' const txes = genMinModel() @@ -224,26 +224,27 @@ describe('memdb', () => { expect(regex).toHaveLength(expectedLength) }) - it('should push to array', async () => { - const hierarchy = new Hierarchy() - for (const tx of txes) hierarchy.tx(tx) - const model = new TxOperations(new ClientModel(hierarchy), core.account.System) - for (const tx of txes) await model.tx(tx) - const space = await model.createDoc(core.class.Space, core.space.Model, { - name: 'name', - description: 'desc', - private: false, - members: [], - archived: false - }) - const account = await model.createDoc(core.class.Account, core.space.Model, { - email: 'email', - role: AccountRole.User - }) - await model.updateDoc(core.class.Space, core.space.Model, space, { $push: { members: account } }) - const txSpace = await model.findAll(core.class.Space, { _id: space }) - expect(txSpace[0].members).toEqual(expect.arrayContaining([account])) - }) + // TODO: fix this test + // it('should push to array', async () => { + // const hierarchy = new Hierarchy() + // for (const tx of txes) hierarchy.tx(tx) + // const model = new TxOperations(new ClientModel(hierarchy), core.account.System) + // for (const tx of txes) await model.tx(tx) + // const space = await model.createDoc(core.class.Space, core.space.Model, { + // name: 'name', + // description: 'desc', + // private: false, + // members: [], + // archived: false + // }) + // const account = await model.createDoc(core.class.Account, core.space.Model, { + // email: 'email', + // role: AccountRole.User + // }) + // await model.updateDoc(core.class.Space, core.space.Model, space, { $push: { members: account } }) + // const txSpace = await model.findAll(core.class.Space, { _id: space }) + // expect(txSpace[0].members).toEqual(expect.arrayContaining([account])) + // }) it('limit and sorting', async () => { const hierarchy = new Hierarchy() @@ -396,41 +397,4 @@ describe('memdb', () => { expect(e).toEqual(new Error('createDoc cannot be used for objects inherited from AttachedDoc')) } }) - - it('has correct accounts', async () => { - const modTxes = [...txes] - - modTxes.push( - createDoc(core.class.Account, { - email: 'system_admin', - role: AccountRole.Owner - }) - ) - - const system1Account = createDoc(core.class.Account, { - email: 'system1', - role: AccountRole.Maintainer - }) - modTxes.push(system1Account) - - const user1Account = createDoc(core.class.Account, { - email: 'user1', - role: AccountRole.User - }) - modTxes.push(user1Account) - - modTxes.push(updateDoc(core.class.Account, core.space.Model, system1Account.objectId, { email: 'user1' })) - - modTxes.push(deleteDoc(core.class.Account, core.space.Model, user1Account.objectId)) - - const { model } = await createModel(modTxes) - - expect(model.getAccountByEmail('system_admin')).not.toBeUndefined() - expect(model.getAccountByEmail('system_admin')?.role).toBe(AccountRole.Owner) - - expect(model.getAccountByEmail('system1')).toBeUndefined() - - expect(model.getAccountByEmail('user1')).not.toBeUndefined() - expect(model.getAccountByEmail('user1')?.role).toBe(AccountRole.Maintainer) - }) }) diff --git a/packages/core/src/__tests__/minmodel.ts b/packages/core/src/__tests__/minmodel.ts index 134fc778fd..d76002d2e5 100644 --- a/packages/core/src/__tests__/minmodel.ts +++ b/packages/core/src/__tests__/minmodel.ts @@ -135,14 +135,14 @@ export function genMinModel (): TxCUD[] { domain: DOMAIN_MODEL }) ) - txes.push( - createClass(core.class.Account, { - label: 'Account' as IntlString, - extends: core.class.Doc, - kind: ClassifierKind.CLASS, - domain: DOMAIN_MODEL - }) - ) + // txes.push( + // createClass(core.class.Account, { + // label: 'Account' as IntlString, + // extends: core.class.Doc, + // kind: ClassifierKind.CLASS, + // domain: DOMAIN_MODEL + // }) + // ) txes.push( createInterface(test.interface.WithState, { diff --git a/packages/core/src/__tests__/test.json b/packages/core/src/__tests__/test.json new file mode 100644 index 0000000000..fc22aa8e01 --- /dev/null +++ b/packages/core/src/__tests__/test.json @@ -0,0 +1,126 @@ +[ + { + "_class": "core:class:Class", + "_id": "core:class:Obj", + "createdBy": "core:account:System", + "createdOn": 1729970852414, + "kind": 0, + "label": "Obj", + "modifiedBy": "core:account:System", + "modifiedOn": 1729970852414, + "space": "core:space:Model" + }, + { + "_class": "core:class:Class", + "_id": "core:class:Doc", + "createdBy": "core:account:System", + "createdOn": 1729970852414, + "extends": "core:class:Obj", + "kind": 0, + "label": "Doc", + "modifiedBy": "core:account:System", + "modifiedOn": 1729970852414, + "space": "core:space:Model" + }, + { + "_class": "core:class:Class", + "_id": "core:class:AttachedDoc", + "createdBy": "core:account:System", + "createdOn": 1729970852414, + "extends": "core:class:Doc", + "kind": 2, + "label": "AttachedDoc", + "modifiedBy": "core:account:System", + "modifiedOn": 1729970852414, + "space": "core:space:Model" + }, + { + "_class": "core:class:Class", + "_id": "core:class:Class", + "createdBy": "core:account:System", + "createdOn": 1729970852414, + "domain": "model", + "extends": "core:class:Doc", + "kind": 0, + "label": "Class", + "modifiedBy": "core:account:System", + "modifiedOn": 1729970852414, + "space": "core:space:Model" + }, + { + "_class": "core:class:Class", + "_id": "core:class:Interface", + "createdBy": "core:account:System", + "createdOn": 1729970852414, + "extends": "core:class:Doc", + "kind": 0, + "label": "Interface", + "modifiedBy": "core:account:System", + "modifiedOn": 1729970852414, + "space": "core:space:Model" + }, + { + "_class": "core:class:Class", + "_id": "core:class:Space", + "createdBy": "core:account:System", + "createdOn": 1729970852414, + "domain": "model", + "extends": "core:class:Doc", + "kind": 0, + "label": "Space", + "modifiedBy": "core:account:System", + "modifiedOn": 1729970852414, + "space": "core:space:Model" + }, + { + "_class": "core:class:Class", + "_id": "core:class:Account", + "createdBy": "core:account:System", + "createdOn": 1729970852414, + "domain": "model", + "extends": "core:class:Doc", + "kind": 0, + "label": "Account", + "modifiedBy": "core:account:System", + "modifiedOn": 1729970852414, + "space": "core:space:Model" + }, + { + "_class": "core:class:Interface", + "_id": "test:interface:WithState", + "createdBy": "core:account:System", + "createdOn": 1729970852414, + "extends": [], + "kind": 1, + "label": "WithState", + "modifiedBy": "core:account:System", + "modifiedOn": 1729970852414, + "space": "core:space:Model" + }, + { + "_class": "core:class:Class", + "_id": "core:class:Tx", + "createdBy": "core:account:System", + "createdOn": 1729970852414, + "domain": "tx", + "extends": "core:class:Doc", + "kind": 0, + "label": "Tx", + "modifiedBy": "core:account:System", + "modifiedOn": 1729970852414, + "space": "core:space:Model" + }, + { + "_class": "core:class:Class", + "_id": "core:class:TxCUD", + "createdBy": "core:account:System", + "createdOn": 1729970852414, + "domain": "tx", + "extends": "core:class:Tx", + "kind": 0, + "label": "TxCUD", + "modifiedBy": "core:account:System", + "modifiedOn": 1729970852414, + "space": "core:space:Model" + } +] diff --git a/packages/core/src/classes.ts b/packages/core/src/classes.ts index 766994966d..393952e9c9 100644 --- a/packages/core/src/classes.ts +++ b/packages/core/src/classes.ts @@ -16,6 +16,7 @@ import type { Asset, IntlString, Plugin } from '@hcengineering/platform' import type { DocumentQuery } from './storage' +import { WorkspaceDataId, WorkspaceUuid } from './utils' /** * @public @@ -68,6 +69,31 @@ export interface Obj { _class: Ref> } +export interface Account { + uuid: PersonUuid + role: AccountRole + primarySocialId: PersonId + socialIds: PersonId[] +} + +/** + * @public + * Global person UUID. + */ +export type PersonUuid = string & { __personUuid: true } + +/** + * @public + * String representation of a social id linked to a global person. + * E.g. email:pied.piper@hcengineering.com or huly:ea3bf257-94b5-4a31-a7da-466d578d850f + */ +export type PersonId = string & { __personId: true } + +export interface BasePerson { + name: string + personUuid?: PersonUuid +} + /** * @public */ @@ -75,8 +101,8 @@ export interface Doc extends Obj { _id: Ref space: Ref modifiedOn: Timestamp - modifiedBy: Ref - createdBy?: Ref // Marked as optional since it will be filled by platform. + modifiedBy: PersonId + createdBy?: PersonId // Marked as optional since it will be filled by platform. createdOn?: Timestamp // Marked as optional since it will be filled by platform. } @@ -405,9 +431,9 @@ export interface Space extends Doc { name: string description: string private: boolean - members: Arr> + members: Arr archived: boolean - owners?: Ref[] + owners?: PersonId[] autoJoin?: boolean } @@ -448,7 +474,7 @@ export interface SpaceType extends Doc { name: string shortDescription?: string descriptor: Ref - members?: Ref[] // this members will be added automatically to new space, also change this fiield will affect existing spaces + members?: PersonId[] // this members will be added automatically to new space, also change this fiield will affect existing spaces autoJoin?: boolean // if true, all new users will be added to space automatically targetClass: Ref> // A dynamic mixin for Spaces to hold custom attributes and roles assignment of the space type roles: CollectionSize @@ -467,7 +493,7 @@ export interface Role extends AttachedDoc { * @public * Defines assignment of employees to a role within a space */ -export type RolesAssignment = Record, Ref[] | undefined> +export type RolesAssignment = Record, PersonId[] | undefined> /** * @public @@ -479,16 +505,6 @@ export interface Permission extends Doc { icon?: Asset } -/** - * @public - */ -export interface Account extends Doc { - email: string - role: AccountRole - - person?: Ref -} - /** * @public */ @@ -504,19 +520,31 @@ export enum AccountRole { * @public */ export const roleOrder: Record = { - [AccountRole.DocGuest]: 0, - [AccountRole.Guest]: 1, - [AccountRole.User]: 2, - [AccountRole.Maintainer]: 3, - [AccountRole.Owner]: 4 + [AccountRole.DocGuest]: 10, + [AccountRole.Guest]: 20, + [AccountRole.User]: 30, + [AccountRole.Maintainer]: 40, + [AccountRole.Owner]: 50 } /** * @public */ +export interface Person { + uuid: string + firstName: string + lastName: string + country?: string + city?: string +} + +/** + * @public + */ +// TODO: move to contact export interface UserStatus extends Doc { online: boolean - user: Ref + user: PersonUuid } /** @@ -758,6 +786,18 @@ export type WorkspaceUpdateEvent = | 'delete-started' | 'delete-done' +export interface WorkspaceInfo { + uuid: WorkspaceUuid + dataId?: WorkspaceDataId // Old workspace identifier. E.g. Database name in Mongo, bucket in R2, etc. + name: string + url: string + region?: string + branding?: string + createdOn: number + createdBy: PersonUuid + billingAccount: PersonUuid +} + export interface BackupStatus { dataSize: number blobsSize: number @@ -768,26 +808,35 @@ export interface BackupStatus { backups: number } -export interface BaseWorkspaceInfo { - workspace: string // An uniq workspace name, Database names - uuid?: string // An uuid for a workspace to be used already for cockroach data - - disabled?: boolean +export interface WorkspaceInfoWithStatus extends WorkspaceInfo { + isDisabled?: boolean version?: Data - branding?: string - - workspaceUrl?: string | null // An optional url to the workspace, if not set workspace will be used - workspaceName?: string // An displayed workspace name - createdOn: number - lastVisit: number - createdBy: string + lastVisit?: number mode: WorkspaceMode - progress?: number // Some progress - - endpoint: string - - region?: string // Transactor group name - targetRegion?: string // Transactor region to move to - + processingProgress?: number backupInfo?: BackupStatus } + +export interface WorkspaceMemberInfo { + person: string + role: AccountRole +} + +export enum SocialIdType { + EMAIL = 'email', + GITHUB = 'github', + GOOGLE = 'google', + PHONE = 'phone', + OIDC = 'oidc', + HULY = 'huly', + TELEGRAM = 'telegram' +} +export interface SocialId { + id: string + type: SocialIdType + value: string + key: PersonId + verifiedOn?: number +} + +export type SocialKey = Pick diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index f0c66ecffd..c3d74131a2 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -15,7 +15,7 @@ import { Analytics } from '@hcengineering/analytics' import { BackupClient, DocChunk } from './backup' -import { Account, Class, DOMAIN_MODEL, Doc, Domain, Ref, Timestamp } from './classes' +import { Class, DOMAIN_MODEL, Doc, Domain, Ref, Timestamp } from './classes' import core from './component' import { Hierarchy } from './hierarchy' import { MeasureContext, MeasureMetricsContext } from './measurements' @@ -45,13 +45,6 @@ export interface Client extends Storage, FulltextStorage { close: () => Promise } -/** - * @public - */ -export interface AccountClient extends Client { - getAccount: () => Promise -} - /** * @public */ @@ -87,12 +80,11 @@ export interface ClientConnection extends Storage, FulltextStorage, BackupClient // If hash is passed, will return LoadModelResponse loadModel: (last: Timestamp, hash?: string) => Promise - getAccount: () => Promise getLastHash?: (ctx: MeasureContext) => Promise } -class ClientImpl implements AccountClient, BackupClient { +class ClientImpl implements Client, BackupClient { notify?: (...tx: Tx[]) => void hierarchy!: Hierarchy model!: ModelDb @@ -198,10 +190,6 @@ class ClientImpl implements AccountClient, BackupClient { await this.conn.clean(domain, docs) } - async getAccount (): Promise { - return await this.conn.getAccount() - } - async sendForceClose (): Promise { await this.conn.sendForceClose() } @@ -226,7 +214,7 @@ export async function createClient ( modelFilter?: ModelFilter, txPersistence?: TxPersistenceStore, _ctx?: MeasureContext -): Promise { +): Promise { const ctx = _ctx ?? new MeasureMetricsContext('createClient', {}) let client: ClientImpl | null = null @@ -337,6 +325,7 @@ export async function createClient ( } // Ignore Employee accounts. +// We may still have them in transactions in old workspaces even with global accounts. function isPersonAccount (tx: Tx): boolean { return ( (tx._class === core.class.TxCreateDoc || diff --git a/packages/core/src/component.ts b/packages/core/src/component.ts index db60627c65..89066b7942 100644 --- a/packages/core/src/component.ts +++ b/packages/core/src/component.ts @@ -14,8 +14,8 @@ // import type { Asset, IntlString, Plugin, StatusCode } from '@hcengineering/platform' import { plugin } from '@hcengineering/platform' -import { Mixin, Version, type Rank } from '.' import type { BenchmarkDoc } from './benchmark' +import { AccountRole, Mixin, Version, type Rank } from '.' import type { Account, AnyAttribute, @@ -40,6 +40,7 @@ import type { MigrationState, Obj, Permission, + PersonId, PluginConfiguration, Ref, RefTo, @@ -56,7 +57,8 @@ import type { Type, TypeAny, TypedSpace, - UserStatus + UserStatus, + PersonUuid } from './classes' import { Status, StatusCategory } from './status' import type { @@ -79,7 +81,15 @@ export const coreId = 'core' as Plugin /** * @public */ +// TODO: consider removing email? export const systemAccountEmail = 'anticrm@hc.engineering' +export const systemAccountUuid = '1749089e-22e6-48de-af4e-165e18fbd2f9' as PersonUuid +export const systemAccount: Account = { + uuid: systemAccountUuid, + role: AccountRole.Owner, + primarySocialId: '' as PersonId, + socialIds: [] +} export default plugin(coreId, { class: { @@ -107,7 +117,6 @@ export default plugin(coreId, { SpaceType: '' as Ref>, Role: '' as Ref>, Permission: '' as Ref>, - Account: '' as Ref>, Type: '' as Ref>>, TypeRelation: '' as Ref>>, TypeString: '' as Ref>>, @@ -123,6 +132,7 @@ export default plugin(coreId, { TypeTimestamp: '' as Ref>>, TypeDate: '' as Ref>>, TypeCollaborativeDoc: '' as Ref>>, + TypePersonId: '' as Ref>>, RefTo: '' as Ref>>, ArrOf: '' as Ref>>, Enum: '' as Ref>, @@ -178,8 +188,8 @@ export default plugin(coreId, { Workspace: '' as Ref }, account: { - System: '' as Ref, - ConfigUser: '' as Ref + System: '' as PersonId, + ConfigUser: '' as PersonId }, status: { ObjectNotFound: '' as StatusCode<{ _id: Ref }>, @@ -210,6 +220,7 @@ export default plugin(coreId, { Collaborative: '' as IntlString, CollaborativeDoc: '' as IntlString, MarkupBlobRef: '' as IntlString, + PersonId: '' as IntlString, Number: '' as IntlString, Boolean: '' as IntlString, Timestamp: '' as IntlString, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 2677cb35d4..7a7df3bf7b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -16,7 +16,7 @@ export * from './classes' export * from './client' export * from './collaboration' -export { coreId, systemAccountEmail, default } from './component' +export { coreId, systemAccountUuid, systemAccount, default } from './component' export * from './hierarchy' export * from './measurements' export * from './memdb' diff --git a/packages/core/src/memdb.ts b/packages/core/src/memdb.ts index 79314d6e84..3980de5764 100644 --- a/packages/core/src/memdb.ts +++ b/packages/core/src/memdb.ts @@ -15,7 +15,8 @@ import { PlatformError, Severity, Status } from '@hcengineering/platform' import { Lookup, MeasureContext, ReverseLookups, getObjectValue } from '.' -import type { Account, Class, Doc, Ref } from './classes' +import type { Class, Doc, Ref } from './classes' + import core from './component' import { Hierarchy } from './hierarchy' import { checkMixinKey, matchQuery, resultSort } from './query' @@ -40,9 +41,6 @@ export abstract class MemDb extends TxProcessor implements Storage { private readonly objectsByClass = new Map>, Map, Doc>>() private readonly objectById = new Map, Doc>() - private readonly accountByPersonId = new Map, Account[]>() - private readonly accountByEmail = new Map() - constructor (protected readonly hierarchy: Hierarchy) { super() } @@ -87,21 +85,6 @@ export abstract class MemDb extends TxProcessor implements Storage { return doc as T } - getAccountByPersonId (ref: Ref): Account[] { - return this.accountByPersonId.get(ref) ?? [] - } - - getAccountByEmail (email: Account['email']): Account | undefined { - const accounts = this.accountByEmail.get(email) - if (accounts === undefined || accounts.length === 0) { - return undefined - } - - if (accounts.length > 0) { - return accounts[accounts.length - 1][1] - } - } - findObject(_id: Ref): T | undefined { const doc = this.objectById.get(_id) return doc as T @@ -274,44 +257,15 @@ export abstract class MemDb extends TxProcessor implements Storage { ) } - addAccount (account: Account): void { - if (!this.accountByEmail.has(account.email)) { - this.accountByEmail.set(account.email, []) - } - - this.accountByEmail.get(account.email)?.push([account._id, account]) - } - addDoc (doc: Doc): void { this.hierarchy.getAncestors(doc._class).forEach((_class) => { const arr = this.getObjectsByClass(_class) arr.set(doc._id, doc) }) - if (this.hierarchy.isDerived(doc._class, core.class.Account)) { - const account = doc as Account - this.addAccount(account) - - if (account.person !== undefined) { - this.accountByPersonId.set(account.person, [...(this.accountByPersonId.get(account.person) ?? []), account]) - } - } this.objectById.set(doc._id, doc) } - delAccount (account: Account): void { - const accounts = this.accountByEmail.get(account.email) - if (accounts !== undefined) { - const newAccounts = accounts.filter((it) => it[0] !== account._id) - - if (newAccounts.length === 0) { - this.accountByEmail.delete(account.email) - } else { - this.accountByEmail.set(account.email, newAccounts) - } - } - } - delDoc (_id: Ref): void { const doc = this.objectById.get(_id) if (doc === undefined) { @@ -321,42 +275,10 @@ export abstract class MemDb extends TxProcessor implements Storage { this.hierarchy.getAncestors(doc._class).forEach((_class) => { this.cleanObjectByClass(_class, _id) }) - if (this.hierarchy.isDerived(doc._class, core.class.Account)) { - const account = doc as Account - this.delAccount(account) - - if (account.person !== undefined) { - const acc = this.accountByPersonId.get(account.person) ?? [] - this.accountByPersonId.set( - account.person, - acc.filter((it) => it._id !== _id) - ) - } - } } updateDoc (_id: Ref, doc: Doc, update: TxUpdateDoc | TxMixin): void { - if (this.hierarchy.isDerived(doc._class, core.class.Account) && update._class === core.class.TxUpdateDoc) { - const newEmail = (update as TxUpdateDoc).operations.email - if ((update as TxUpdateDoc).operations.person !== undefined) { - const account = doc as Account - if (account.person !== undefined) { - const acc = this.accountByPersonId.get(account.person) ?? [] - this.accountByPersonId.set( - account.person, - acc.filter((it) => it._id !== _id) - ) - } - const newPerson = (update as TxUpdateDoc).operations.person - if (newPerson !== undefined) { - this.accountByPersonId.set(newPerson, [...(this.accountByPersonId.get(newPerson) ?? []), account]) - } - } else if (newEmail !== undefined) { - const account = doc as Account - this.delAccount(account) - this.addAccount({ ...account, email: newEmail }) - } - } + // TODO: track updates on Contact to adjust memdb accounts? } } diff --git a/packages/core/src/operations.ts b/packages/core/src/operations.ts index 725e87c322..6557986ed0 100644 --- a/packages/core/src/operations.ts +++ b/packages/core/src/operations.ts @@ -2,7 +2,7 @@ import { Analytics } from '@hcengineering/analytics' import { deepEqual } from 'fast-equals' import { DocumentUpdate, DOMAIN_MODEL, Hierarchy, MixinData, MixinUpdate, ModelDb, toFindResult } from '.' import type { - Account, + PersonId, AnyAttribute, AttachedData, AttachedDoc, @@ -40,7 +40,7 @@ export class TxOperations implements Omit { constructor ( readonly client: Client, - readonly user: Ref, + readonly user: PersonId, readonly isDerived: boolean = false ) { this.txFactory = new TxFactory(user, isDerived) @@ -88,7 +88,7 @@ export class TxOperations implements Omit { attributes: Data, id?: Ref, modifiedOn?: Timestamp, - modifiedBy?: Ref + modifiedBy?: PersonId ): Promise> { const hierarchy = this.client.getHierarchy() if (hierarchy.isDerived(_class, core.class.AttachedDoc)) { @@ -111,7 +111,7 @@ export class TxOperations implements Omit { attributes: AttachedData

, id?: Ref

, modifiedOn?: Timestamp, - modifiedBy?: Ref + modifiedBy?: PersonId ): Promise> { const tx = this.txFactory.createTxCollectionCUD( attachedToClass, @@ -136,7 +136,7 @@ export class TxOperations implements Omit { operations: DocumentUpdate

, retrieve?: boolean, modifiedOn?: Timestamp, - modifiedBy?: Ref + modifiedBy?: PersonId ): Promise> { const tx = this.txFactory.createTxCollectionCUD( attachedToClass, @@ -159,7 +159,7 @@ export class TxOperations implements Omit { attachedToClass: Ref>, collection: Extract | string, modifiedOn?: Timestamp, - modifiedBy?: Ref + modifiedBy?: PersonId ): Promise> { const tx = this.txFactory.createTxCollectionCUD( attachedToClass, @@ -181,7 +181,7 @@ export class TxOperations implements Omit { operations: DocumentUpdate, retrieve?: boolean, modifiedOn?: Timestamp, - modifiedBy?: Ref + modifiedBy?: PersonId ): Promise { const tx = this.txFactory.createTxUpdateDoc(_class, space, objectId, operations, retrieve, modifiedOn, modifiedBy) return this.tx(tx) @@ -192,7 +192,7 @@ export class TxOperations implements Omit { space: Ref, objectId: Ref, modifiedOn?: Timestamp, - modifiedBy?: Ref + modifiedBy?: PersonId ): Promise { const tx = this.txFactory.createTxRemoveDoc(_class, space, objectId, modifiedOn, modifiedBy) return this.tx(tx) @@ -205,7 +205,7 @@ export class TxOperations implements Omit { mixin: Ref>, attributes: MixinData, modifiedOn?: Timestamp, - modifiedBy?: Ref + modifiedBy?: PersonId ): Promise { const tx = this.txFactory.createTxMixin( objectId, @@ -226,7 +226,7 @@ export class TxOperations implements Omit { mixin: Ref>, attributes: MixinUpdate, modifiedOn?: Timestamp, - modifiedBy?: Ref + modifiedBy?: PersonId ): Promise { const tx = this.txFactory.createTxMixin( objectId, @@ -245,7 +245,7 @@ export class TxOperations implements Omit { update: DocumentUpdate, retrieve?: boolean, modifiedOn?: Timestamp, - modifiedBy?: Ref + modifiedBy?: PersonId ): Promise { const hierarchy = this.client.getHierarchy() const mixClass = Hierarchy.mixinOrClass(doc) @@ -296,7 +296,7 @@ export class TxOperations implements Omit { return await this.updateDoc(doc._class, doc.space, doc._id, update, retrieve, modifiedOn, modifiedBy) } - remove(doc: T, modifiedOn?: Timestamp, modifiedBy?: Ref): Promise { + remove(doc: T, modifiedOn?: Timestamp, modifiedBy?: PersonId): Promise { if (this.client.getHierarchy().isDerived(doc._class, core.class.AttachedDoc)) { const adoc = doc as unknown as AttachedDoc return this.removeCollection( @@ -321,7 +321,7 @@ export class TxOperations implements Omit { doc: T, update: T | Data | DocumentUpdate, date?: Timestamp, - account?: Ref + account?: PersonId ): Promise { // We need to update fields if they are different. const documentUpdate: DocumentUpdate = {} @@ -345,7 +345,7 @@ export class TxOperations implements Omit { doc: Doc, raw: Doc | Data, mixin: Ref>>, - modifiedBy: Ref, + modifiedBy: PersonId, modifiedOn: Timestamp ): Promise { // We need to update fields if they are different. @@ -536,7 +536,7 @@ export class TxBuilder extends TxOperations { constructor ( readonly hierarchy: Hierarchy, readonly modelDb: ModelDb, - user: Ref + user: PersonId ) { const txClient: Client = { getHierarchy: () => this.hierarchy, diff --git a/packages/core/src/server.ts b/packages/core/src/server.ts index 5a482642d4..52f9df1955 100644 --- a/packages/core/src/server.ts +++ b/packages/core/src/server.ts @@ -13,11 +13,11 @@ // limitations under the License. // -import type { Account, Doc, DocIndexState, Domain, Ref } from './classes' +import type { Account, Doc, DocIndexState, Domain, PersonId, PersonUuid, Ref } from './classes' import { MeasureContext } from './measurements' import { DocumentQuery, FindOptions } from './storage' import type { DocumentUpdate, Tx } from './tx' -import type { WorkspaceIdWithUrl } from './utils' +import { WorkspaceIds } from './utils' /** * @public @@ -45,20 +45,13 @@ export interface SessionData { } contextCache: Map removedMap: Map, Doc> - - userEmail: string + account: Account sessionId: string admin?: boolean - isTriggerCtx?: boolean - - account: Account - - getAccount: (account: Ref) => Account | undefined - - workspace: WorkspaceIdWithUrl + workspace: WorkspaceIds branding: Branding | null - + socialStringsToUsers: Map fulltextUpdates?: Map, DocIndexState> asyncRequests?: (() => Promise)[] diff --git a/packages/core/src/tx.ts b/packages/core/src/tx.ts index 0df5846d4f..63ed17bfc4 100644 --- a/packages/core/src/tx.ts +++ b/packages/core/src/tx.ts @@ -15,7 +15,7 @@ import type { KeysByType } from 'simplytyped' import type { - Account, + PersonId, Arr, AttachedDoc, Class, @@ -452,7 +452,7 @@ export abstract class TxProcessor implements WithTx { export class TxFactory { private readonly txSpace: Ref constructor ( - readonly account: Ref, + readonly account: PersonId, readonly isDerived: boolean = false ) { this.txSpace = isDerived ? core.space.DerivedTx : core.space.Tx @@ -464,7 +464,7 @@ export class TxFactory { attributes: Data, objectId?: Ref, modifiedOn?: Timestamp, - modifiedBy?: Ref + modifiedBy?: PersonId ): TxCreateDoc { return { _id: generateId(), @@ -487,7 +487,7 @@ export class TxFactory { collection: string, tx: TxCUD

, modifiedOn?: Timestamp, - modifiedBy?: Ref + modifiedBy?: PersonId ): TxCUD

{ return { ...tx, @@ -506,7 +506,7 @@ export class TxFactory { operations: DocumentUpdate, retrieve?: boolean, modifiedOn?: Timestamp, - modifiedBy?: Ref + modifiedBy?: PersonId ): TxUpdateDoc { return { _id: generateId(), @@ -527,7 +527,7 @@ export class TxFactory { space: Ref, objectId: Ref, modifiedOn?: Timestamp, - modifiedBy?: Ref + modifiedBy?: PersonId ): TxRemoveDoc { return { _id: generateId(), @@ -548,7 +548,7 @@ export class TxFactory { mixin: Ref>, attributes: MixinUpdate, modifiedOn?: Timestamp, - modifiedBy?: Ref + modifiedBy?: PersonId ): TxMixin { return { _id: generateId(), @@ -574,7 +574,7 @@ export class TxFactory { notify: boolean = true, extraNotify: Ref>[] = [], modifiedOn?: Timestamp, - modifiedBy?: Ref + modifiedBy?: PersonId ): TxApplyIf { return { _id: generateId(), diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index 022aefd8fb..1e322c4c1d 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -34,9 +34,13 @@ import { IndexKind, Obj, Permission, + PersonId, Ref, Role, roleOrder, + SocialId, + SocialIdType, + SocialKey, Space, TypedSpace, WorkspaceMode, @@ -80,6 +84,11 @@ export function generateId (join: string = ''): Ref { return (timestamp() + join + random + join + count()) as Ref } +export function generateUuid (): string { + // Consider own implementation if it will be slow + return crypto.randomUUID() +} + /** @public */ export function isId (value: any): value is Ref { return typeof value === 'string' && /^[0-9a-f]{24,24}$/.test(value) @@ -120,39 +129,12 @@ export function toFindResult (docs: T[], total?: number, lookupMa return Object.assign(docs, { total: length, lookupMap }) } -/** - * @public - */ -export interface WorkspaceId { - name: string - uuid?: string -} - -/** - * @public - */ -export interface WorkspaceIdWithUrl extends WorkspaceId { - workspaceUrl: string - workspaceName: string -} - -/** - * @public - * - * Previously was combining workspace with productId, if not equal '' - * Now just returning workspace as is. Keeping it to simplify further refactoring of ws id. - */ -export function getWorkspaceId (workspace: string): WorkspaceId { - return { - name: workspace - } -} - -/** - * @public - */ -export function toWorkspaceString (id: WorkspaceId): string { - return id.name +export type WorkspaceUuid = string & { __workspaceUuid: true } +export type WorkspaceDataId = string & { __workspaceDataId: true } +export interface WorkspaceIds { + uuid: WorkspaceUuid + url: string + dataId?: WorkspaceDataId // Old workspace identifier. E.g. Database name in Mongo, bucket in R2, etc. } /** @@ -555,6 +537,14 @@ export function cutObjectArray (obj: any): any { return r } +export function includesAny (arr1: string[] | null | undefined, arr2: string[] | null | undefined): boolean { + if (arr1 == null || arr1.length === 0 || arr2 == null || arr2.length === 0) { + return false + } + + return arr1.some((m) => arr2.includes(m)) +} + export const isEnum = (e: T) => (token: any): token is T[keyof T] => { @@ -578,7 +568,7 @@ export async function checkPermission ( const me = getCurrentAccount() const asMixin = client.getHierarchy().as(space, mixin) - const myRoles = type.$lookup?.roles?.filter((role) => (asMixin as any)[role._id]?.includes(me._id)) as Role[] + const myRoles = type.$lookup?.roles?.filter((role) => includesAny((asMixin as any)[role._id], me.socialIds)) as Role[] if (myRoles === undefined) { return false @@ -843,7 +833,6 @@ export function pluginFilterTx ( /** * @public */ - export class TimeRateLimiter { idCounter: number = 0 active: number = 0 @@ -917,3 +906,25 @@ export function combineAttributes ( ) ).filter((v) => v != null) } + +export function buildSocialIdString (key: SocialKey): PersonId { + return `${key.type}:${key.value}` as PersonId +} + +export function parseSocialIdString (id: PersonId): SocialKey { + const [type, value] = id.split(':') + + return { type: type as SocialIdType, value } +} + +export function pickPrimarySocialId (socialIds: SocialId[]): SocialId { + if (socialIds.length === 0) { + throw new Error('No social ids provided') + } + + return socialIds[0] +} + +export function notEmpty (id: T | undefined | null): id is T { + return id !== undefined && id !== null && id !== '' +} diff --git a/packages/importer/src/clickup/clickup.ts b/packages/importer/src/clickup/clickup.ts index 431c759bfd..7f0cb6aa1d 100644 --- a/packages/importer/src/clickup/clickup.ts +++ b/packages/importer/src/clickup/clickup.ts @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. // -import contact, { type Person, type PersonAccount } from '@hcengineering/contact' -import { type Ref, type Timestamp, type TxOperations } from '@hcengineering/core' +import contact, { type Person } from '@hcengineering/contact' +import { SocialIdType, buildSocialIdString, type Ref, type Timestamp, type TxOperations } from '@hcengineering/core' import { MarkupNodeType, traverseNode, type MarkupNode } from '@hcengineering/text' import tracker from '@hcengineering/tracker' import csv from 'csvtojson' @@ -82,7 +82,7 @@ interface TasksProcessResult { class ClickupImporter { private personsByName = new Map>() - private accountsByEmail = new Map>() + private knownEmailSocialStrings: string[] = [] constructor ( private readonly client: TxOperations, @@ -122,7 +122,7 @@ class ClickupImporter { private async processClickupTasks (file: string): Promise { await this.fillPersonsByNames() - await this.fillAccountsByEmails() + await this.fillKnownEmails() const projects = new Set() const statuses = new Set() @@ -242,11 +242,13 @@ class ClickupImporter { private convertToImportComments (clickup: string): ImportComment[] { return JSON.parse(clickup).map((comment: ClickupComment) => { - const author = this.accountsByEmail.get(comment.by) + const authorSocialString = buildSocialIdString({ type: SocialIdType.EMAIL, value: comment.by }) + const knownAuthor = this.knownEmailSocialStrings.includes(authorSocialString) + return { - text: author !== undefined ? comment.text : `${comment.text}\n\n*(comment by ${comment.by})*`, + text: knownAuthor ? comment.text : `${comment.text}\n\n*(comment by ${comment.by})*`, date: new Date(comment.date).getTime(), - author + author: knownAuthor ? authorSocialString : undefined } }) } @@ -298,12 +300,9 @@ class ClickupImporter { }, new Map()) } - private async fillAccountsByEmails (): Promise { - const accounts = await this.client.findAll(contact.class.PersonAccount, {}) - this.accountsByEmail = accounts.reduce((accountsByEmail, account) => { - accountsByEmail.set(account.email, account._id) - return accountsByEmail - }, new Map()) + private async fillKnownEmails (): Promise { + const emailSocialIds = await this.client.findAll(contact.class.SocialIdentity, { type: SocialIdType.EMAIL }) + this.knownEmailSocialStrings = emailSocialIds.map(buildSocialIdString) } private fixClickupString (content: string): string { diff --git a/packages/importer/src/huly/unified.ts b/packages/importer/src/huly/unified.ts index 38c013637c..a176073dde 100644 --- a/packages/importer/src/huly/unified.ts +++ b/packages/importer/src/huly/unified.ts @@ -12,10 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. // - +/* eslint-disable @typescript-eslint/no-unused-vars */ import { type Attachment } from '@hcengineering/attachment' -import contact, { Employee, type Person, type PersonAccount } from '@hcengineering/contact' -import { type Class, type Doc, generateId, type Ref, type Space, type TxOperations } from '@hcengineering/core' +import contact, { Employee, type Person } from '@hcengineering/contact' +import { + type Class, + type Doc, + generateId, + PersonId, + type Ref, + type Space, + type TxOperations +} from '@hcengineering/core' import document, { type Document } from '@hcengineering/document' import { MarkupMarkType, type MarkupNode, MarkupNodeType, traverseNode, traverseNodeMarks } from '@hcengineering/text' import tracker, { type Issue, Project } from '@hcengineering/tracker' @@ -328,8 +336,8 @@ export class UnifiedFormatImporter { private readonly ctrlDocTemplateIdByPath = new Map>() private personsByName = new Map>() - private accountsByEmail = new Map>() - private employeesByName = new Map>() + // private accountsByEmail = new Map>() + // private employeesByName = new Map>() constructor ( private readonly client: TxOperations, @@ -587,20 +595,24 @@ export class UnifiedFormatImporter { return person } - private findAccountByEmail (email: string): Ref { - const account = this.accountsByEmail.get(email) - if (account === undefined) { - throw new Error(`Account not found: ${email}`) - } - return account + private findAccountByEmail (email: string): PersonId { + // TODO: FIXME + throw new Error('Not implemented') + // const account = this.accountsByEmail.get(email) + // if (account === undefined) { + // throw new Error(`Account not found: ${email}`) + // } + // return account } private findEmployeeByName (name: string): Ref { - const employee = this.employeesByName.get(name) - if (employee === undefined) { - throw new Error(`Employee not found: ${name}`) - } - return employee + // TODO: FIXME + throw new Error('Not implemented') + // const employee = this.employeesByName.get(name) + // if (employee === undefined) { + // throw new Error(`Employee not found: ${name}`) + // } + // return employee } private async processDocumentsRecursively ( @@ -939,25 +951,29 @@ export class UnifiedFormatImporter { } private async cacheAccountsByEmails (): Promise { - const accounts = await this.client.findAll(contact.class.PersonAccount, {}) - this.accountsByEmail = accounts.reduce((map, account) => { - map.set(account.email, account._id) - return map - }, new Map()) + // TODO: FIXME + throw new Error('Not implemented') + // const accounts = await this.client.findAll(contact.class.PersonAccount, {}) + // this.accountsByEmail = accounts.reduce((map, account) => { + // map.set(account.email, account._id) + // return map + // }, new Map()) } private async cacheEmployeesByName (): Promise { - this.employeesByName = (await this.client.findAll(contact.mixin.Employee, {})) - .map((employee) => { - return { - _id: employee._id, - name: employee.name.split(',').reverse().join(' ') - } - }) - .reduce((refByName, employee) => { - refByName.set(employee.name, employee._id) - return refByName - }, new Map()) + // TODO: FIXME + throw new Error('Not implemented') + // this.employeesByName = (await this.client.findAll(contact.mixin.Employee, {})) + // .map((employee) => { + // return { + // _id: employee._id, + // name: employee.name.split(',').reverse().join(' ') + // } + // }) + // .reduce((refByName, employee) => { + // refByName.set(employee.name, employee._id) + // return refByName + // }, new Map()) } private async collectFileMetadata (folderPath: string): Promise { diff --git a/packages/importer/src/importer/frontUploader.ts b/packages/importer/src/importer/frontUploader.ts index 95fe9b09b7..805b790aff 100644 --- a/packages/importer/src/importer/frontUploader.ts +++ b/packages/importer/src/importer/frontUploader.ts @@ -38,6 +38,7 @@ export class FrontFileUploader implements FileUploader { constructor ( private readonly frontUrl: string, private readonly workspaceId: string, + private readonly workspaceDataId: string, private readonly token: string ) { this.getFileUrl = this.getFileUrl.bind(this) @@ -78,7 +79,10 @@ export class FrontFileUploader implements FileUploader { } public getFileUrl (id: string): string { - return concatLink(this.frontUrl, `/files/${this.workspaceId}/${id}?file=${id}&workspace=${this.workspaceId}`) + return concatLink( + this.frontUrl, + `/files/${this.workspaceDataId ?? this.workspaceId}/${id}?file=${id}&workspace=${this.workspaceId}` + ) } public async uploadCollaborativeDoc (collabId: CollaborativeDoc, content: Markup): Promise { diff --git a/packages/importer/src/importer/importer.ts b/packages/importer/src/importer/importer.ts index 34b8eadf68..81577e4d66 100644 --- a/packages/importer/src/importer/importer.ts +++ b/packages/importer/src/importer/importer.ts @@ -30,7 +30,6 @@ import documents, { useDocumentTemplate } from '@hcengineering/controlled-documents' import core, { - type Account, type AttachedData, type Class, type CollaborativeDoc, @@ -47,7 +46,8 @@ import core, { type Space, type Status, type Timestamp, - type TxOperations + type TxOperations, + type PersonId } from '@hcengineering/core' import document, { type Document, getFirstRank, type Teamspace } from '@hcengineering/document' import task, { @@ -102,8 +102,8 @@ export interface ImportSpace { archived?: boolean description?: string emoji?: string - owners?: Ref[] - members?: Ref[] + owners?: PersonId[] + members?: PersonId[] docs: T[] } export interface ImportDoc { @@ -147,7 +147,7 @@ export interface ImportIssue extends ImportDoc { export interface ImportComment { text: string - author?: Ref + author?: PersonId date?: Timestamp attachments?: ImportAttachment[] } @@ -175,9 +175,9 @@ export interface ImportDrawing { export type ImportControlledDoc = ImportControlledDocument | ImportControlledDocumentTemplate // todo: rename export interface ImportOrgSpace extends ImportSpace { class: Ref> - qualified?: Ref - manager?: Ref - qara?: Ref + qualified?: PersonId + manager?: PersonId + qara?: PersonId } export interface ImportControlledDocumentTemplate extends ImportDoc { diff --git a/packages/importer/src/importer/storageUploader.ts b/packages/importer/src/importer/storageUploader.ts index 3668d55985..37ba96fe2c 100644 --- a/packages/importer/src/importer/storageUploader.ts +++ b/packages/importer/src/importer/storageUploader.ts @@ -19,7 +19,8 @@ import { MeasureContext, Blob as PlatformBlob, Ref, - WorkspaceIdWithUrl + WorkspaceDataId, + WorkspaceIds } from '@hcengineering/core' import type { StorageAdapter } from '@hcengineering/server-core' import { FileUploader, UploadResult } from './uploader' @@ -28,16 +29,21 @@ export class StorageFileUploader implements FileUploader { constructor ( private readonly ctx: MeasureContext, private readonly storageAdapter: StorageAdapter, - private readonly wsUrl: WorkspaceIdWithUrl + private readonly wsIds: WorkspaceIds ) { this.uploadFile = this.uploadFile.bind(this) } + get dataId (): WorkspaceDataId { + return this.wsIds.dataId ?? (this.wsIds.uuid as unknown as WorkspaceDataId) + } + public async uploadFile (id: string, blob: Blob): Promise { try { const arrayBuffer = await blob.arrayBuffer() const buffer = Buffer.from(arrayBuffer) - await this.storageAdapter.put(this.ctx, this.wsUrl, id, buffer, blob.type, buffer.byteLength) + + await this.storageAdapter.put(this.ctx, this.dataId, id, buffer, blob.type, buffer.byteLength) return { success: true, id: id as Ref } } catch (error) { return { success: false, error: error instanceof Error ? error.message : String(error) } @@ -46,7 +52,7 @@ export class StorageFileUploader implements FileUploader { public async uploadCollaborativeDoc (collabId: CollaborativeDoc, content: Markup): Promise { try { - const blobId = await saveCollabJson(this.ctx, this.storageAdapter, this.wsUrl, collabId, content) + const blobId = await saveCollabJson(this.ctx, this.storageAdapter, this.dataId, collabId, content) return { success: true, id: blobId } } catch (error) { return { success: false, error: error instanceof Error ? error.message : String(error) } diff --git a/packages/model/package.json b/packages/model/package.json index 75152d2f86..ddf531a031 100644 --- a/packages/model/package.json +++ b/packages/model/package.json @@ -43,6 +43,7 @@ "@hcengineering/storage": "^0.6.0", "@hcengineering/analytics": "^0.6.0", "@hcengineering/rank": "^0.6.4", + "@hcengineering/account-client": "^0.6.0", "toposort": "^2.0.2", "fast-equals": "^5.2.2" } diff --git a/packages/model/src/dsl.ts b/packages/model/src/dsl.ts index 0f07a46dd0..2f13509b7f 100644 --- a/packages/model/src/dsl.ts +++ b/packages/model/src/dsl.ts @@ -14,7 +14,7 @@ // import core, { - Account, + PersonId, AttachedDoc, Attribute, Class, @@ -350,7 +350,7 @@ export class Builder { space: Ref, attributes: Data, objectId?: Ref, - modifiedBy?: Ref + modifiedBy?: PersonId ): T { const tx = txFactory.createTxCreateDoc(_class, space, attributes, objectId) if (modifiedBy !== undefined) { @@ -523,3 +523,7 @@ export function TypeCollaborativeDoc (): Type { export function TypeRank (): Type { return { _class: core.class.TypeRank, label: core.string.Rank, icon: core.icon.TypeRank } } + +export function TypePersonId (): Type { + return { _class: core.class.TypePersonId, label: core.string.PersonId } +} diff --git a/packages/model/src/migration.ts b/packages/model/src/migration.ts index d38add6d8e..0d3bee9733 100644 --- a/packages/model/src/migration.ts +++ b/packages/model/src/migration.ts @@ -21,10 +21,11 @@ import core, { Space, TxOperations, UnsetOptions, - WorkspaceId, + WorkspaceIds, generateId } from '@hcengineering/core' import { makeRank } from '@hcengineering/rank' +import { AccountClient } from '@hcengineering/account-client' import { StorageAdapter } from '@hcengineering/storage' import { ModelLogger } from './utils' @@ -115,8 +116,9 @@ export interface MigrationClient { migrateState: Map> storageAdapter: StorageAdapter + accountClient: AccountClient - workspaceId: WorkspaceId + wsIds: WorkspaceIds } /** diff --git a/packages/platform/lang/en.json b/packages/platform/lang/en.json index 54f70b1452..40a13583ca 100644 --- a/packages/platform/lang/en.json +++ b/packages/platform/lang/en.json @@ -10,10 +10,9 @@ "UnknownMethod": "Unknown method: {method}", "InternalServerError": "Internal server error", "MaintenanceWarning": "Maintenance Scheduled in {time, plural, =1 {less than a minute} other {# minutes}}", - "AccountNotFound": "Account not found", + "AccountNotFound": "Account not found or the provided credentials are incorrect", "AccountNotConfirmed": "Account not confirmed", "WorkspaceNotFound": "Workspace not found", - "InvalidPassword": "Invalid password", "AccountAlreadyExists": "Account already exists", "WorkspaceRateLimit": "Server is busy, Please wait a bit and try again", "AccountAlreadyConfirmed": "Account already confirmed", diff --git a/packages/platform/src/platform.ts b/packages/platform/src/platform.ts index ec85cb5687..4c6c969254 100644 --- a/packages/platform/src/platform.ts +++ b/packages/platform/src/platform.ts @@ -146,13 +146,16 @@ export default plugin(platformId, { UnknownMethod: '' as StatusCode<{ method: string }>, InternalServerError: '' as StatusCode, MaintenanceWarning: '' as StatusCode<{ time: number }>, - AccountNotFound: '' as StatusCode<{ account: string }>, - AccountNotConfirmed: '' as StatusCode<{ account: string }>, - WorkspaceNotFound: '' as StatusCode<{ workspace: string }>, - WorkspaceArchived: '' as StatusCode<{ workspace: string }>, + AccountNotFound: '' as StatusCode<{ account?: string }>, + AccountNotConfirmed: '' as StatusCode, + WorkspaceNotFound: '' as StatusCode<{ workspaceUuid?: string, workspaceName?: string, workspaceUrl?: string }>, + WorkspaceArchived: '' as StatusCode<{ workspaceUuid: string }>, + SocialIdNotFound: '' as StatusCode<{ socialId: string, type: string }>, + SocialIdNotConfirmed: '' as StatusCode<{ socialId: string, type: string }>, + SocialIdAlreadyConfirmed: '' as StatusCode<{ socialId: string, type: string }>, + PersonNotFound: '' as StatusCode<{ person: string }>, InvalidPassword: '' as StatusCode<{ account: string }>, - AccountAlreadyExists: '' as StatusCode<{ account: string }>, - AccountAlreadyConfirmed: '' as StatusCode<{ account: string }>, + AccountAlreadyExists: '' as StatusCode, WorkspaceAlreadyExists: '' as StatusCode<{ workspace: string }>, WorkspaceRateLimit: '' as StatusCode<{ workspace: string }>, InvalidOtp: '' as StatusCode, diff --git a/packages/presentation/src/collaborator.ts b/packages/presentation/src/collaborator.ts index 8b3ad682e2..9b624f8748 100644 --- a/packages/presentation/src/collaborator.ts +++ b/packages/presentation/src/collaborator.ts @@ -14,17 +14,21 @@ // import { type CollaboratorClient, getClient as getCollaborator } from '@hcengineering/collaborator-client' -import { type Blob, type CollaborativeDoc, type Markup, type Ref, getWorkspaceId } from '@hcengineering/core' +import { type Blob, type CollaborativeDoc, type Markup, type Ref } from '@hcengineering/core' import { getMetadata } from '@hcengineering/platform' import presentation from './plugin' function getClient (): CollaboratorClient { - const workspaceId = getWorkspaceId(getMetadata(presentation.metadata.WorkspaceId) ?? '') + const workspaceUuid = getMetadata(presentation.metadata.WorkspaceUuid) + if (workspaceUuid === undefined) { + throw new Error('WorkspaceUuid is not defined') + } + const token = getMetadata(presentation.metadata.Token) ?? '' const collaboratorURL = getMetadata(presentation.metadata.CollaboratorUrl) ?? '' - return getCollaborator(workspaceId, token, collaboratorURL) + return getCollaborator(workspaceUuid, token, collaboratorURL) } /** @public */ diff --git a/packages/presentation/src/components/SpacesPopup.svelte b/packages/presentation/src/components/SpacesPopup.svelte index 00d0d5a82c..d1f9ef632a 100644 --- a/packages/presentation/src/components/SpacesPopup.svelte +++ b/packages/presentation/src/components/SpacesPopup.svelte @@ -45,13 +45,11 @@ } : undefined - const me = getCurrentAccount()._id - const query = createQuery() $: query.query( _class, { - members: me, + members: { $in: getCurrentAccount().socialIds }, ...(spaceQuery ?? {}), ...(search !== undefined && search !== '' ? { diff --git a/packages/presentation/src/file.ts b/packages/presentation/src/file.ts index c5beacb19a..4b0e6860e8 100644 --- a/packages/presentation/src/file.ts +++ b/packages/presentation/src/file.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import { concatLink, type Blob as PlatformBlob, type Ref } from '@hcengineering/core' +import { concatLink, type WorkspaceDataId, type Blob as PlatformBlob, type Ref } from '@hcengineering/core' import { PlatformError, Severity, Status, getMetadata } from '@hcengineering/platform' import { v4 as uuid } from 'uuid' @@ -99,8 +99,12 @@ function getFilesUrl (): string { return filesUrl.includes('://') ? filesUrl : concatLink(frontUrl, filesUrl) } -export function getCurrentWorkspaceId (): string { - return getMetadata(plugin.metadata.WorkspaceId) ?? '' +export function getCurrentWorkspaceId (): WorkspaceDataId { + return ( + getMetadata(plugin.metadata.WorkspaceDataId) ?? + (getMetadata(plugin.metadata.WorkspaceUuid) as unknown as WorkspaceDataId) ?? + ('' as WorkspaceDataId) + ) } /** diff --git a/packages/presentation/src/plugin.ts b/packages/presentation/src/plugin.ts index 7737e4a664..c7704e663e 100644 --- a/packages/presentation/src/plugin.ts +++ b/packages/presentation/src/plugin.ts @@ -15,6 +15,8 @@ // import { + type WorkspaceDataId, + type WorkspaceUuid, type Class, type Client, type Doc, @@ -141,8 +143,8 @@ export default plugin(presentationId, { CollaboratorUrl: '' as Metadata, Token: '' as Metadata, Endpoint: '' as Metadata, - Workspace: '' as Metadata, - WorkspaceId: '' as Metadata, + WorkspaceUuid: '' as Metadata, + WorkspaceDataId: '' as Metadata, FrontUrl: '' as Asset, LinkPreviewUrl: '' as Metadata, UploadConfig: '' as Metadata, diff --git a/packages/presentation/src/stats.ts b/packages/presentation/src/stats.ts index a380c20d2e..4dda2d34f6 100644 --- a/packages/presentation/src/stats.ts +++ b/packages/presentation/src/stats.ts @@ -1,4 +1,4 @@ -import type { Metrics } from '@hcengineering/core' +import type { Metrics, PersonId } from '@hcengineering/core' // Copy from server/core/stats.ts for UI usage. export interface MemoryStatistics { @@ -22,7 +22,7 @@ export interface StatisticsElement { } export interface UserStatistics { - userId: string + userId: PersonId // TODO: FIXME - test if it's PersonId or PersonUuid sessionId: string data: any mins5: StatisticsElement diff --git a/packages/presentation/src/utils.ts b/packages/presentation/src/utils.ts index dafc5333c1..aacb4b3156 100644 --- a/packages/presentation/src/utils.ts +++ b/packages/presentation/src/utils.ts @@ -46,7 +46,8 @@ import core, { type TxCUD, type TxResult, type TypeAny, - type WithLookup + type WithLookup, + type WorkspaceDataId } from '@hcengineering/core' import { getMetadata, getResource } from '@hcengineering/platform' import { LiveQuery as LQ } from '@hcengineering/query' @@ -100,7 +101,7 @@ class UIClient extends TxOperations implements Client { client: Client, private readonly liveQuery: Client ) { - super(client, getCurrentAccount()._id) + super(client, getCurrentAccount().primarySocialId) } protected pendingTxes = new Set>() @@ -254,14 +255,14 @@ export function getClient (): TxOperations & Client { return clientProxy } -export type OnClientListener = (client: Client, account: Account) => void +export type OnClientListener = (client: Client, account: Account) => void | Promise const onClientListeners: OnClientListener[] = [] export function onClient (l: OnClientListener): void { onClientListeners.push(l) if (client !== undefined) { setTimeout(() => { - l(client, getCurrentAccount()) + void l(client, getCurrentAccount()) }) } } @@ -321,7 +322,7 @@ export async function setClient (_client: Client): Promise { } const acc = getCurrentAccount() onClientListeners.forEach((l) => { - l(_client, acc) + void l(_client, acc) }) } /** @@ -744,7 +745,9 @@ export function decodeTokenPayload (token: string): any { } export function isAdminUser (): boolean { - return decodeTokenPayload(getMetadata(plugin.metadata.Token) ?? '').admin === 'true' + // TODO: fixme + return false + // return decodeTokenPayload(getMetadata(plugin.metadata.Token) ?? '').admin === 'true' } export function isSpace (space: Doc): space is Space { @@ -755,15 +758,17 @@ export function isSpaceClass (_class: Ref>): boolean { return client.getHierarchy().isDerived(_class, core.class.Space) } -export function setPresentationCookie (token: string, workspaceId: string): void { +export function setPresentationCookie (token: string, workspaceUuid: WorkspaceDataId): void { function setToken (path: string): void { - document.cookie = + const res = encodeURIComponent(plugin.metadata.Token.replaceAll(':', '-')) + '=' + encodeURIComponent(token) + `; path=${path}` + console.log('setting cookie', res) + document.cookie = res } - setToken('/files/' + workspaceId) + setToken('/files/' + workspaceUuid) } export const upgradeDownloadProgress = writable(-1) diff --git a/packages/query/src/__tests__/connection.ts b/packages/query/src/__tests__/connection.ts index 6c41e0cd76..b8b490c7a4 100644 --- a/packages/query/src/__tests__/connection.ts +++ b/packages/query/src/__tests__/connection.ts @@ -14,10 +14,9 @@ // import core, { - Account, - AccountClient, BackupClient, Class, + Client, ClientConnectEvent, ClientConnection, Doc, @@ -43,7 +42,7 @@ import core, { import { genMinModel } from './minmodel' export async function connect (handler: (tx: Tx) => void): Promise< -AccountClient & +Client & BackupClient & FulltextStorage & { isConnected: () => boolean @@ -103,10 +102,6 @@ FulltextStorage & { return this.model } - async getAccount (): Promise { - return {} as unknown as any - } - async tx (tx: Tx): Promise { if (tx.objectSpace === core.space.Model) { this.hierarchy.tx(tx) diff --git a/packages/query/src/__tests__/minmodel.ts b/packages/query/src/__tests__/minmodel.ts index fffb687a68..57ce373345 100644 --- a/packages/query/src/__tests__/minmodel.ts +++ b/packages/query/src/__tests__/minmodel.ts @@ -14,7 +14,7 @@ // import type { - Account, + PersonId, Arr, Class, Data, @@ -27,7 +27,7 @@ import type { TxCreateDoc, TxCUD } from '@hcengineering/core' -import core, { AccountRole, AttachedDoc, ClassifierKind, DOMAIN_MODEL, DOMAIN_TX, TxFactory } from '@hcengineering/core' +import core, { AttachedDoc, ClassifierKind, DOMAIN_MODEL, DOMAIN_TX, TxFactory } from '@hcengineering/core' import type { IntlString, Plugin } from '@hcengineering/platform' import { plugin } from '@hcengineering/platform' @@ -44,7 +44,7 @@ export function createDoc ( _class: Ref>, attributes: Data, id?: Ref, - modifiedBy?: Ref + modifiedBy?: PersonId ): TxCreateDoc { const result = txFactory.createTxCreateDoc(_class, core.space.Model, attributes, id) if (modifiedBy !== undefined) { @@ -142,14 +142,15 @@ export function genMinModel (): TxCUD[] { domain: DOMAIN_MODEL }) ) - txes.push( - createClass(core.class.Account, { - label: 'Account' as IntlString, - extends: core.class.Doc, - kind: ClassifierKind.CLASS, - domain: DOMAIN_MODEL - }) - ) + // TODO: fixme! + // txes.push( + // createClass(core.class.Account, { + // label: 'Account' as IntlString, + // extends: core.class.Doc, + // kind: ClassifierKind.CLASS, + // domain: DOMAIN_MODEL + // }) + // ) txes.push( createClass(core.class.Tx, { @@ -239,11 +240,12 @@ export function genMinModel (): TxCUD[] { }) ) - const u1 = 'User1' as Ref - const u2 = 'User2' as Ref + const u1 = 'User1' as PersonId + const u2 = 'User2' as PersonId + // TODO: fixme! txes.push( - createDoc(core.class.Account, { email: 'user1@site.com', role: AccountRole.User }, u1), - createDoc(core.class.Account, { email: 'user2@site.com', role: AccountRole.User }, u2), + // createDoc(core.class.Account, { email: 'user1@site.com', role: AccountRole.User }, u1), + // createDoc(core.class.Account, { email: 'user2@site.com', role: AccountRole.User }, u2), createDoc(core.class.Space, { name: 'Sp1', description: '', diff --git a/packages/query/src/__tests__/query.test.ts b/packages/query/src/__tests__/query.test.ts index 3e19654ecc..dcea4759a6 100644 --- a/packages/query/src/__tests__/query.test.ts +++ b/packages/query/src/__tests__/query.test.ts @@ -14,7 +14,6 @@ // import core, { - AccountRole, createClient, Doc, generateId, @@ -105,10 +104,11 @@ describe('query', () => { }) }) - await factory.createDoc(core.class.Account, core.space.Model, { - email: 'user1@site.com', - role: AccountRole.User - }) + // TODO: fixme! + // await factory.createDoc(core.class.Account, core.space.Model, { + // email: 'user1@site.com', + // role: AccountRole.User + // }) await factory.createDoc(core.class.Space, core.space.Model, { private: true, name: '#0', diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index ad17858262..7ce25ca442 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -13,12 +13,11 @@ // limitations under the License. // -import { type Blob, type MeasureContext, type StorageIterator, type WorkspaceId } from '@hcengineering/core' +import { type Blob, type MeasureContext, type StorageIterator, type WorkspaceDataId } from '@hcengineering/core' import { PlatformError, unknownError } from '@hcengineering/platform' import { type Readable } from 'stream' export type ListBlobResult = Omit - export interface UploadedObjectInfo { etag: string versionId: string | null @@ -36,37 +35,37 @@ export interface BucketInfo { } export interface StorageAdapter { - initialize: (ctx: MeasureContext, workspaceId: WorkspaceId) => Promise + initialize: (ctx: MeasureContext, dataId: WorkspaceDataId) => Promise close: () => Promise - exists: (ctx: MeasureContext, workspaceId: WorkspaceId) => Promise - make: (ctx: MeasureContext, workspaceId: WorkspaceId) => Promise - delete: (ctx: MeasureContext, workspaceId: WorkspaceId) => Promise + exists: (ctx: MeasureContext, dataId: WorkspaceDataId) => Promise + make: (ctx: MeasureContext, dataId: WorkspaceDataId) => Promise + delete: (ctx: MeasureContext, dataId: WorkspaceDataId) => Promise listBuckets: (ctx: MeasureContext) => Promise - remove: (ctx: MeasureContext, workspaceId: WorkspaceId, objectNames: string[]) => Promise - listStream: (ctx: MeasureContext, workspaceId: WorkspaceId) => Promise - stat: (ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string) => Promise - get: (ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string) => Promise + remove: (ctx: MeasureContext, dataId: WorkspaceDataId, objectNames: string[]) => Promise + listStream: (ctx: MeasureContext, dataId: WorkspaceDataId) => Promise + stat: (ctx: MeasureContext, dataId: WorkspaceDataId, objectName: string) => Promise + get: (ctx: MeasureContext, dataId: WorkspaceDataId, objectName: string) => Promise put: ( ctx: MeasureContext, - workspaceId: WorkspaceId, + dataId: WorkspaceDataId, objectName: string, stream: Readable | Buffer | string, contentType: string, size?: number ) => Promise - read: (ctx: MeasureContext, workspaceId: WorkspaceId, name: string) => Promise + read: (ctx: MeasureContext, dataId: WorkspaceDataId, name: string) => Promise partial: ( ctx: MeasureContext, - workspaceId: WorkspaceId, + dataId: WorkspaceDataId, objectName: string, offset: number, length?: number ) => Promise - getUrl: (ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string) => Promise + getUrl: (ctx: MeasureContext, dataId: WorkspaceDataId, objectName: string) => Promise } export interface NamedStorageAdapter { @@ -77,7 +76,7 @@ export interface NamedStorageAdapter { export interface StorageAdapterEx extends StorageAdapter { adapters?: NamedStorageAdapter[] - find: (ctx: MeasureContext, workspaceId: WorkspaceId) => StorageIterator + find: (ctx: MeasureContext, dataId: WorkspaceDataId) => StorageIterator } /** @@ -85,19 +84,19 @@ export interface StorageAdapterEx extends StorageAdapter { */ export class DummyStorageAdapter implements StorageAdapter, StorageAdapterEx { defaultAdapter: string = '' - async syncBlobFromStorage (ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string): Promise { + async syncBlobFromStorage (ctx: MeasureContext, dataId: WorkspaceDataId, objectName: string): Promise { throw new PlatformError(unknownError('Method not implemented')) } - async initialize (ctx: MeasureContext, workspaceId: WorkspaceId): Promise {} + async initialize (ctx: MeasureContext, dataId: WorkspaceDataId): Promise {} async close (): Promise {} - async exists (ctx: MeasureContext, workspaceId: WorkspaceId): Promise { + async exists (ctx: MeasureContext, dataId: WorkspaceDataId): Promise { return false } - find (ctx: MeasureContext, workspaceId: WorkspaceId): StorageIterator { + find (ctx: MeasureContext, dataId: WorkspaceDataId): StorageIterator { return { next: async (ctx) => [], close: async (ctx) => {} @@ -108,17 +107,17 @@ export class DummyStorageAdapter implements StorageAdapter, StorageAdapterEx { return [] } - async make (ctx: MeasureContext, workspaceId: WorkspaceId): Promise {} + async make (ctx: MeasureContext, dataId: WorkspaceDataId): Promise {} - async delete (ctx: MeasureContext, workspaceId: WorkspaceId): Promise {} + async delete (ctx: MeasureContext, dataId: WorkspaceDataId): Promise {} - async remove (ctx: MeasureContext, workspaceId: WorkspaceId, objectNames: string[]): Promise {} + async remove (ctx: MeasureContext, dataId: WorkspaceDataId, objectNames: string[]): Promise {} - async list (ctx: MeasureContext, workspaceId: WorkspaceId): Promise { + async list (ctx: MeasureContext, dataId: WorkspaceDataId): Promise { return [] } - async listStream (ctx: MeasureContext, workspaceId: WorkspaceId): Promise { + async listStream (ctx: MeasureContext, dataId: WorkspaceDataId): Promise { return { next: async (): Promise => { return [] @@ -127,17 +126,17 @@ export class DummyStorageAdapter implements StorageAdapter, StorageAdapterEx { } } - async stat (ctx: MeasureContext, workspaceId: WorkspaceId, name: string): Promise { + async stat (ctx: MeasureContext, dataId: WorkspaceDataId, name: string): Promise { return undefined } - async get (ctx: MeasureContext, workspaceId: WorkspaceId, name: string): Promise { + async get (ctx: MeasureContext, dataId: WorkspaceDataId, name: string): Promise { throw new Error('not implemented') } async partial ( ctx: MeasureContext, - workspaceId: WorkspaceId, + dataId: WorkspaceDataId, objectName: string, offset: number, length?: number | undefined @@ -145,13 +144,13 @@ export class DummyStorageAdapter implements StorageAdapter, StorageAdapterEx { throw new Error('not implemented') } - async read (ctx: MeasureContext, workspaceId: WorkspaceId, name: string): Promise { + async read (ctx: MeasureContext, dataId: WorkspaceDataId, name: string): Promise { throw new Error('not implemented') } async put ( ctx: MeasureContext, - workspaceId: WorkspaceId, + dataId: WorkspaceDataId, objectName: string, stream: string | Readable | Buffer, contentType: string, @@ -160,7 +159,7 @@ export class DummyStorageAdapter implements StorageAdapter, StorageAdapterEx { throw new Error('not implemented') } - async getUrl (ctx: MeasureContext, workspaceId: WorkspaceId, objectName: string): Promise { + async getUrl (ctx: MeasureContext, dataId: WorkspaceDataId, objectName: string): Promise { throw new Error('not implemented') } } @@ -172,11 +171,11 @@ export function createDummyStorageAdapter (): StorageAdapter { export async function removeAllObjects ( ctx: MeasureContext, storage: StorageAdapter, - workspaceId: WorkspaceId + dataId: WorkspaceDataId ): Promise { - ctx.warn('removing all objects from workspace', { workspaceId }) + ctx.warn('removing all objects from workspace', { dataId }) // We need to list all files and delete them - const iterator = await storage.listStream(ctx, workspaceId) + const iterator = await storage.listStream(ctx, dataId) let bulk: string[] = [] while (true) { const objs = await iterator.next() @@ -186,13 +185,13 @@ export async function removeAllObjects ( for (const obj of objs) { bulk.push(obj._id) if (bulk.length > 50) { - await storage.remove(ctx, workspaceId, bulk) + await storage.remove(ctx, dataId, bulk) bulk = [] } } } if (bulk.length > 0) { - await storage.remove(ctx, workspaceId, bulk) + await storage.remove(ctx, dataId, bulk) bulk = [] } await iterator.close() @@ -201,10 +200,10 @@ export async function removeAllObjects ( export async function objectsToArray ( ctx: MeasureContext, storage: StorageAdapter, - workspaceId: WorkspaceId + dataId: WorkspaceDataId ): Promise { // We need to list all files and delete them - const iterator = await storage.listStream(ctx, workspaceId) + const iterator = await storage.listStream(ctx, dataId) const bulk: ListBlobResult[] = [] while (true) { const obj = await iterator.next() diff --git a/plugins/activity-resources/src/activityMessagesUtils.ts b/plugins/activity-resources/src/activityMessagesUtils.ts index a13c1caec6..128e0f363a 100644 --- a/plugins/activity-resources/src/activityMessagesUtils.ts +++ b/plugins/activity-resources/src/activityMessagesUtils.ts @@ -38,7 +38,7 @@ import contact, { type Person } from '@hcengineering/contact' import { type IntlString } from '@hcengineering/platform' import { type AnyComponent } from '@hcengineering/ui' import { get } from 'svelte/store' -import { personAccountByIdStore } from '@hcengineering/contact-resources' +import { personRefByPersonIdStore } from '@hcengineering/contact-resources' import activity, { type ActivityMessage, type DisplayActivityMessage, @@ -366,17 +366,16 @@ function groupByTime (messages: T[]): T[][] { } function getDocUpdateMessageKey (message: DocUpdateMessage): string { - const personAccountById = get(personAccountByIdStore) - const person = personAccountById.get(message.createdBy as any)?.person ?? message.createdBy + const personRef = get(personRefByPersonIdStore).get(message.createdBy as any) if (message.action === 'update') { - return [message._class, message.attachedTo, message.action, person, getAttributeUpdatesKey(message)].join('_') + return [message._class, message.attachedTo, message.action, personRef, getAttributeUpdatesKey(message)].join('_') } return [ message._class, message.attachedTo, - person, + personRef, message.updateCollection, message.objectId === message.attachedTo ].join('_') diff --git a/plugins/activity-resources/src/components/BasePreview.svelte b/plugins/activity-resources/src/components/BasePreview.svelte index e67a8b4520..e07da4709f 100644 --- a/plugins/activity-resources/src/components/BasePreview.svelte +++ b/plugins/activity-resources/src/components/BasePreview.svelte @@ -15,9 +15,9 @@ - {#if currentAccount.person === targetDoc._id} + {#if currentEmployee === targetDoc._id}

- {#each reactionAccounts as acc} - + {#each persons as person} + {/each}
diff --git a/plugins/activity-resources/src/utils.ts b/plugins/activity-resources/src/utils.ts index 06df26ee33..49feb2f91a 100644 --- a/plugins/activity-resources/src/utils.ts +++ b/plugins/activity-resources/src/utils.ts @@ -22,12 +22,13 @@ export async function updateDocReactions (reactions: Reaction[], object?: Doc, e const client = getClient() const currentAccount = getCurrentAccount() - const reaction = reactions.find((r) => r.emoji === emoji && r.createBy === currentAccount._id) + const socialStrings = currentAccount.socialIds + const reaction = reactions.find((r) => r.emoji === emoji && socialStrings.includes(r.createBy)) if (reaction == null) { await client.addCollection(activity.class.Reaction, object.space, object._id, object._class, 'reactions', { emoji, - createBy: currentAccount._id + createBy: currentAccount.primarySocialId }) } else { await client.remove(reaction) diff --git a/plugins/activity/src/index.ts b/plugins/activity/src/index.ts index 69ff2ee48e..9e602708cb 100644 --- a/plugins/activity/src/index.ts +++ b/plugins/activity/src/index.ts @@ -15,7 +15,7 @@ import { Person } from '@hcengineering/contact' import { - Account, + PersonId, AttachedDoc, Class, Doc, @@ -37,7 +37,7 @@ import type { Action } from '@hcengineering/view' * @public */ export interface ActivityMessage extends AttachedDoc { - modifiedBy: Ref + modifiedBy: PersonId modifiedOn: Timestamp isPinned?: boolean @@ -220,7 +220,7 @@ export interface Reaction extends AttachedDoc { attachedTo: Ref attachedToClass: Ref> emoji: string - createBy: Ref + createBy: PersonId } /** diff --git a/plugins/ai-bot-resources/src/components/OnboardingChannelAsideExtension.svelte b/plugins/ai-bot-resources/src/components/OnboardingChannelAsideExtension.svelte index cf589accbc..7fa63390c8 100644 --- a/plugins/ai-bot-resources/src/components/OnboardingChannelAsideExtension.svelte +++ b/plugins/ai-bot-resources/src/components/OnboardingChannelAsideExtension.svelte @@ -21,7 +21,7 @@ const req: OnboardingEventRequest = { event: OnboardingEvent.OpenChatInSidebar, data: { - email: object.email, + personId: object.socialString, workspace: object.workspaceId } } diff --git a/plugins/ai-bot/src/index.ts b/plugins/ai-bot/src/index.ts index 483c317c49..e6799360c4 100644 --- a/plugins/ai-bot/src/index.ts +++ b/plugins/ai-bot/src/index.ts @@ -12,8 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // - -import { Account, type Mixin, Ref } from '@hcengineering/core' +import { type PersonId, buildSocialIdString, type Mixin, type Ref, SocialIdType, PersonUuid } from '@hcengineering/core' import type { Metadata, Plugin } from '@hcengineering/platform' import { plugin } from '@hcengineering/platform' import { ChatMessage } from '@hcengineering/chunter' @@ -24,6 +23,11 @@ export * from './rest' export const aiBotId = 'ai-bot' as Plugin export const aiBotAccountEmail = 'huly.ai.bot@hc.engineering' +export const aiBotEmailSocialId = buildSocialIdString({ + type: SocialIdType.EMAIL, + value: aiBotAccountEmail +}) +export const aiBotAccount = '5a1a5faa-582c-42a6-8613-fc80a15e3ae8' as PersonUuid export interface TransferredMessage extends ChatMessage { messageId: Ref @@ -38,7 +42,7 @@ const aiBot = plugin(aiBotId, { TransferredMessage: '' as Ref> }, account: { - AIBot: '' as Ref + AIBot: '' as PersonId } }) diff --git a/plugins/ai-bot/src/rest.ts b/plugins/ai-bot/src/rest.ts index 226ccafe3c..61fbdaa33c 100644 --- a/plugins/ai-bot/src/rest.ts +++ b/plugins/ai-bot/src/rest.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import { Account, Class, Doc, Markup, Ref, Space, Timestamp } from '@hcengineering/core' +import { Class, Doc, Markup, PersonId, Ref, Space, Timestamp, type WorkspaceUuid } from '@hcengineering/core' import { ChatMessage } from '@hcengineering/chunter' import { Room, RoomLanguage } from '@hcengineering/love' import { Person } from '@hcengineering/contact' @@ -36,14 +36,14 @@ export interface AIMessageEventRequest extends AIEventRequest { objectId: Ref objectClass: Ref> objectSpace: Ref - user: Ref + user: PersonId email: string } export interface AITransferEventRequest extends AIEventRequest { - toEmail: string - toWorkspace: string - fromWorkspace: string + toPersonId: PersonId + toWorkspace: WorkspaceUuid + fromWorkspace: WorkspaceUuid fromWorkspaceName: string fromWorkspaceUrl: string parentMessageId?: Ref diff --git a/plugins/ai-bot/src/types.ts b/plugins/ai-bot/src/types.ts index 5d447977d0..c1fd913c63 100644 --- a/plugins/ai-bot/src/types.ts +++ b/plugins/ai-bot/src/types.ts @@ -13,13 +13,15 @@ // limitations under the License. // +import { type WorkspaceUuid, type PersonId } from '@hcengineering/core' + export enum OnboardingEvent { OpenChatInSidebar = 'openChatInSidebar' } export interface OpenChatInSidebarData { - email: string - workspace: string + personId: PersonId + workspace: WorkspaceUuid } export interface OnboardingEventRequest> { diff --git a/plugins/analytics-collector-assets/lang/en.json b/plugins/analytics-collector-assets/lang/en.json index 63ed2dbb61..eeeef6ecdd 100644 --- a/plugins/analytics-collector-assets/lang/en.json +++ b/plugins/analytics-collector-assets/lang/en.json @@ -14,7 +14,7 @@ "WorkspaceId": "Workspace id", "WorkspaceName": "Workspace name", "WorkspaceUrl": "Workspace url", - "Email": "Email", + "SocialId": "Social Id", "UserName": "User name", "DisableAIReplies": "Disable AI replies", "ShowAIReplies": "Show AI replies" diff --git a/plugins/analytics-collector-assets/lang/es.json b/plugins/analytics-collector-assets/lang/es.json index 47fc2e088d..a00892a8b3 100644 --- a/plugins/analytics-collector-assets/lang/es.json +++ b/plugins/analytics-collector-assets/lang/es.json @@ -14,7 +14,7 @@ "WorkspaceId": "Workspace id", "WorkspaceName": "Workspace name", "WorkspaceUrl": "Workspace url", - "Email": "Email", + "SocialId": "Social Id", "UserName": "User name", "DisableAIReplies": "Disable AI replies", "ShowAIReplies": "Show AI replies" diff --git a/plugins/analytics-collector-assets/lang/pt.json b/plugins/analytics-collector-assets/lang/pt.json index 47fc2e088d..a00892a8b3 100644 --- a/plugins/analytics-collector-assets/lang/pt.json +++ b/plugins/analytics-collector-assets/lang/pt.json @@ -14,7 +14,7 @@ "WorkspaceId": "Workspace id", "WorkspaceName": "Workspace name", "WorkspaceUrl": "Workspace url", - "Email": "Email", + "SocialId": "Social Id", "UserName": "User name", "DisableAIReplies": "Disable AI replies", "ShowAIReplies": "Show AI replies" diff --git a/plugins/analytics-collector-assets/lang/ru.json b/plugins/analytics-collector-assets/lang/ru.json index 37ce460825..4f68cd1d12 100644 --- a/plugins/analytics-collector-assets/lang/ru.json +++ b/plugins/analytics-collector-assets/lang/ru.json @@ -14,7 +14,7 @@ "WorkspaceId": "Id пространства", "WorkspaceName": "Имя пространства", "WorkspaceUrl": "URL пространства", - "Email": "Email", + "SocialId": "Социальный ID", "UserName": "Имя пользователя", "DisableAIReplies": "Отключить ответы ИИ", "ShowAIReplies": "Показать ответы ИИ" diff --git a/plugins/analytics-collector/src/index.ts b/plugins/analytics-collector/src/index.ts index f1534b6fda..67b4b77a84 100644 --- a/plugins/analytics-collector/src/index.ts +++ b/plugins/analytics-collector/src/index.ts @@ -52,7 +52,7 @@ const analyticsCollector = plugin(analyticsCollectorId, { WorkspaceId: '' as IntlString, WorkspaceName: '' as IntlString, WorkspaceUrl: '' as IntlString, - Email: '' as IntlString, + SocialId: '' as IntlString, UserName: '' as IntlString, DisableAIReplies: '' as IntlString, ShowAIReplies: '' as IntlString diff --git a/plugins/analytics-collector/src/types.ts b/plugins/analytics-collector/src/types.ts index 5f244103e3..88000c4cd4 100644 --- a/plugins/analytics-collector/src/types.ts +++ b/plugins/analytics-collector/src/types.ts @@ -14,6 +14,7 @@ // import { Channel } from '@hcengineering/chunter' +import type { PersonId, WorkspaceUuid } from '@hcengineering/core' export enum AnalyticEventType { SetUser = 'setUser', @@ -30,10 +31,10 @@ export interface AnalyticEvent { } export interface OnboardingChannel extends Channel { - workspaceId: string + workspaceId: WorkspaceUuid workspaceName: string workspaceUrl: string - email: string + socialString: PersonId userName: string disableAIReplies: boolean showAIReplies: boolean diff --git a/plugins/analytics-collector/src/utils.ts b/plugins/analytics-collector/src/utils.ts index 9b2c888ea3..fe9aa776a8 100644 --- a/plugins/analytics-collector/src/utils.ts +++ b/plugins/analytics-collector/src/utils.ts @@ -13,6 +13,8 @@ // limitations under the License. // -export function getOnboardingChannelName (worksapce: string, email: string): string { - return `${email}; ${worksapce}` +import type { PersonId } from '@hcengineering/core' + +export function getOnboardingChannelName (worksapceUrl: string, personId: PersonId): string { + return `${personId}; ${worksapceUrl}` } diff --git a/plugins/attachment-resources/src/components/AttachmentPresenter.svelte b/plugins/attachment-resources/src/components/AttachmentPresenter.svelte index 44279e5c25..2d1a904afd 100644 --- a/plugins/attachment-resources/src/components/AttachmentPresenter.svelte +++ b/plugins/attachment-resources/src/components/AttachmentPresenter.svelte @@ -14,6 +14,7 @@ // limitations under the License. -->
- +
diff --git a/plugins/attachment-resources/src/components/FileBrowser.svelte b/plugins/attachment-resources/src/components/FileBrowser.svelte index 87d2e12112..72f594930f 100644 --- a/plugins/attachment-resources/src/components/FileBrowser.svelte +++ b/plugins/attachment-resources/src/components/FileBrowser.svelte @@ -14,11 +14,12 @@ --> diff --git a/plugins/chunter-resources/src/components/chat-message/ChatMessageInput.svelte b/plugins/chunter-resources/src/components/chat-message/ChatMessageInput.svelte index 5d22e2e659..003476bf12 100644 --- a/plugins/chunter-resources/src/components/chat-message/ChatMessageInput.svelte +++ b/plugins/chunter-resources/src/components/chat-message/ChatMessageInput.svelte @@ -17,14 +17,14 @@ import { Analytics } from '@hcengineering/analytics' import { AttachmentRefInput } from '@hcengineering/attachment-resources' import chunter, { ChatMessage, ChunterEvents, ThreadMessage } from '@hcengineering/chunter' - import { Class, Doc, generateId, Ref, type CommitResult, getCurrentAccount } from '@hcengineering/core' + import { Class, Doc, generateId, Ref, type CommitResult } from '@hcengineering/core' import { createQuery, DraftController, draftsStore, getClient } from '@hcengineering/presentation' import { EmptyMarkup, isEmptyMarkup } from '@hcengineering/text' import { createEventDispatcher } from 'svelte' import { getObjectId } from '@hcengineering/view-resources' import { ThrottledCaller } from '@hcengineering/ui' import { getSpace } from '@hcengineering/activity-resources' - import { PersonAccount } from '@hcengineering/contact' + import { getCurrentEmployee } from '@hcengineering/contact' import { presenceByObjectId, updateMyPresence } from '@hcengineering/presence-resources' import { type PresenceTyping } from '../../types' @@ -102,7 +102,7 @@ } } - const me = getCurrentAccount() as PersonAccount + const me = getCurrentEmployee() const throttle = new ThrottledCaller(500) async function deleteTypingInfo (): Promise { @@ -116,7 +116,7 @@ throttle.call(() => { const room = { objectId: object._id, objectClass: object._class } - const typing = { person: me.person, lastTyping: Date.now() } + const typing = { person: me, lastTyping: Date.now() } updateMyPresence(room, { typing }) }) } diff --git a/plugins/chunter-resources/src/components/chat-message/ChatMessagePresenter.svelte b/plugins/chunter-resources/src/components/chat-message/ChatMessagePresenter.svelte index 295f79e975..c726b47fb7 100644 --- a/plugins/chunter-resources/src/components/chat-message/ChatMessagePresenter.svelte +++ b/plugins/chunter-resources/src/components/chat-message/ChatMessagePresenter.svelte @@ -13,9 +13,9 @@ // limitations under the License. --> - import { Person, PersonAccount, getName } from '@hcengineering/contact' - import core, { IdMap, Ref, Space } from '@hcengineering/core' + import { Person, getName } from '@hcengineering/contact' + import { Ref, Space, notEmpty } from '@hcengineering/core' import presentation, { getClient } from '@hcengineering/presentation' import { ActionIcon, Button, IconClose, Label } from '@hcengineering/ui' import { createEventDispatcher } from 'svelte' import contact from '../plugin' import UsersPopup from './UsersPopup.svelte' + import { personRefByPersonIdStore, personByIdStore, primarySocialIdByPersonRefStore } from '../utils' export let value: Space const dispatch = createEventDispatcher() const client = getClient() - const employees: IdMap = new Map() + let membersToAdd: Ref[] = [] + const channelMembers: Ref[] = value.members + .map((pid) => { + const personRef = $personRefByPersonIdStore.get(pid) - let membersToAdd: PersonAccount[] = [] - let channelMembers: Ref[] = [] - client.findAll(core.class.Account, { _id: { $in: value.members } }).then((res) => { - channelMembers = res.filter((e) => e._class === contact.class.PersonAccount).map((e) => (e as PersonAccount).person) - }) + if (personRef === undefined) { + console.error(`Person with social id ${pid} not found`) + return undefined + } - async function changeMembersToAdd (employees: Ref[]) { - if (employees) { - await client.findAll(contact.class.PersonAccount, { person: { $in: employees } }).then((res) => { - if (res) { - membersToAdd = res - } - }) - } + return personRef + }) + .filter(notEmpty) + + async function changeMembersToAdd (employees: Ref[]): Promise { + membersToAdd = employees } - $: selectedEmployees = membersToAdd.map((e) => e.person) - - function removeMember (_id: Ref) { - membersToAdd = membersToAdd.filter((m) => m._id !== _id) + function removeMember (_id: Ref): void { + membersToAdd = membersToAdd.filter((m) => m !== _id) } @@ -54,15 +53,15 @@ {#if membersToAdd.length}
{#each membersToAdd as m} - {@const employee = employees.get(m.person)} + {@const employee = $personByIdStore.get(m)}
- {employee ? getName(client.getHierarchy(), employee) : ''} + {employee !== undefined ? getName(client.getHierarchy(), employee) : ''}
{ - removeMember(m._id) + removeMember(m) }} />
@@ -79,7 +78,7 @@ }} multiSelect={true} allowDeselect={true} - selectedUsers={selectedEmployees} + selectedUsers={membersToAdd} ignoreUsers={channelMembers} shadows={false} on:update={(ev) => changeMembersToAdd(ev.detail)} @@ -89,7 +88,7 @@ on:click={() => { dispatch( 'close', - membersToAdd.map((m) => m._id) + membersToAdd.map((m) => $primarySocialIdByPersonRefStore.get(m)) ) }} label={presentation.string.Add} diff --git a/plugins/contact-resources/src/components/AssigneePopup.svelte b/plugins/contact-resources/src/components/AssigneePopup.svelte index dc58b83a1d..e42966b12a 100644 --- a/plugins/contact-resources/src/components/AssigneePopup.svelte +++ b/plugins/contact-resources/src/components/AssigneePopup.svelte @@ -13,8 +13,8 @@ // limitations under the License. --> -{#if showStatus && accounts.length > 0} +{#if showStatus && person}
{#if person} diff --git a/plugins/contact-resources/src/components/CreateEmployee.svelte b/plugins/contact-resources/src/components/CreateEmployee.svelte index 32472b7fa4..a80915a04a 100644 --- a/plugins/contact-resources/src/components/CreateEmployee.svelte +++ b/plugins/contact-resources/src/components/CreateEmployee.svelte @@ -13,23 +13,24 @@ // limitations under the License. --> - -{#if value} - - {#if employee} - - {:else if person} - - {:else} -
- - - -
- {/if} -{/if} - - diff --git a/plugins/contact-resources/src/components/PersonAccountRefPresenter.svelte b/plugins/contact-resources/src/components/PersonAccountRefPresenter.svelte deleted file mode 100644 index b360bdcf57..0000000000 --- a/plugins/contact-resources/src/components/PersonAccountRefPresenter.svelte +++ /dev/null @@ -1,53 +0,0 @@ - - - -{#if account} - -{:else} -