diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 53022a30ba..35c0e09c52 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -1678,6 +1678,9 @@ importers: '@types/markdown-it': specifier: ~13.0.0 version: 13.0.8 + '@types/maxmind': + specifier: ^2.0.5 + version: 2.0.5 '@types/mime': specifier: ^3.0.1 version: 3.0.4 @@ -2074,6 +2077,9 @@ importers: markdown-it: specifier: ^14.0.0 version: 14.0.0 + maxmind: + specifier: ^4.3.6 + version: 4.3.28 mermaid: specifier: ~11.4.1 version: 11.4.1 @@ -5271,7 +5277,7 @@ packages: version: 0.0.0 '@rush-temp/pod-analytics-collector@file:projects/pod-analytics-collector.tgz': - resolution: {integrity: sha512-YsXK0lM7Spe4gaV61mzcLnesPCnB3kxryetFQJEFTNEh+azKvSQTXGvyozSPJL8OriZDNMC/NqsqXqO2/fFLPg==, tarball: file:projects/pod-analytics-collector.tgz} + resolution: {integrity: sha512-NyLI9rsK/JB0vx2gpodNNTmfT+ApGsmY8QKkPs1T3cD+zOx15AlsSty43i9f7feiCwCBA2JbfgK60mb5QePr3w==, tarball: file:projects/pod-analytics-collector.tgz} version: 0.0.0 '@rush-temp/pod-backup@file:projects/pod-backup.tgz': @@ -5351,7 +5357,7 @@ packages: version: 0.0.0 '@rush-temp/pod-process@file:projects/pod-process.tgz': - resolution: {integrity: sha512-CX94IUHO3sQtTelqy4CBPydusE7MTCJ/l26hsifScPSCKEZTc4+2Qz3aZSwo2SMLerH1ZrXATpDeI4l2sgOT0A==, tarball: file:projects/pod-process.tgz} + resolution: {integrity: sha512-lM7eqcK1IQzWvzrzQ2Iu1H7e1ETVWovBtvpXPpNMMyr6ied0Hbj60PgxUZsTE5ZwKcGv0ZFDfP8HlJSPXrG2zg==, tarball: file:projects/pod-process.tgz} version: 0.0.0 '@rush-temp/pod-server@file:projects/pod-server.tgz': @@ -5727,7 +5733,7 @@ packages: version: 0.0.0 '@rush-temp/server-process@file:projects/server-process.tgz': - resolution: {integrity: sha512-pKJ3cCSJOnjIfOQwJp1rHv6WDYuFLhp+J6/O5t1+evZ0YB8/ynv1lb9PPbg9rjGwwGQ41AQlaTn9cCDhfZdAOg==, tarball: file:projects/server-process.tgz} + resolution: {integrity: sha512-0FfWPqe+aaK/TXdXElATagm3zoh7G3p8k269zjs34NEpWNHOwK4NWypuh82G8YB3D9CgtF/jKov2KuQRPms8wA==, tarball: file:projects/server-process.tgz} version: 0.0.0 '@rush-temp/server-recruit-resources@file:projects/server-recruit-resources.tgz': @@ -6912,6 +6918,10 @@ packages: '@types/markdown-it@14.1.2': resolution: {integrity: sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==} + '@types/maxmind@2.0.5': + resolution: {integrity: sha512-fGqeOLsZI189WqKrAvaGfLzaLg5T8NNenAHrvmRuIg6lLfQCMICw4kg9wxyxzLAJnxwN89CJhTq6DZQ/w1K5hw==} + deprecated: This is a stub types definition. maxmind provides its own type definitions, so you do not need this installed. + '@types/mdurl@1.0.5': resolution: {integrity: sha512-6L6VymKTzYSrEf4Nev4Xa1LCHKrlTlYCBMTlQKFuddo1CvQcE52I0mwfOJayueUC7MJuXOeHTcIU683lzd0cUA==} @@ -11220,6 +11230,10 @@ packages: resolution: {integrity: sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==} engines: {node: '>=10'} + maxmind@4.3.28: + resolution: {integrity: sha512-K3PcTVjhrSU6xzY7niQf/CHPmx/qzkMIpMumd3x0JXMkIDJMerL4LY9RsEhArHb4T3IK0NBxQcUlwjsxRm9rIw==} + engines: {node: '>=12', npm: '>=6'} + md5@2.3.0: resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==} @@ -11423,6 +11437,10 @@ packages: mlly@1.7.4: resolution: {integrity: sha512-qmdSIPC4bDJXgZTCR7XosJiNKySV7O215tsPtDN9iEO/7q/76b/ijtgRu/+epFXSJhijtTCCGp3DWS549P3xKw==} + mmdb-lib@2.2.1: + resolution: {integrity: sha512-DXO4L9W+08T+A7h5+xdT32l7IMot8z7WOH+7C1Maol571PnktQ8un7Ni4CyPFp4H+vht/FDA5/tpjRvWMFQDMw==} + engines: {node: '>=10', npm: '>=6'} + module-details-from-path@1.0.3: resolution: {integrity: sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==} @@ -13448,6 +13466,10 @@ packages: timm@1.7.1: resolution: {integrity: sha512-IjZc9KIotudix8bMaBW6QvMuq64BrJWFs1+4V0lXwWGQZwH+LnX87doAYhem4caOEusRP9/g6jVDQmZ8XOk1nw==} + tiny-lru@11.3.3: + resolution: {integrity: sha512-/ShxBZOgHXDdZi7FxajcsH0MfcBqwP+t7i4T3PGjI//NUA5aCpC7cB9bbdAYrAeQLBUTJfg2rk191fzZGeo7DA==} + engines: {node: '>=12'} + tiny-typed-emitter@2.1.0: resolution: {integrity: sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==} @@ -23370,6 +23392,7 @@ snapshots: '@types/cors': 2.8.17 '@types/express': 4.17.21 '@types/jest': 29.5.12 + '@types/maxmind': 2.0.5 '@types/node': 22.15.29 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.3))(eslint@8.56.0)(typescript@5.8.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.8.3) @@ -23384,6 +23407,7 @@ snapshots: eslint-plugin-promise: 6.1.1(eslint@8.56.0) express: 4.21.2 jest: 29.7.0(@types/node@22.15.29)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.8.3)) + maxmind: 4.3.28 prettier: 3.2.5 ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@22.15.29)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.8.3)))(typescript@5.8.3) ts-node: 10.9.2(@types/node@22.15.29)(typescript@5.8.3) @@ -30481,6 +30505,10 @@ snapshots: '@types/linkify-it': 5.0.0 '@types/mdurl': 2.0.0 + '@types/maxmind@2.0.5': + dependencies: + maxmind: 4.3.28 + '@types/mdurl@1.0.5': {} '@types/mdurl@2.0.0': {} @@ -35776,6 +35804,11 @@ snapshots: escape-string-regexp: 4.0.0 optional: true + maxmind@4.3.28: + dependencies: + mmdb-lib: 2.2.1 + tiny-lru: 11.3.3 + md5@2.3.0: dependencies: charenc: 0.0.2 @@ -35988,6 +36021,8 @@ snapshots: pkg-types: 1.3.1 ufo: 1.5.4 + mmdb-lib@2.2.1: {} + module-details-from-path@1.0.3: {} moment@2.30.1: {} @@ -38260,6 +38295,8 @@ snapshots: timm@1.7.1: {} + tiny-lru@11.3.3: {} + tiny-typed-emitter@2.1.0: {} tinycolor2@1.6.0: {} diff --git a/packages/analytics-providers/src/utils.ts b/packages/analytics-providers/src/utils.ts index fc81309eb4..c2ea002081 100644 --- a/packages/analytics-providers/src/utils.ts +++ b/packages/analytics-providers/src/utils.ts @@ -79,13 +79,30 @@ function getDeviceType (type: string | undefined | null): string { export function collectEventMetadata (properties: Record = {}): Record { const trackingParams = getUrlTrackingParams() - const referrer = document.referrer === '' ? '$direct' : document.referrer + + const referrer = (() => { + if (document.referrer === '') return '$direct' + + try { + const currentDomain = window.location.hostname + const referrerDomain = new URL(document.referrer).hostname + + if (currentDomain === referrerDomain) return '$direct' + + return document.referrer + } catch { + return document.referrer + } + })() const referringDomain = referrer !== '$direct' ? new URL(referrer).hostname : '$direct' const now = new Date() const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone const timezoneOffset = -now.getTimezoneOffset() + const browserLanguage = navigator.language !== '' ? navigator.language : 'en-US' + const browserLanguagePrefix = browserLanguage.split('-')[0] !== '' ? browserLanguage.split('-')[0] : 'en' + return { ...properties, $timestamp: new Date().toISOString(), @@ -110,6 +127,8 @@ export function collectEventMetadata (properties: Record = {}): Rec title: document.title, $timezone: timezone, $timezone_offset: timezoneOffset, + $browser_language: browserLanguage, + $browser_language_prefix: browserLanguagePrefix, ...trackingParams } } diff --git a/plugins/workbench-resources/src/connect.ts b/plugins/workbench-resources/src/connect.ts index 6253cf35cf..f5dd528433 100644 --- a/plugins/workbench-resources/src/connect.ts +++ b/plugins/workbench-resources/src/connect.ts @@ -427,7 +427,7 @@ export async function connect (title: string): Promise { } Analytics.setUser(data.social_id, data) - Analytics.setWorkspace(workspace.name, guestRole) + Analytics.setWorkspace(workspace.url, guestRole) Analytics.handleEvent(WorkbenchEvents.Connect) console.log('Logged in with account: ', me) setCurrentAccount(me) diff --git a/services/analytics-collector/pod-analytics-collector/Dockerfile b/services/analytics-collector/pod-analytics-collector/Dockerfile index 6c5815a7e0..b2af8a2e9b 100644 --- a/services/analytics-collector/pod-analytics-collector/Dockerfile +++ b/services/analytics-collector/pod-analytics-collector/Dockerfile @@ -3,5 +3,15 @@ WORKDIR /usr/src/app COPY bundle/bundle.js ./ +# Create geodb directory and optionally copy MaxMind databases +RUN mkdir -p /usr/src/geodb +# Copy entire geodb directory +COPY geodb/ /usr/src/geodb/ + +# Log what databases are available +RUN echo "GeoIP databases status:" && \ + (ls -la /usr/src/geodb/*.mmdb 2>/dev/null && echo "MaxMind databases found") || \ + echo "No MaxMind databases found - geolocation will be disabled" + EXPOSE 4007 CMD [ "node", "bundle.js" ] diff --git a/services/analytics-collector/pod-analytics-collector/geodb/GeoLite2-City.mmdb b/services/analytics-collector/pod-analytics-collector/geodb/GeoLite2-City.mmdb new file mode 100644 index 0000000000..eee9de5299 Binary files /dev/null and b/services/analytics-collector/pod-analytics-collector/geodb/GeoLite2-City.mmdb differ diff --git a/services/analytics-collector/pod-analytics-collector/geodb/GeoLite2-Country.mmdb b/services/analytics-collector/pod-analytics-collector/geodb/GeoLite2-Country.mmdb new file mode 100644 index 0000000000..819c34409f Binary files /dev/null and b/services/analytics-collector/pod-analytics-collector/geodb/GeoLite2-Country.mmdb differ diff --git a/services/analytics-collector/pod-analytics-collector/package.json b/services/analytics-collector/pod-analytics-collector/package.json index 54c88576b6..91da1f5799 100644 --- a/services/analytics-collector/pod-analytics-collector/package.json +++ b/services/analytics-collector/pod-analytics-collector/package.json @@ -34,6 +34,7 @@ "@tsconfig/node16": "^1.0.4", "@types/cors": "^2.8.12", "@types/express": "^4.17.13", + "@types/maxmind": "^2.0.5", "@types/node": "^22.15.29", "@typescript-eslint/eslint-plugin": "^6.11.0", "@typescript-eslint/parser": "^6.11.0", @@ -63,6 +64,7 @@ "@hcengineering/server-token": "^0.6.11", "cors": "^2.8.5", "dotenv": "~16.0.0", - "express": "^4.21.2" + "express": "^4.21.2", + "maxmind": "^4.3.6" } } diff --git a/services/analytics-collector/pod-analytics-collector/src/geoip.ts b/services/analytics-collector/pod-analytics-collector/src/geoip.ts new file mode 100644 index 0000000000..742cccb194 --- /dev/null +++ b/services/analytics-collector/pod-analytics-collector/src/geoip.ts @@ -0,0 +1,365 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import * as maxmind from 'maxmind' +import * as path from 'path' +import * as fs from 'fs' +import { type MeasureContext } from '@hcengineering/core' + +// MaxMind reader instances +let cityReader: maxmind.Reader | null = null +let countryReader: maxmind.Reader | null = null +let loadingPromise: Promise | null = null +let isInitialized = false +let logContext: MeasureContext | null = null + +// Database paths - can be overridden via environment variables +const DB_DIR = process.env.GEOIP_DB_DIR ?? path.join(__dirname, '../geodb') +const CITY_DB_PATH = process.env.GEOIP_CITY_DB ?? path.join(DB_DIR, 'GeoLite2-City.mmdb') +const COUNTRY_DB_PATH = process.env.GEOIP_COUNTRY_DB ?? path.join(DB_DIR, 'GeoLite2-Country.mmdb') + +/** + * Set logging context for structured logging + */ +export function setGeoipLogContext (ctx: MeasureContext): void { + logContext = ctx +} + +export const geoFieldMapping = { + country: '$geoip_country_code', + country_name: '$geoip_country_name', + city: '$geoip_city_name', + latitude: '$geoip_latitude', + longitude: '$geoip_longitude', + timezone_geoip: '$geoip_time_zone', + continent_code: '$geoip_continent_code', + continent_name: '$geoip_continent_name', + subdivision_1_code: '$geoip_subdivision_1_code', + subdivision_1_name: '$geoip_subdivision_1_name', + postal_code: '$geoip_postal_code', + accuracy_radius: '$geoip_accuracy_radius' +} + +export interface GeoLocationData { + // Basic geodata + country: string | null + country_name: string | null + city: string | null + latitude: number | null + longitude: number | null + timezone_geoip: string | null + + // Additional PostHog data + continent_code: string | null + continent_name: string | null + subdivision_1_code: string | null + subdivision_1_name: string | null + postal_code: string | null + accuracy_radius: number | null +} + +// Initialize MaxMind databases +async function initializeMaxMind (): Promise { + if (loadingPromise !== null) { + await loadingPromise + return + } + + loadingPromise = (async () => { + try { + const log = logContext ?? { info: console.log, warn: console.warn, error: console.error } + log.info('Initializing MaxMind GeoIP databases...', { dbDir: DB_DIR }) + + // Check if database files exist + const cityDbExists = fs.existsSync(CITY_DB_PATH) + const countryDbExists = fs.existsSync(COUNTRY_DB_PATH) + + if (!cityDbExists && !countryDbExists) { + log.warn('MaxMind database files not found - IP geolocation disabled', { + cityPath: CITY_DB_PATH, + countryPath: COUNTRY_DB_PATH + }) + isInitialized = true + return + } + + // Load City database (priority - has more detailed info) + if (cityDbExists) { + try { + cityReader = await maxmind.open(CITY_DB_PATH) + log.info('GeoLite2-City database loaded successfully', { path: CITY_DB_PATH }) + } catch (error) { + log.error('Failed to load GeoLite2-City database', { path: CITY_DB_PATH, error }) + } + } + + // Load Country database (fallback) + if (countryDbExists && cityReader === null) { + try { + countryReader = await maxmind.open(COUNTRY_DB_PATH) + log.info('GeoLite2-Country database loaded successfully', { path: COUNTRY_DB_PATH }) + } catch (error) { + log.error('Failed to load GeoLite2-Country database', { path: COUNTRY_DB_PATH, error }) + } + } + + if (cityReader !== null || countryReader !== null) { + const databases = [] + if (cityReader !== null) databases.push('GeoLite2-City') + if (countryReader !== null) databases.push('GeoLite2-Country') + log.info('MaxMind GeoIP ready - IP geolocation enabled', { databases }) + } else { + log.warn('No MaxMind databases could be loaded - IP geolocation disabled') + } + + isInitialized = true + } catch (error) { + const log = logContext ?? { error: console.error } + log.error('Failed to initialize MaxMind', { error }) + isInitialized = true + } + })() + + await loadingPromise +} + +/** + * Gets the real client IP address from request headers, + * taking into account proxy servers and load balancers + */ +export function getClientIp (req: any): string { + // Check various headers that may contain the real IP + const forwardedFor = req.headers['x-forwarded-for'] + if (forwardedFor != null && typeof forwardedFor === 'string') { + // X-Forwarded-For may contain multiple IPs separated by commas + const ips = forwardedFor.split(',').map((ip: string) => ip.trim()) + // Find first non-private IP (original client) + for (const ip of ips) { + const normalizedIp = normalizeIp(ip) + if (isValidIp(normalizedIp) && !isPrivateIp(normalizedIp)) { + return normalizedIp + } + } + // If no public IP found, return first IP + return normalizeIp(ips[0]) + } + + // Cloudflare (higher priority than generic headers) + const cfConnectingIp = req.headers['cf-connecting-ip'] + if (cfConnectingIp != null && typeof cfConnectingIp === 'string') { + return normalizeIp(cfConnectingIp) + } + + const realIp = req.headers['x-real-ip'] + if (realIp != null && typeof realIp === 'string') { + return normalizeIp(realIp) + } + + const xClientIp = req.headers['x-client-ip'] + if (xClientIp != null && typeof xClientIp === 'string') { + return normalizeIp(xClientIp) + } + + // Additional headers for completeness + const trueClientIp = req.headers['true-client-ip'] + if (trueClientIp != null && typeof trueClientIp === 'string') { + return normalizeIp(trueClientIp) + } + + const xOriginalForwardedFor = req.headers['x-original-forwarded-for'] + if (xOriginalForwardedFor != null && typeof xOriginalForwardedFor === 'string') { + return normalizeIp(xOriginalForwardedFor) + } + + // If no headers are present, use standard req.ip + const ip = req.ip ?? req.connection?.remoteAddress ?? req.socket?.remoteAddress ?? '127.0.0.1' + return normalizeIp(ip) +} + +/** + * Normalize IP address (convert IPv6-mapped IPv4 to plain IPv4) + */ +function normalizeIp (ip: string): string { + if (ip == null) return '127.0.0.1' + + // Convert IPv6-mapped IPv4 addresses (::ffff:192.168.1.1) to IPv4 (192.168.1.1) + if (ip.startsWith('::ffff:')) { + return ip.substring(7) + } + + // Remove IPv6 zone identifier if present + if (ip.includes('%')) { + return ip.split('%')[0] + } + + return ip.trim() +} + +/** + * Validates if string is a proper IP address (IPv4 or IPv6) + */ +function isValidIp (ip: string): boolean { + if (ip == null || ip === '') return false + + // IPv4 regex + const ipv4Regex = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/ + if (ipv4Regex.test(ip)) return true + + // IPv6 regex (simplified - covers most cases) + const ipv6Regex = /^(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$|^::1$|^::$/ + if (ipv6Regex.test(ip)) return true + + return false +} + +/** + * Checks if IP address is local/private + */ +function isPrivateIp (ip: string): boolean { + if (ip == null || ip === '127.0.0.1' || ip === '::1' || ip === 'localhost') { + return true + } + + // IPv4 private ranges + const ipv4PrivateRanges = [ + /^10\./, // 10.0.0.0/8 + /^172\.(1[6-9]|2[0-9]|3[01])\./, // 172.16.0.0/12 + /^192\.168\./, // 192.168.0.0/16 + /^169\.254\./, // 169.254.0.0/16 (Link-local) + /^127\./ // 127.0.0.0/8 (Loopback) + ] + + return ipv4PrivateRanges.some((range) => range.test(ip)) +} + +/** + * Gets geolocation data by IP address using MaxMind GeoLite2 databases + */ +export async function getGeoLocationFromIp (req: any): Promise { + const ip = getClientIp(req) + + // If IP is private or local, return null data + if (isPrivateIp(ip)) { + return createNullGeoData() + } + + try { + // Ensure MaxMind is initialized + if (!isInitialized) { + await initializeMaxMind() + } + + // Try City database first (has more detailed info) + if (cityReader !== null) { + const geo = cityReader.get(ip) + if (geo !== null) { + const geoData = extractGeoDataFromCity(geo) + + // Check if we actually got useful data + const hasGeoData = geoData.country !== null || geoData.city !== null || geoData.latitude !== null + + if (hasGeoData) { + return geoData + } + } + } + + // Fallback to Country database + if (countryReader !== null) { + const geo = countryReader.get(ip) + if (geo !== null) { + const geoData = extractGeoDataFromCountry(geo) + + // Check if we actually got useful data + const hasGeoData = geoData.country !== null + + if (hasGeoData) { + return geoData + } + } + } + + return createNullGeoData() + } catch (error) { + const log = logContext ?? { warn: console.warn } + log.warn('Failed to get geo location for IP', { ip, error }) + return createNullGeoData() + } +} + +/** + * Extract geo data from MaxMind City response + */ +function extractGeoDataFromCity (geo: maxmind.CityResponse): GeoLocationData { + return { + country: geo.country?.iso_code ?? null, + country_name: geo.country?.names?.en ?? null, + city: geo.city?.names?.en ?? null, + latitude: geo.location?.latitude ?? null, + longitude: geo.location?.longitude ?? null, + timezone_geoip: geo.location?.time_zone ?? null, + continent_code: geo.continent?.code ?? null, + continent_name: geo.continent?.names?.en ?? null, + subdivision_1_code: geo.subdivisions?.[0]?.iso_code ?? null, + subdivision_1_name: geo.subdivisions?.[0]?.names?.en ?? null, + postal_code: geo.postal?.code ?? null, + accuracy_radius: geo.location?.accuracy_radius ?? null + } +} + +/** + * Extract geo data from MaxMind Country response + */ +function extractGeoDataFromCountry (geo: maxmind.CountryResponse): GeoLocationData { + return { + country: geo.country?.iso_code ?? null, + country_name: geo.country?.names?.en ?? null, + city: null, // Country DB doesn't have city info + latitude: null, + longitude: null, + timezone_geoip: null, + continent_code: geo.continent?.code ?? null, + continent_name: geo.continent?.names?.en ?? null, + subdivision_1_code: null, + subdivision_1_name: null, + postal_code: null, + accuracy_radius: null + } +} + +/** + * Creates null geo data object + */ +function createNullGeoData (): GeoLocationData { + return { + country: null, + country_name: null, + city: null, + latitude: null, + longitude: null, + timezone_geoip: null, + continent_code: null, + continent_name: null, + subdivision_1_code: null, + subdivision_1_name: null, + postal_code: null, + accuracy_radius: null + } +} + +/** + * Initialize MaxMind on module load + */ +initializeMaxMind().catch(console.error) diff --git a/services/analytics-collector/pod-analytics-collector/src/main.ts b/services/analytics-collector/pod-analytics-collector/src/main.ts index 08bce2b2a5..df6184dee2 100644 --- a/services/analytics-collector/pod-analytics-collector/src/main.ts +++ b/services/analytics-collector/pod-analytics-collector/src/main.ts @@ -25,6 +25,7 @@ import { join } from 'path' import { initStatisticsContext } from '@hcengineering/server-core' import config from './config' import { createServer, listen } from './server' +import { setGeoipLogContext } from './geoip' configureAnalytics('analytics-collector', process.env.VERSION ?? '0.7.0') const ctx = initStatisticsContext('analytics-collector', { @@ -48,6 +49,9 @@ export const main = async (): Promise => { setMetadata(serverClient.metadata.Endpoint, config.AccountsUrl) setMetadata(serverClient.metadata.UserAgent, config.ServiceID) + // Set context for geoip logging + setGeoipLogContext(ctx) + ctx.info('Analytics service started', { accountsUrl: config.AccountsUrl }) diff --git a/services/analytics-collector/pod-analytics-collector/src/server.ts b/services/analytics-collector/pod-analytics-collector/src/server.ts index 87751c2ddc..2052cb7402 100644 --- a/services/analytics-collector/pod-analytics-collector/src/server.ts +++ b/services/analytics-collector/pod-analytics-collector/src/server.ts @@ -21,6 +21,7 @@ import { AnalyticEvent, AnalyticEventType } from '@hcengineering/analytics-colle import { extractToken } from '@hcengineering/server-client' import config from './config' import { ApiError } from './error' +import { getGeoLocationFromIp, getClientIp, geoFieldMapping } from './geoip' type AsyncRequestHandler = (req: Request, res: Response, token: Token, next: NextFunction) => Promise @@ -110,7 +111,7 @@ function getRecordsByType (event: AnalyticEvent): Record { } } -function preparePostHogEvent (event: AnalyticEvent, req: Request): Record { +async function preparePostHogEvent (event: AnalyticEvent, req: Request): Promise> { let errorMessage = 'Unknown error' let errorType = 'Error' @@ -137,9 +138,18 @@ function preparePostHogEvent (event: AnalyticEvent, req: Request): Record = { alias: event.properties.alias, @@ -239,9 +249,11 @@ export function createServer (): Express { console.log(`Received batch: ${events.length} events, ${payloadSize} bytes`) - const posthogEvents = events.map((event) => { - return preparePostHogEvent(event, req) - }) + const posthogEvents = await Promise.all( + events.map(async (event) => { + return await preparePostHogEvent(event, req) + }) + ) const payload = { api_key: config.PostHogAPI,