mirror of
https://github.com/hcengineering/platform.git
synced 2026-09-10 03:37:43 +02:00
Add IP geolocation support to analytics collector (#9632)
Signed-off-by: Alexander Platov <alexander.platov@hardcoreeng.com>
This commit is contained in:
Generated
+40
-3
@@ -1678,6 +1678,9 @@ importers:
|
||||
'@types/markdown-it':
|
||||
specifier: ~13.0.0
|
||||
version: 13.0.8
|
||||
'@types/maxmind':
|
||||
specifier: ^2.0.5
|
||||
version: 2.0.5
|
||||
'@types/mime':
|
||||
specifier: ^3.0.1
|
||||
version: 3.0.4
|
||||
@@ -2074,6 +2077,9 @@ importers:
|
||||
markdown-it:
|
||||
specifier: ^14.0.0
|
||||
version: 14.0.0
|
||||
maxmind:
|
||||
specifier: ^4.3.6
|
||||
version: 4.3.28
|
||||
mermaid:
|
||||
specifier: ~11.4.1
|
||||
version: 11.4.1
|
||||
@@ -5271,7 +5277,7 @@ packages:
|
||||
version: 0.0.0
|
||||
|
||||
'@rush-temp/pod-analytics-collector@file:projects/pod-analytics-collector.tgz':
|
||||
resolution: {integrity: sha512-YsXK0lM7Spe4gaV61mzcLnesPCnB3kxryetFQJEFTNEh+azKvSQTXGvyozSPJL8OriZDNMC/NqsqXqO2/fFLPg==, tarball: file:projects/pod-analytics-collector.tgz}
|
||||
resolution: {integrity: sha512-NyLI9rsK/JB0vx2gpodNNTmfT+ApGsmY8QKkPs1T3cD+zOx15AlsSty43i9f7feiCwCBA2JbfgK60mb5QePr3w==, tarball: file:projects/pod-analytics-collector.tgz}
|
||||
version: 0.0.0
|
||||
|
||||
'@rush-temp/pod-backup@file:projects/pod-backup.tgz':
|
||||
@@ -5351,7 +5357,7 @@ packages:
|
||||
version: 0.0.0
|
||||
|
||||
'@rush-temp/pod-process@file:projects/pod-process.tgz':
|
||||
resolution: {integrity: sha512-CX94IUHO3sQtTelqy4CBPydusE7MTCJ/l26hsifScPSCKEZTc4+2Qz3aZSwo2SMLerH1ZrXATpDeI4l2sgOT0A==, tarball: file:projects/pod-process.tgz}
|
||||
resolution: {integrity: sha512-lM7eqcK1IQzWvzrzQ2Iu1H7e1ETVWovBtvpXPpNMMyr6ied0Hbj60PgxUZsTE5ZwKcGv0ZFDfP8HlJSPXrG2zg==, tarball: file:projects/pod-process.tgz}
|
||||
version: 0.0.0
|
||||
|
||||
'@rush-temp/pod-server@file:projects/pod-server.tgz':
|
||||
@@ -5727,7 +5733,7 @@ packages:
|
||||
version: 0.0.0
|
||||
|
||||
'@rush-temp/server-process@file:projects/server-process.tgz':
|
||||
resolution: {integrity: sha512-pKJ3cCSJOnjIfOQwJp1rHv6WDYuFLhp+J6/O5t1+evZ0YB8/ynv1lb9PPbg9rjGwwGQ41AQlaTn9cCDhfZdAOg==, tarball: file:projects/server-process.tgz}
|
||||
resolution: {integrity: sha512-0FfWPqe+aaK/TXdXElATagm3zoh7G3p8k269zjs34NEpWNHOwK4NWypuh82G8YB3D9CgtF/jKov2KuQRPms8wA==, tarball: file:projects/server-process.tgz}
|
||||
version: 0.0.0
|
||||
|
||||
'@rush-temp/server-recruit-resources@file:projects/server-recruit-resources.tgz':
|
||||
@@ -6912,6 +6918,10 @@ packages:
|
||||
'@types/markdown-it@14.1.2':
|
||||
resolution: {integrity: sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==}
|
||||
|
||||
'@types/maxmind@2.0.5':
|
||||
resolution: {integrity: sha512-fGqeOLsZI189WqKrAvaGfLzaLg5T8NNenAHrvmRuIg6lLfQCMICw4kg9wxyxzLAJnxwN89CJhTq6DZQ/w1K5hw==}
|
||||
deprecated: This is a stub types definition. maxmind provides its own type definitions, so you do not need this installed.
|
||||
|
||||
'@types/mdurl@1.0.5':
|
||||
resolution: {integrity: sha512-6L6VymKTzYSrEf4Nev4Xa1LCHKrlTlYCBMTlQKFuddo1CvQcE52I0mwfOJayueUC7MJuXOeHTcIU683lzd0cUA==}
|
||||
|
||||
@@ -11220,6 +11230,10 @@ packages:
|
||||
resolution: {integrity: sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
maxmind@4.3.28:
|
||||
resolution: {integrity: sha512-K3PcTVjhrSU6xzY7niQf/CHPmx/qzkMIpMumd3x0JXMkIDJMerL4LY9RsEhArHb4T3IK0NBxQcUlwjsxRm9rIw==}
|
||||
engines: {node: '>=12', npm: '>=6'}
|
||||
|
||||
md5@2.3.0:
|
||||
resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==}
|
||||
|
||||
@@ -11423,6 +11437,10 @@ packages:
|
||||
mlly@1.7.4:
|
||||
resolution: {integrity: sha512-qmdSIPC4bDJXgZTCR7XosJiNKySV7O215tsPtDN9iEO/7q/76b/ijtgRu/+epFXSJhijtTCCGp3DWS549P3xKw==}
|
||||
|
||||
mmdb-lib@2.2.1:
|
||||
resolution: {integrity: sha512-DXO4L9W+08T+A7h5+xdT32l7IMot8z7WOH+7C1Maol571PnktQ8un7Ni4CyPFp4H+vht/FDA5/tpjRvWMFQDMw==}
|
||||
engines: {node: '>=10', npm: '>=6'}
|
||||
|
||||
module-details-from-path@1.0.3:
|
||||
resolution: {integrity: sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==}
|
||||
|
||||
@@ -13448,6 +13466,10 @@ packages:
|
||||
timm@1.7.1:
|
||||
resolution: {integrity: sha512-IjZc9KIotudix8bMaBW6QvMuq64BrJWFs1+4V0lXwWGQZwH+LnX87doAYhem4caOEusRP9/g6jVDQmZ8XOk1nw==}
|
||||
|
||||
tiny-lru@11.3.3:
|
||||
resolution: {integrity: sha512-/ShxBZOgHXDdZi7FxajcsH0MfcBqwP+t7i4T3PGjI//NUA5aCpC7cB9bbdAYrAeQLBUTJfg2rk191fzZGeo7DA==}
|
||||
engines: {node: '>=12'}
|
||||
|
||||
tiny-typed-emitter@2.1.0:
|
||||
resolution: {integrity: sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==}
|
||||
|
||||
@@ -23370,6 +23392,7 @@ snapshots:
|
||||
'@types/cors': 2.8.17
|
||||
'@types/express': 4.17.21
|
||||
'@types/jest': 29.5.12
|
||||
'@types/maxmind': 2.0.5
|
||||
'@types/node': 22.15.29
|
||||
'@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.56.0)(typescript@5.8.3))(eslint@8.56.0)(typescript@5.8.3)
|
||||
'@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.8.3)
|
||||
@@ -23384,6 +23407,7 @@ snapshots:
|
||||
eslint-plugin-promise: 6.1.1(eslint@8.56.0)
|
||||
express: 4.21.2
|
||||
jest: 29.7.0(@types/node@22.15.29)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.8.3))
|
||||
maxmind: 4.3.28
|
||||
prettier: 3.2.5
|
||||
ts-jest: 29.1.2(@babel/core@7.23.9)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.23.9))(esbuild@0.24.2)(jest@29.7.0(@types/node@22.15.29)(ts-node@10.9.2(@types/node@22.15.29)(typescript@5.8.3)))(typescript@5.8.3)
|
||||
ts-node: 10.9.2(@types/node@22.15.29)(typescript@5.8.3)
|
||||
@@ -30481,6 +30505,10 @@ snapshots:
|
||||
'@types/linkify-it': 5.0.0
|
||||
'@types/mdurl': 2.0.0
|
||||
|
||||
'@types/maxmind@2.0.5':
|
||||
dependencies:
|
||||
maxmind: 4.3.28
|
||||
|
||||
'@types/mdurl@1.0.5': {}
|
||||
|
||||
'@types/mdurl@2.0.0': {}
|
||||
@@ -35776,6 +35804,11 @@ snapshots:
|
||||
escape-string-regexp: 4.0.0
|
||||
optional: true
|
||||
|
||||
maxmind@4.3.28:
|
||||
dependencies:
|
||||
mmdb-lib: 2.2.1
|
||||
tiny-lru: 11.3.3
|
||||
|
||||
md5@2.3.0:
|
||||
dependencies:
|
||||
charenc: 0.0.2
|
||||
@@ -35988,6 +36021,8 @@ snapshots:
|
||||
pkg-types: 1.3.1
|
||||
ufo: 1.5.4
|
||||
|
||||
mmdb-lib@2.2.1: {}
|
||||
|
||||
module-details-from-path@1.0.3: {}
|
||||
|
||||
moment@2.30.1: {}
|
||||
@@ -38260,6 +38295,8 @@ snapshots:
|
||||
|
||||
timm@1.7.1: {}
|
||||
|
||||
tiny-lru@11.3.3: {}
|
||||
|
||||
tiny-typed-emitter@2.1.0: {}
|
||||
|
||||
tinycolor2@1.6.0: {}
|
||||
|
||||
@@ -79,13 +79,30 @@ function getDeviceType (type: string | undefined | null): string {
|
||||
|
||||
export function collectEventMetadata (properties: Record<string, any> = {}): Record<string, any> {
|
||||
const trackingParams = getUrlTrackingParams()
|
||||
const referrer = document.referrer === '' ? '$direct' : document.referrer
|
||||
|
||||
const referrer = (() => {
|
||||
if (document.referrer === '') return '$direct'
|
||||
|
||||
try {
|
||||
const currentDomain = window.location.hostname
|
||||
const referrerDomain = new URL(document.referrer).hostname
|
||||
|
||||
if (currentDomain === referrerDomain) return '$direct'
|
||||
|
||||
return document.referrer
|
||||
} catch {
|
||||
return document.referrer
|
||||
}
|
||||
})()
|
||||
const referringDomain = referrer !== '$direct' ? new URL(referrer).hostname : '$direct'
|
||||
|
||||
const now = new Date()
|
||||
const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone
|
||||
const timezoneOffset = -now.getTimezoneOffset()
|
||||
|
||||
const browserLanguage = navigator.language !== '' ? navigator.language : 'en-US'
|
||||
const browserLanguagePrefix = browserLanguage.split('-')[0] !== '' ? browserLanguage.split('-')[0] : 'en'
|
||||
|
||||
return {
|
||||
...properties,
|
||||
$timestamp: new Date().toISOString(),
|
||||
@@ -110,6 +127,8 @@ export function collectEventMetadata (properties: Record<string, any> = {}): Rec
|
||||
title: document.title,
|
||||
$timezone: timezone,
|
||||
$timezone_offset: timezoneOffset,
|
||||
$browser_language: browserLanguage,
|
||||
$browser_language_prefix: browserLanguagePrefix,
|
||||
...trackingParams
|
||||
}
|
||||
}
|
||||
|
||||
@@ -427,7 +427,7 @@ export async function connect (title: string): Promise<Client | undefined> {
|
||||
}
|
||||
|
||||
Analytics.setUser(data.social_id, data)
|
||||
Analytics.setWorkspace(workspace.name, guestRole)
|
||||
Analytics.setWorkspace(workspace.url, guestRole)
|
||||
Analytics.handleEvent(WorkbenchEvents.Connect)
|
||||
console.log('Logged in with account: ', me)
|
||||
setCurrentAccount(me)
|
||||
|
||||
@@ -3,5 +3,15 @@ WORKDIR /usr/src/app
|
||||
|
||||
COPY bundle/bundle.js ./
|
||||
|
||||
# Create geodb directory and optionally copy MaxMind databases
|
||||
RUN mkdir -p /usr/src/geodb
|
||||
# Copy entire geodb directory
|
||||
COPY geodb/ /usr/src/geodb/
|
||||
|
||||
# Log what databases are available
|
||||
RUN echo "GeoIP databases status:" && \
|
||||
(ls -la /usr/src/geodb/*.mmdb 2>/dev/null && echo "MaxMind databases found") || \
|
||||
echo "No MaxMind databases found - geolocation will be disabled"
|
||||
|
||||
EXPOSE 4007
|
||||
CMD [ "node", "bundle.js" ]
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 59 MiB |
Binary file not shown.
@@ -34,6 +34,7 @@
|
||||
"@tsconfig/node16": "^1.0.4",
|
||||
"@types/cors": "^2.8.12",
|
||||
"@types/express": "^4.17.13",
|
||||
"@types/maxmind": "^2.0.5",
|
||||
"@types/node": "^22.15.29",
|
||||
"@typescript-eslint/eslint-plugin": "^6.11.0",
|
||||
"@typescript-eslint/parser": "^6.11.0",
|
||||
@@ -63,6 +64,7 @@
|
||||
"@hcengineering/server-token": "^0.6.11",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "~16.0.0",
|
||||
"express": "^4.21.2"
|
||||
"express": "^4.21.2",
|
||||
"maxmind": "^4.3.6"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
//
|
||||
// Copyright © 2025 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import * as maxmind from 'maxmind'
|
||||
import * as path from 'path'
|
||||
import * as fs from 'fs'
|
||||
import { type MeasureContext } from '@hcengineering/core'
|
||||
|
||||
// MaxMind reader instances
|
||||
let cityReader: maxmind.Reader<maxmind.CityResponse> | null = null
|
||||
let countryReader: maxmind.Reader<maxmind.CountryResponse> | null = null
|
||||
let loadingPromise: Promise<void> | null = null
|
||||
let isInitialized = false
|
||||
let logContext: MeasureContext | null = null
|
||||
|
||||
// Database paths - can be overridden via environment variables
|
||||
const DB_DIR = process.env.GEOIP_DB_DIR ?? path.join(__dirname, '../geodb')
|
||||
const CITY_DB_PATH = process.env.GEOIP_CITY_DB ?? path.join(DB_DIR, 'GeoLite2-City.mmdb')
|
||||
const COUNTRY_DB_PATH = process.env.GEOIP_COUNTRY_DB ?? path.join(DB_DIR, 'GeoLite2-Country.mmdb')
|
||||
|
||||
/**
|
||||
* Set logging context for structured logging
|
||||
*/
|
||||
export function setGeoipLogContext (ctx: MeasureContext): void {
|
||||
logContext = ctx
|
||||
}
|
||||
|
||||
export const geoFieldMapping = {
|
||||
country: '$geoip_country_code',
|
||||
country_name: '$geoip_country_name',
|
||||
city: '$geoip_city_name',
|
||||
latitude: '$geoip_latitude',
|
||||
longitude: '$geoip_longitude',
|
||||
timezone_geoip: '$geoip_time_zone',
|
||||
continent_code: '$geoip_continent_code',
|
||||
continent_name: '$geoip_continent_name',
|
||||
subdivision_1_code: '$geoip_subdivision_1_code',
|
||||
subdivision_1_name: '$geoip_subdivision_1_name',
|
||||
postal_code: '$geoip_postal_code',
|
||||
accuracy_radius: '$geoip_accuracy_radius'
|
||||
}
|
||||
|
||||
export interface GeoLocationData {
|
||||
// Basic geodata
|
||||
country: string | null
|
||||
country_name: string | null
|
||||
city: string | null
|
||||
latitude: number | null
|
||||
longitude: number | null
|
||||
timezone_geoip: string | null
|
||||
|
||||
// Additional PostHog data
|
||||
continent_code: string | null
|
||||
continent_name: string | null
|
||||
subdivision_1_code: string | null
|
||||
subdivision_1_name: string | null
|
||||
postal_code: string | null
|
||||
accuracy_radius: number | null
|
||||
}
|
||||
|
||||
// Initialize MaxMind databases
|
||||
async function initializeMaxMind (): Promise<void> {
|
||||
if (loadingPromise !== null) {
|
||||
await loadingPromise
|
||||
return
|
||||
}
|
||||
|
||||
loadingPromise = (async () => {
|
||||
try {
|
||||
const log = logContext ?? { info: console.log, warn: console.warn, error: console.error }
|
||||
log.info('Initializing MaxMind GeoIP databases...', { dbDir: DB_DIR })
|
||||
|
||||
// Check if database files exist
|
||||
const cityDbExists = fs.existsSync(CITY_DB_PATH)
|
||||
const countryDbExists = fs.existsSync(COUNTRY_DB_PATH)
|
||||
|
||||
if (!cityDbExists && !countryDbExists) {
|
||||
log.warn('MaxMind database files not found - IP geolocation disabled', {
|
||||
cityPath: CITY_DB_PATH,
|
||||
countryPath: COUNTRY_DB_PATH
|
||||
})
|
||||
isInitialized = true
|
||||
return
|
||||
}
|
||||
|
||||
// Load City database (priority - has more detailed info)
|
||||
if (cityDbExists) {
|
||||
try {
|
||||
cityReader = await maxmind.open<maxmind.CityResponse>(CITY_DB_PATH)
|
||||
log.info('GeoLite2-City database loaded successfully', { path: CITY_DB_PATH })
|
||||
} catch (error) {
|
||||
log.error('Failed to load GeoLite2-City database', { path: CITY_DB_PATH, error })
|
||||
}
|
||||
}
|
||||
|
||||
// Load Country database (fallback)
|
||||
if (countryDbExists && cityReader === null) {
|
||||
try {
|
||||
countryReader = await maxmind.open<maxmind.CountryResponse>(COUNTRY_DB_PATH)
|
||||
log.info('GeoLite2-Country database loaded successfully', { path: COUNTRY_DB_PATH })
|
||||
} catch (error) {
|
||||
log.error('Failed to load GeoLite2-Country database', { path: COUNTRY_DB_PATH, error })
|
||||
}
|
||||
}
|
||||
|
||||
if (cityReader !== null || countryReader !== null) {
|
||||
const databases = []
|
||||
if (cityReader !== null) databases.push('GeoLite2-City')
|
||||
if (countryReader !== null) databases.push('GeoLite2-Country')
|
||||
log.info('MaxMind GeoIP ready - IP geolocation enabled', { databases })
|
||||
} else {
|
||||
log.warn('No MaxMind databases could be loaded - IP geolocation disabled')
|
||||
}
|
||||
|
||||
isInitialized = true
|
||||
} catch (error) {
|
||||
const log = logContext ?? { error: console.error }
|
||||
log.error('Failed to initialize MaxMind', { error })
|
||||
isInitialized = true
|
||||
}
|
||||
})()
|
||||
|
||||
await loadingPromise
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the real client IP address from request headers,
|
||||
* taking into account proxy servers and load balancers
|
||||
*/
|
||||
export function getClientIp (req: any): string {
|
||||
// Check various headers that may contain the real IP
|
||||
const forwardedFor = req.headers['x-forwarded-for']
|
||||
if (forwardedFor != null && typeof forwardedFor === 'string') {
|
||||
// X-Forwarded-For may contain multiple IPs separated by commas
|
||||
const ips = forwardedFor.split(',').map((ip: string) => ip.trim())
|
||||
// Find first non-private IP (original client)
|
||||
for (const ip of ips) {
|
||||
const normalizedIp = normalizeIp(ip)
|
||||
if (isValidIp(normalizedIp) && !isPrivateIp(normalizedIp)) {
|
||||
return normalizedIp
|
||||
}
|
||||
}
|
||||
// If no public IP found, return first IP
|
||||
return normalizeIp(ips[0])
|
||||
}
|
||||
|
||||
// Cloudflare (higher priority than generic headers)
|
||||
const cfConnectingIp = req.headers['cf-connecting-ip']
|
||||
if (cfConnectingIp != null && typeof cfConnectingIp === 'string') {
|
||||
return normalizeIp(cfConnectingIp)
|
||||
}
|
||||
|
||||
const realIp = req.headers['x-real-ip']
|
||||
if (realIp != null && typeof realIp === 'string') {
|
||||
return normalizeIp(realIp)
|
||||
}
|
||||
|
||||
const xClientIp = req.headers['x-client-ip']
|
||||
if (xClientIp != null && typeof xClientIp === 'string') {
|
||||
return normalizeIp(xClientIp)
|
||||
}
|
||||
|
||||
// Additional headers for completeness
|
||||
const trueClientIp = req.headers['true-client-ip']
|
||||
if (trueClientIp != null && typeof trueClientIp === 'string') {
|
||||
return normalizeIp(trueClientIp)
|
||||
}
|
||||
|
||||
const xOriginalForwardedFor = req.headers['x-original-forwarded-for']
|
||||
if (xOriginalForwardedFor != null && typeof xOriginalForwardedFor === 'string') {
|
||||
return normalizeIp(xOriginalForwardedFor)
|
||||
}
|
||||
|
||||
// If no headers are present, use standard req.ip
|
||||
const ip = req.ip ?? req.connection?.remoteAddress ?? req.socket?.remoteAddress ?? '127.0.0.1'
|
||||
return normalizeIp(ip)
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize IP address (convert IPv6-mapped IPv4 to plain IPv4)
|
||||
*/
|
||||
function normalizeIp (ip: string): string {
|
||||
if (ip == null) return '127.0.0.1'
|
||||
|
||||
// Convert IPv6-mapped IPv4 addresses (::ffff:192.168.1.1) to IPv4 (192.168.1.1)
|
||||
if (ip.startsWith('::ffff:')) {
|
||||
return ip.substring(7)
|
||||
}
|
||||
|
||||
// Remove IPv6 zone identifier if present
|
||||
if (ip.includes('%')) {
|
||||
return ip.split('%')[0]
|
||||
}
|
||||
|
||||
return ip.trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates if string is a proper IP address (IPv4 or IPv6)
|
||||
*/
|
||||
function isValidIp (ip: string): boolean {
|
||||
if (ip == null || ip === '') return false
|
||||
|
||||
// IPv4 regex
|
||||
const ipv4Regex = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/
|
||||
if (ipv4Regex.test(ip)) return true
|
||||
|
||||
// IPv6 regex (simplified - covers most cases)
|
||||
const ipv6Regex = /^(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$|^::1$|^::$/
|
||||
if (ipv6Regex.test(ip)) return true
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if IP address is local/private
|
||||
*/
|
||||
function isPrivateIp (ip: string): boolean {
|
||||
if (ip == null || ip === '127.0.0.1' || ip === '::1' || ip === 'localhost') {
|
||||
return true
|
||||
}
|
||||
|
||||
// IPv4 private ranges
|
||||
const ipv4PrivateRanges = [
|
||||
/^10\./, // 10.0.0.0/8
|
||||
/^172\.(1[6-9]|2[0-9]|3[01])\./, // 172.16.0.0/12
|
||||
/^192\.168\./, // 192.168.0.0/16
|
||||
/^169\.254\./, // 169.254.0.0/16 (Link-local)
|
||||
/^127\./ // 127.0.0.0/8 (Loopback)
|
||||
]
|
||||
|
||||
return ipv4PrivateRanges.some((range) => range.test(ip))
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets geolocation data by IP address using MaxMind GeoLite2 databases
|
||||
*/
|
||||
export async function getGeoLocationFromIp (req: any): Promise<GeoLocationData> {
|
||||
const ip = getClientIp(req)
|
||||
|
||||
// If IP is private or local, return null data
|
||||
if (isPrivateIp(ip)) {
|
||||
return createNullGeoData()
|
||||
}
|
||||
|
||||
try {
|
||||
// Ensure MaxMind is initialized
|
||||
if (!isInitialized) {
|
||||
await initializeMaxMind()
|
||||
}
|
||||
|
||||
// Try City database first (has more detailed info)
|
||||
if (cityReader !== null) {
|
||||
const geo = cityReader.get(ip)
|
||||
if (geo !== null) {
|
||||
const geoData = extractGeoDataFromCity(geo)
|
||||
|
||||
// Check if we actually got useful data
|
||||
const hasGeoData = geoData.country !== null || geoData.city !== null || geoData.latitude !== null
|
||||
|
||||
if (hasGeoData) {
|
||||
return geoData
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to Country database
|
||||
if (countryReader !== null) {
|
||||
const geo = countryReader.get(ip)
|
||||
if (geo !== null) {
|
||||
const geoData = extractGeoDataFromCountry(geo)
|
||||
|
||||
// Check if we actually got useful data
|
||||
const hasGeoData = geoData.country !== null
|
||||
|
||||
if (hasGeoData) {
|
||||
return geoData
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return createNullGeoData()
|
||||
} catch (error) {
|
||||
const log = logContext ?? { warn: console.warn }
|
||||
log.warn('Failed to get geo location for IP', { ip, error })
|
||||
return createNullGeoData()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract geo data from MaxMind City response
|
||||
*/
|
||||
function extractGeoDataFromCity (geo: maxmind.CityResponse): GeoLocationData {
|
||||
return {
|
||||
country: geo.country?.iso_code ?? null,
|
||||
country_name: geo.country?.names?.en ?? null,
|
||||
city: geo.city?.names?.en ?? null,
|
||||
latitude: geo.location?.latitude ?? null,
|
||||
longitude: geo.location?.longitude ?? null,
|
||||
timezone_geoip: geo.location?.time_zone ?? null,
|
||||
continent_code: geo.continent?.code ?? null,
|
||||
continent_name: geo.continent?.names?.en ?? null,
|
||||
subdivision_1_code: geo.subdivisions?.[0]?.iso_code ?? null,
|
||||
subdivision_1_name: geo.subdivisions?.[0]?.names?.en ?? null,
|
||||
postal_code: geo.postal?.code ?? null,
|
||||
accuracy_radius: geo.location?.accuracy_radius ?? null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract geo data from MaxMind Country response
|
||||
*/
|
||||
function extractGeoDataFromCountry (geo: maxmind.CountryResponse): GeoLocationData {
|
||||
return {
|
||||
country: geo.country?.iso_code ?? null,
|
||||
country_name: geo.country?.names?.en ?? null,
|
||||
city: null, // Country DB doesn't have city info
|
||||
latitude: null,
|
||||
longitude: null,
|
||||
timezone_geoip: null,
|
||||
continent_code: geo.continent?.code ?? null,
|
||||
continent_name: geo.continent?.names?.en ?? null,
|
||||
subdivision_1_code: null,
|
||||
subdivision_1_name: null,
|
||||
postal_code: null,
|
||||
accuracy_radius: null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates null geo data object
|
||||
*/
|
||||
function createNullGeoData (): GeoLocationData {
|
||||
return {
|
||||
country: null,
|
||||
country_name: null,
|
||||
city: null,
|
||||
latitude: null,
|
||||
longitude: null,
|
||||
timezone_geoip: null,
|
||||
continent_code: null,
|
||||
continent_name: null,
|
||||
subdivision_1_code: null,
|
||||
subdivision_1_name: null,
|
||||
postal_code: null,
|
||||
accuracy_radius: null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize MaxMind on module load
|
||||
*/
|
||||
initializeMaxMind().catch(console.error)
|
||||
@@ -25,6 +25,7 @@ import { join } from 'path'
|
||||
import { initStatisticsContext } from '@hcengineering/server-core'
|
||||
import config from './config'
|
||||
import { createServer, listen } from './server'
|
||||
import { setGeoipLogContext } from './geoip'
|
||||
|
||||
configureAnalytics('analytics-collector', process.env.VERSION ?? '0.7.0')
|
||||
const ctx = initStatisticsContext('analytics-collector', {
|
||||
@@ -48,6 +49,9 @@ export const main = async (): Promise<void> => {
|
||||
setMetadata(serverClient.metadata.Endpoint, config.AccountsUrl)
|
||||
setMetadata(serverClient.metadata.UserAgent, config.ServiceID)
|
||||
|
||||
// Set context for geoip logging
|
||||
setGeoipLogContext(ctx)
|
||||
|
||||
ctx.info('Analytics service started', {
|
||||
accountsUrl: config.AccountsUrl
|
||||
})
|
||||
|
||||
@@ -21,6 +21,7 @@ import { AnalyticEvent, AnalyticEventType } from '@hcengineering/analytics-colle
|
||||
import { extractToken } from '@hcengineering/server-client'
|
||||
import config from './config'
|
||||
import { ApiError } from './error'
|
||||
import { getGeoLocationFromIp, getClientIp, geoFieldMapping } from './geoip'
|
||||
|
||||
type AsyncRequestHandler = (req: Request, res: Response, token: Token, next: NextFunction) => Promise<void>
|
||||
|
||||
@@ -110,7 +111,7 @@ function getRecordsByType (event: AnalyticEvent): Record<string, any> {
|
||||
}
|
||||
}
|
||||
|
||||
function preparePostHogEvent (event: AnalyticEvent, req: Request): Record<string, any> {
|
||||
async function preparePostHogEvent (event: AnalyticEvent, req: Request): Promise<Record<string, any>> {
|
||||
let errorMessage = 'Unknown error'
|
||||
let errorType = 'Error'
|
||||
|
||||
@@ -137,9 +138,18 @@ function preparePostHogEvent (event: AnalyticEvent, req: Request): Record<string
|
||||
for (const [key, value] of Object.entries(recordsByType)) {
|
||||
baseEvent[key] = value
|
||||
}
|
||||
baseEvent.$ip = req.ip
|
||||
|
||||
// Add IP address
|
||||
baseEvent.$ip = getClientIp(req)
|
||||
baseEvent.timestamp = event.properties.$timestamp ?? new Date(event.timestamp).toISOString()
|
||||
|
||||
// Add geolocation data
|
||||
const geoData = await getGeoLocationFromIp(req)
|
||||
|
||||
for (const [sourceField, targetField] of Object.entries(geoFieldMapping)) {
|
||||
baseEvent[targetField] = geoData[sourceField as keyof typeof geoData]
|
||||
}
|
||||
|
||||
if (event.event === AnalyticEventType.SetAlias && typeof event.properties.alias === 'string') {
|
||||
const aliasProperties: Record<string, any> = {
|
||||
alias: event.properties.alias,
|
||||
@@ -239,9 +249,11 @@ export function createServer (): Express {
|
||||
|
||||
console.log(`Received batch: ${events.length} events, ${payloadSize} bytes`)
|
||||
|
||||
const posthogEvents = events.map((event) => {
|
||||
return preparePostHogEvent(event, req)
|
||||
})
|
||||
const posthogEvents = await Promise.all(
|
||||
events.map(async (event) => {
|
||||
return await preparePostHogEvent(event, req)
|
||||
})
|
||||
)
|
||||
|
||||
const payload = {
|
||||
api_key: config.PostHogAPI,
|
||||
|
||||
Reference in New Issue
Block a user