From 83b9fdbf9e61d214218980901ad6a5a48bb4c95c Mon Sep 17 00:00:00 2001 From: weicao Date: Sun, 21 Sep 2025 15:04:40 +0800 Subject: [PATCH 1/8] fixed the date format issue and add monthly registered users metric --- .../services/starrocks_metrics_service.py | 48 ++++++++++++------- .../routes/starrocks_metrics/metrics_query.py | 4 +- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/apps/metrics/backend/services/starrocks_metrics_service.py b/apps/metrics/backend/services/starrocks_metrics_service.py index 1317336..9977570 100644 --- a/apps/metrics/backend/services/starrocks_metrics_service.py +++ b/apps/metrics/backend/services/starrocks_metrics_service.py @@ -19,15 +19,27 @@ class StarRocksMetricsService: "freeleaps": { "daily_registered_users": """ SELECT - date_id, + date, product_id, - registered_cnt, - updated_at - FROM dws_daily_registered_users - WHERE date_id >= %s - AND date_id < %s - AND product_id = %s - ORDER BY date_id ASC + value, + updated_date + FROM dws_daily_registered_users_test + WHERE date >= %s + AND date < %s + AND product_id = %s + ORDER BY date ASC + """, + "monthly_registered_users": """ + SELECT + date, + product_id, + value, + updated_date + FROM dws_monthly_registered_users_test + WHERE date >= %s + AND date < %s + AND product_id = %s + ORDER BY date ASC """, }, "magicleaps": { @@ -125,22 +137,22 @@ class StarRocksMetricsService: # Parse date strings if they are strings if isinstance(start_date, str): try: - start_dt = datetime.strptime(start_date, '%Y-%m-%d').date() + start_dt = datetime.strptime(start_date, '%Y-%m-%d %H:%M:%S') except ValueError: raise HTTPException( status_code=400, - detail="Invalid start_date format. Expected YYYY-MM-DD" + detail="Invalid start_date format. Expected YYYY-MM-DD HH:MM:SS" ) else: start_dt = start_date if isinstance(end_date, str): try: - end_dt = datetime.strptime(end_date, '%Y-%m-%d').date() + end_dt = datetime.strptime(end_date, '%Y-%m-%d %H:%M:%S') except ValueError: raise HTTPException( status_code=400, - detail="Invalid end_date format. Expected YYYY-MM-DD" + detail="Invalid start_date format. Expected YYYY-MM-DD HH:MM:SS" ) else: end_dt = end_date @@ -168,7 +180,7 @@ class StarRocksMetricsService: f"Querying metric '{metric_name}' from product '{product_id}' from {start_dt} to {end_dt}") # Execute the query - result = self.starrocks_client.execute_query( + result = await self.starrocks_client.execute_query( query=sql_query, params=(start_dt, end_dt, product_id) ) @@ -200,17 +212,21 @@ class StarRocksMetricsService: for row in starrocks_result: # Format the date - date_value = row.get("date_id") + date_value = row.get("date") if date_value: if isinstance(date_value, str): date_str = date_value else: - date_str = str(date_value) + # 如果是datetime对象，格式化为字符串 + if hasattr(date_value, 'strftime'): + date_str = date_value.strftime('%Y-%m-%d %H:%M:%S') + else: + date_str = str(date_value) else: continue # Get the value - value = row.get("registered_cnt", 0) + value = row.get("value", 0) if value is None: value = 0 diff --git a/apps/metrics/webapi/routes/starrocks_metrics/metrics_query.py b/apps/metrics/webapi/routes/starrocks_metrics/metrics_query.py index 77078f9..37ac3ea 100644 --- a/apps/metrics/webapi/routes/starrocks_metrics/metrics_query.py +++ b/apps/metrics/webapi/routes/starrocks_metrics/metrics_query.py @@ -26,8 +26,8 @@ class MetricQueryRequest(BaseModel): """Request model for metric query.""" product_id: str = Field(..., description="Product ID to identify which product's data to query") metric_name: str = Field(..., description="Name of the metric to query") - start_date: str = Field(..., description="Start date in YYYY-MM-DD format") - end_date: str = Field(..., description="End date in YYYY-MM-DD format") + start_date: str = Field(..., description="Start date in YYYY-MM-DD HH:MM:SS format") + end_date: str = Field(..., description="End date in YYYY-MM-DD HH:MM:SS format") router = APIRouter() From 7882f77befa09606e0c5e6e5e26bd82581b3e72f Mon Sep 17 00:00:00 2001 From: weicao Date: Mon, 22 Sep 2025 13:34:38 +0800 Subject: [PATCH 2/8] chinese --- apps/metrics/backend/services/starrocks_metrics_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/metrics/backend/services/starrocks_metrics_service.py b/apps/metrics/backend/services/starrocks_metrics_service.py index 9977570..30ff289 100644 --- a/apps/metrics/backend/services/starrocks_metrics_service.py +++ b/apps/metrics/backend/services/starrocks_metrics_service.py @@ -217,7 +217,7 @@ class StarRocksMetricsService: if isinstance(date_value, str): date_str = date_value else: - # 如果是datetime对象，格式化为字符串 + # If it's a datetime object, format it as a string if hasattr(date_value, 'strftime'): date_str = date_value.strftime('%Y-%m-%d %H:%M:%S') else: From a1a7624637e8a8950670703f41619ab264b07894 Mon Sep 17 00:00:00 2001 From: weicao Date: Mon, 22 Sep 2025 14:34:44 +0800 Subject: [PATCH 3/8] some change --- .../services/starrocks_metrics_service.py | 71 ++++++++++++++----- 1 file changed, 54 insertions(+), 17 deletions(-) diff --git a/apps/metrics/backend/services/starrocks_metrics_service.py b/apps/metrics/backend/services/starrocks_metrics_service.py index 30ff289..3588e07 100644 --- a/apps/metrics/backend/services/starrocks_metrics_service.py +++ b/apps/metrics/backend/services/starrocks_metrics_service.py @@ -17,25 +17,25 @@ class StarRocksMetricsService: # Global dictionary mapping metric names to their corresponding SQL queries METRIC_SQL_MAP: Dict[str, Dict[str, str]] = { "freeleaps": { - "daily_registered_users": """ + "dru": """ SELECT date, product_id, value, updated_date - FROM dws_daily_registered_users_test + FROM dws_dru WHERE date >= %s AND date < %s AND product_id = %s ORDER BY date ASC """, - "monthly_registered_users": """ + "mru": """ SELECT date, product_id, value, updated_date - FROM dws_monthly_registered_users_test + FROM dws_dru WHERE date >= %s AND date < %s AND product_id = %s @@ -98,6 +98,9 @@ class StarRocksMetricsService: """ Query metric data for a specific date range. + This method will fill missing dates in the range with 0 values to ensure + a complete time series with no gaps. + Args: product_id: Product ID to identify which product's metrics to query metric_name: Name of the metric to query @@ -105,7 +108,8 @@ class StarRocksMetricsService: end_date: End date for the query (ISO string or date) Returns: - List of dictionaries with 'date' and 'value' keys + List of dictionaries with 'date' and 'value' keys. Missing dates + in the range will be filled with 0 values. Raises: ValueError: If product_id or metric_name is not found in the SQL mapping @@ -118,7 +122,8 @@ class StarRocksMetricsService: start_date=date.today() - timedelta(days=30), end_date=date.today() ) - # Returns: [{"date": "2024-01-01", "value": 45, "labels": {...}}, ...] + # Returns: [{"date": "2024-01-01", "value": 45, "labels": {...}}, + # {"date": "2024-01-02", "value": 0, "labels": {...}}, ...] """ # Check if product_id exists in the mapping if product_id not in self.METRIC_SQL_MAP: @@ -186,7 +191,7 @@ class StarRocksMetricsService: ) # Parse the result and format it - formatted_data = self._format_query_result(result, metric_name, product_id) + formatted_data = self._format_query_result(result, metric_name, product_id, start_dt, end_dt) await self.module_logger.log_info( f"Successfully queried metric '{metric_name}' with {len(formatted_data)} data points") @@ -196,20 +201,23 @@ class StarRocksMetricsService: await self.module_logger.log_error(f"Failed to query metric '{metric_name}': {e}") raise - def _format_query_result(self, starrocks_result: List[Dict[str, Any]], metric_name: str, product_id: str) -> List[Dict[str, Any]]: + def _format_query_result(self, starrocks_result: List[Dict[str, Any]], metric_name: str, product_id: str, start_date: datetime, end_date: datetime) -> List[Dict[str, Any]]: """ - Format StarRocks query result into the required format. + Format StarRocks query result into the required format and fill missing dates with 0 values. Args: starrocks_result: Raw result from StarRocks query metric_name: Name of the metric being queried product_id: Product ID for the metric + start_date: Start date of the query range + end_date: End date of the query range Returns: - List of dictionaries with 'date' and 'value' keys + List of dictionaries with 'date' and 'value' keys, with missing dates filled with 0 """ - formatted_data = [] - + # First, process the query results and create a dictionary for quick lookup + result_dict = {} + for row in starrocks_result: # Format the date date_value = row.get("date") @@ -219,7 +227,12 @@ class StarRocksMetricsService: else: # If it's a datetime object, format it as a string if hasattr(date_value, 'strftime'): - date_str = date_value.strftime('%Y-%m-%d %H:%M:%S') + # Convert to date first, then format consistently + if hasattr(date_value, 'date'): + date_obj = date_value.date() if hasattr(date_value, 'date') else date_value + else: + date_obj = date_value + date_str = date_obj.strftime('%Y-%m-%d') + ' 00:00:00' else: date_str = str(date_value) else: @@ -236,15 +249,39 @@ class StarRocksMetricsService: "metric_type": metric_name } - formatted_data.append({ + result_dict[date_str] = { "date": date_str, "value": int(value) if value is not None else 0, "metric": metric_name, "labels": labels - }) + } - # Sort by date - formatted_data.sort(key=lambda x: x["date"]) + # Generate complete date range and fill missing dates with 0 + formatted_data = [] + current_date = start_date.date() + end_date_only = end_date.date() + + while current_date < end_date_only: + date_str = current_date.strftime('%Y-%m-%d') + ' 00:00:00' + + if date_str in result_dict: + # Use existing data + formatted_data.append(result_dict[date_str]) + else: + # Fill missing date with 0 value + labels = { + "product_id": product_id, + "metric_type": metric_name + } + + formatted_data.append({ + "date": date_str, + "value": 0, + "metric": metric_name, + "labels": labels + }) + + current_date += timedelta(days=1) return formatted_data From 6fc505ebb2811ea36ec5a585e32e49e0f5b8fdf5 Mon Sep 17 00:00:00 2001 From: weicao Date: Mon, 22 Sep 2025 14:36:56 +0800 Subject: [PATCH 4/8] dru-mru --- apps/metrics/backend/services/starrocks_metrics_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/metrics/backend/services/starrocks_metrics_service.py b/apps/metrics/backend/services/starrocks_metrics_service.py index 3588e07..7c5f45b 100644 --- a/apps/metrics/backend/services/starrocks_metrics_service.py +++ b/apps/metrics/backend/services/starrocks_metrics_service.py @@ -35,7 +35,7 @@ class StarRocksMetricsService: product_id, value, updated_date - FROM dws_dru + FROM dws_mru WHERE date >= %s AND date < %s AND product_id = %s From 459e56c7bfac627826dc5e77f042038ad60a870e Mon Sep 17 00:00:00 2001 From: icecheng Date: Mon, 22 Sep 2025 15:38:34 +0800 Subject: [PATCH 5/8] feat: enable metrics --- apps/metrics/common/config/app_settings.py | 2 +- apps/metrics/local.env | 6 ++++-- apps/metrics/webapi/providers/metrics.py | 6 ++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/metrics/common/config/app_settings.py b/apps/metrics/common/config/app_settings.py index 927ea09..a37c67d 100644 --- a/apps/metrics/common/config/app_settings.py +++ b/apps/metrics/common/config/app_settings.py @@ -24,7 +24,7 @@ class AppSettings(BaseSettings): # Prometheus settings PROMETHEUS_ENDPOINT: str = "http://localhost:9090" - METRICS_ENABLED: bool = False + METRICS_ENABLED: bool = True PROBES_ENABLED: bool = True diff --git a/apps/metrics/local.env b/apps/metrics/local.env index 4b601db..b9ad5dd 100644 --- a/apps/metrics/local.env +++ b/apps/metrics/local.env @@ -3,7 +3,7 @@ SERVER_HOST=0.0.0.0 SERVER_PORT=8009 SERVICE_API_ACCESS_PORT=8009 SERVICE_API_ACCESS_HOST=0.0.0.0 - +APP_NAME=1 # starrocks settings STARROCKS_HOST=freeleaps-starrocks-cluster-fe-service.freeleaps-data-platform.svc STARROCKS_PORT=9030 @@ -16,4 +16,6 @@ LOG_BASE_PATH=./logs BACKEND_LOG_FILE_NAME=metrics APPLICATION_ACTIVITY_LOG=metrics-activity -PROMETHEUS_ENDPOINT=http://localhost:9090 \ No newline at end of file +PROMETHEUS_ENDPOINT=http://localhost:9090 + +METRICS_ENABLED=True \ No newline at end of file diff --git a/apps/metrics/webapi/providers/metrics.py b/apps/metrics/webapi/providers/metrics.py index 08811ba..612bbef 100644 --- a/apps/metrics/webapi/providers/metrics.py +++ b/apps/metrics/webapi/providers/metrics.py @@ -1,13 +1,15 @@ import logging from prometheus_fastapi_instrumentator import Instrumentator -from common.config.app_settings import app_settings + +from common.config.site_settings import site_settings + def register(app): instrumentator = ( Instrumentator().instrument( app, metric_namespace="freeleaps-mertics", - metric_subsystem=app_settings.APP_NAME) + metric_subsystem=site_settings.NAME) ) @app.on_event("startup") From 7b7c9ca6bd9dcbaf8a22129b6b825e02455d6e25 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 22 Sep 2025 08:28:30 +0000 Subject: [PATCH 6/8] chore(release): bump version to 1.8.0 and upload released assets [ci skip] --- CHANGELOG.md | 7 +++++++ VERSION | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09cf84c..4b20c49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# [1.8.0](https://gitea.freeleaps.mathmast.com/freeleaps/freeleaps-service-hub/compare/v1.7.1...v1.8.0) (2025-09-22) + + +### Features + +* enable metrics ([459e56c](https://gitea.freeleaps.mathmast.com/freeleaps/freeleaps-service-hub/commit/459e56c7bfac627826dc5e77f042038ad60a870e)) + ## [1.7.1](https://gitea.freeleaps.mathmast.com/freeleaps/freeleaps-service-hub/compare/v1.7.0...v1.7.1) (2025-09-19) diff --git a/VERSION b/VERSION index 943f9cb..27f9cd3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.7.1 +1.8.0 From afb90e55e69ee995c7f9785505d97c94c25f5cc7 Mon Sep 17 00:00:00 2001 From: icecheng Date: Mon, 22 Sep 2025 16:48:07 +0800 Subject: [PATCH 7/8] feat: enable metrics --- apps/metrics/webapi/providers/metrics.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/apps/metrics/webapi/providers/metrics.py b/apps/metrics/webapi/providers/metrics.py index 612bbef..18d23d2 100644 --- a/apps/metrics/webapi/providers/metrics.py +++ b/apps/metrics/webapi/providers/metrics.py @@ -1,18 +1,17 @@ import logging from prometheus_fastapi_instrumentator import Instrumentator -from common.config.site_settings import site_settings - def register(app): - instrumentator = ( - Instrumentator().instrument( - app, - metric_namespace="freeleaps-mertics", - metric_subsystem=site_settings.NAME) - ) + # Prometheus metric prefix is : freeleaps_metrics + instrumentator = ( + Instrumentator().instrument( + app, + metric_namespace="freeleaps", + metric_subsystem="metrics", ) + ) - @app.on_event("startup") - async def startup(): - instrumentator.expose(app, endpoint="/api/_/metrics", should_gzip=True) - logging.info("Metrics endpoint exposed at /api/_/metrics") \ No newline at end of file + @app.on_event("startup") + async def startup(): + instrumentator.expose(app, endpoint="/api/_/metrics", should_gzip=True) + logging.info("Metrics endpoint exposed at /api/_/metrics") From ee519ca1bc69453f7e086ed0681de2eba3a222e9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 22 Sep 2025 09:06:49 +0000 Subject: [PATCH 8/8] chore(release): bump version to 1.9.0 and upload released assets [ci skip] --- CHANGELOG.md | 7 +++++++ VERSION | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b20c49..5bc2ba8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# [1.9.0](https://gitea.freeleaps.mathmast.com/freeleaps/freeleaps-service-hub/compare/v1.8.0...v1.9.0) (2025-09-22) + + +### Features + +* enable metrics ([afb90e5](https://gitea.freeleaps.mathmast.com/freeleaps/freeleaps-service-hub/commit/afb90e55e69ee995c7f9785505d97c94c25f5cc7)) + # [1.8.0](https://gitea.freeleaps.mathmast.com/freeleaps/freeleaps-service-hub/compare/v1.7.1...v1.8.0) (2025-09-22) diff --git a/VERSION b/VERSION index 27f9cd3..f8e233b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.8.0 +1.9.0