diff --git a/apps/metrics/backend/services/starrocks_metrics_service.py b/apps/metrics/backend/services/starrocks_metrics_service.py index 30ff289..3588e07 100644 --- a/apps/metrics/backend/services/starrocks_metrics_service.py +++ b/apps/metrics/backend/services/starrocks_metrics_service.py @@ -17,25 +17,25 @@ class StarRocksMetricsService: # Global dictionary mapping metric names to their corresponding SQL queries METRIC_SQL_MAP: Dict[str, Dict[str, str]] = { "freeleaps": { - "daily_registered_users": """ + "dru": """ SELECT date, product_id, value, updated_date - FROM dws_daily_registered_users_test + FROM dws_dru WHERE date >= %s AND date < %s AND product_id = %s ORDER BY date ASC """, - "monthly_registered_users": """ + "mru": """ SELECT date, product_id, value, updated_date - FROM dws_monthly_registered_users_test + FROM dws_dru WHERE date >= %s AND date < %s AND product_id = %s @@ -98,6 +98,9 @@ class StarRocksMetricsService: """ Query metric data for a specific date range. + This method will fill missing dates in the range with 0 values to ensure + a complete time series with no gaps. + Args: product_id: Product ID to identify which product's metrics to query metric_name: Name of the metric to query @@ -105,7 +108,8 @@ class StarRocksMetricsService: end_date: End date for the query (ISO string or date) Returns: - List of dictionaries with 'date' and 'value' keys + List of dictionaries with 'date' and 'value' keys. Missing dates + in the range will be filled with 0 values. Raises: ValueError: If product_id or metric_name is not found in the SQL mapping @@ -118,7 +122,8 @@ class StarRocksMetricsService: start_date=date.today() - timedelta(days=30), end_date=date.today() ) - # Returns: [{"date": "2024-01-01", "value": 45, "labels": {...}}, ...] + # Returns: [{"date": "2024-01-01", "value": 45, "labels": {...}}, + # {"date": "2024-01-02", "value": 0, "labels": {...}}, ...] """ # Check if product_id exists in the mapping if product_id not in self.METRIC_SQL_MAP: @@ -186,7 +191,7 @@ class StarRocksMetricsService: ) # Parse the result and format it - formatted_data = self._format_query_result(result, metric_name, product_id) + formatted_data = self._format_query_result(result, metric_name, product_id, start_dt, end_dt) await self.module_logger.log_info( f"Successfully queried metric '{metric_name}' with {len(formatted_data)} data points") @@ -196,20 +201,23 @@ class StarRocksMetricsService: await self.module_logger.log_error(f"Failed to query metric '{metric_name}': {e}") raise - def _format_query_result(self, starrocks_result: List[Dict[str, Any]], metric_name: str, product_id: str) -> List[Dict[str, Any]]: + def _format_query_result(self, starrocks_result: List[Dict[str, Any]], metric_name: str, product_id: str, start_date: datetime, end_date: datetime) -> List[Dict[str, Any]]: """ - Format StarRocks query result into the required format. + Format StarRocks query result into the required format and fill missing dates with 0 values. Args: starrocks_result: Raw result from StarRocks query metric_name: Name of the metric being queried product_id: Product ID for the metric + start_date: Start date of the query range + end_date: End date of the query range Returns: - List of dictionaries with 'date' and 'value' keys + List of dictionaries with 'date' and 'value' keys, with missing dates filled with 0 """ - formatted_data = [] - + # First, process the query results and create a dictionary for quick lookup + result_dict = {} + for row in starrocks_result: # Format the date date_value = row.get("date") @@ -219,7 +227,12 @@ class StarRocksMetricsService: else: # If it's a datetime object, format it as a string if hasattr(date_value, 'strftime'): - date_str = date_value.strftime('%Y-%m-%d %H:%M:%S') + # Convert to date first, then format consistently + if hasattr(date_value, 'date'): + date_obj = date_value.date() if hasattr(date_value, 'date') else date_value + else: + date_obj = date_value + date_str = date_obj.strftime('%Y-%m-%d') + ' 00:00:00' else: date_str = str(date_value) else: @@ -236,15 +249,39 @@ class StarRocksMetricsService: "metric_type": metric_name } - formatted_data.append({ + result_dict[date_str] = { "date": date_str, "value": int(value) if value is not None else 0, "metric": metric_name, "labels": labels - }) + } - # Sort by date - formatted_data.sort(key=lambda x: x["date"]) + # Generate complete date range and fill missing dates with 0 + formatted_data = [] + current_date = start_date.date() + end_date_only = end_date.date() + + while current_date < end_date_only: + date_str = current_date.strftime('%Y-%m-%d') + ' 00:00:00' + + if date_str in result_dict: + # Use existing data + formatted_data.append(result_dict[date_str]) + else: + # Fill missing date with 0 value + labels = { + "product_id": product_id, + "metric_type": metric_name + } + + formatted_data.append({ + "date": date_str, + "value": 0, + "metric": metric_name, + "labels": labels + }) + + current_date += timedelta(days=1) return formatted_data