icecheng/freeleaps-service-hub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

some change

Browse Source
This commit is contained in:
weicao 2025-09-22 14:34:44 +08:00
parent 7882f77bef
commit a1a7624637
1 changed files with 54 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
apps/metrics/backend/services/starrocks_metrics_service.py
View File

@ -17,25 +17,25 @@ class StarRocksMetricsService:
# Global dictionary mapping metric names to their corresponding SQL queries # Global dictionary mapping metric names to their corresponding SQL queries
METRIC_SQL_MAP: Dict[str, Dict[str, str]] = { METRIC_SQL_MAP: Dict[str, Dict[str, str]] = {
"freeleaps": { "freeleaps": {
"daily_registered_users": """ "dru": """
SELECT SELECT
date, date,
product_id, product_id,
value, value,
updated_date updated_date
FROM dws_daily_registered_users_test FROM dws_dru
WHERE date >= %s WHERE date >= %s
AND date < %s AND date < %s
AND product_id = %s AND product_id = %s
ORDER BY date ASC ORDER BY date ASC
""", """,
"monthly_registered_users": """ "mru": """
SELECT SELECT
date, date,
product_id, product_id,
value, value,
updated_date updated_date
FROM dws_monthly_registered_users_test FROM dws_dru
WHERE date >= %s WHERE date >= %s
AND date < %s AND date < %s
AND product_id = %s AND product_id = %s
@ -98,6 +98,9 @@ class StarRocksMetricsService:
""" """
Query metric data for a specific date range. Query metric data for a specific date range.
This method will fill missing dates in the range with 0 values to ensure
a complete time series with no gaps.
Args: Args:
product_id: Product ID to identify which product's metrics to query product_id: Product ID to identify which product's metrics to query
metric_name: Name of the metric to query metric_name: Name of the metric to query
@ -105,7 +108,8 @@ class StarRocksMetricsService:
end_date: End date for the query (ISO string or date) end_date: End date for the query (ISO string or date)
Returns: Returns:
List of dictionaries with 'date' and 'value' keys List of dictionaries with 'date' and 'value' keys. Missing dates
in the range will be filled with 0 values.
Raises: Raises:
ValueError: If product_id or metric_name is not found in the SQL mapping ValueError: If product_id or metric_name is not found in the SQL mapping
@ -118,7 +122,8 @@ class StarRocksMetricsService:
start_date=date.today() - timedelta(days=30), start_date=date.today() - timedelta(days=30),
end_date=date.today() end_date=date.today()
) )
# Returns: [{"date": "2024-01-01", "value": 45, "labels": {...}}, ...] # Returns: [{"date": "2024-01-01", "value": 45, "labels": {...}},
# {"date": "2024-01-02", "value": 0, "labels": {...}}, ...]
""" """
# Check if product_id exists in the mapping # Check if product_id exists in the mapping
if product_id not in self.METRIC_SQL_MAP: if product_id not in self.METRIC_SQL_MAP:
@ -186,7 +191,7 @@ class StarRocksMetricsService:
) )
# Parse the result and format it # Parse the result and format it
formatted_data = self._format_query_result(result, metric_name, product_id) formatted_data = self._format_query_result(result, metric_name, product_id, start_dt, end_dt)
await self.module_logger.log_info( await self.module_logger.log_info(
f"Successfully queried metric '{metric_name}' with {len(formatted_data)} data points") f"Successfully queried metric '{metric_name}' with {len(formatted_data)} data points")
@ -196,20 +201,23 @@ class StarRocksMetricsService:
await self.module_logger.log_error(f"Failed to query metric '{metric_name}': {e}") await self.module_logger.log_error(f"Failed to query metric '{metric_name}': {e}")
raise raise
def _format_query_result(self, starrocks_result: List[Dict[str, Any]], metric_name: str, product_id: str) -> List[Dict[str, Any]]: def _format_query_result(self, starrocks_result: List[Dict[str, Any]], metric_name: str, product_id: str, start_date: datetime, end_date: datetime) -> List[Dict[str, Any]]:
""" """
Format StarRocks query result into the required format. Format StarRocks query result into the required format and fill missing dates with 0 values.
Args: Args:
starrocks_result: Raw result from StarRocks query starrocks_result: Raw result from StarRocks query
metric_name: Name of the metric being queried metric_name: Name of the metric being queried
product_id: Product ID for the metric product_id: Product ID for the metric
start_date: Start date of the query range
end_date: End date of the query range
Returns: Returns:
List of dictionaries with 'date' and 'value' keys List of dictionaries with 'date' and 'value' keys, with missing dates filled with 0
""" """
formatted_data = [] # First, process the query results and create a dictionary for quick lookup
result_dict = {}
for row in starrocks_result: for row in starrocks_result:
# Format the date # Format the date
date_value = row.get("date") date_value = row.get("date")
@ -219,7 +227,12 @@ class StarRocksMetricsService:
else: else:
# If it's a datetime object, format it as a string # If it's a datetime object, format it as a string
if hasattr(date_value, 'strftime'): if hasattr(date_value, 'strftime'):
date_str = date_value.strftime('%Y-%m-%d %H:%M:%S') # Convert to date first, then format consistently
if hasattr(date_value, 'date'):
date_obj = date_value.date() if hasattr(date_value, 'date') else date_value
else:
date_obj = date_value
date_str = date_obj.strftime('%Y-%m-%d') + ' 00:00:00'
else: else:
date_str = str(date_value) date_str = str(date_value)
else: else:
@ -236,15 +249,39 @@ class StarRocksMetricsService:
"metric_type": metric_name "metric_type": metric_name
} }
formatted_data.append({ result_dict[date_str] = {
"date": date_str, "date": date_str,
"value": int(value) if value is not None else 0, "value": int(value) if value is not None else 0,
"metric": metric_name, "metric": metric_name,
"labels": labels "labels": labels
}) }
# Sort by date # Generate complete date range and fill missing dates with 0
formatted_data.sort(key=lambda x: x["date"]) formatted_data = []
current_date = start_date.date()
end_date_only = end_date.date()
while current_date < end_date_only:
date_str = current_date.strftime('%Y-%m-%d') + ' 00:00:00'
if date_str in result_dict:
# Use existing data
formatted_data.append(result_dict[date_str])
else:
# Fill missing date with 0 value
labels = {
"product_id": product_id,
"metric_type": metric_name
}
formatted_data.append({
"date": date_str,
"value": 0,
"metric": metric_name,
"labels": labels
})
current_date += timedelta(days=1)
return formatted_data return formatted_data