icecheng/freeleaps-service-hub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request 'feature/wc' (#62) from feature/wc into dev

Browse Source 
Reviewed-on: freeleaps/freeleaps-service-hub#62
This commit is contained in:
icecheng 2025-09-22 07:12:28 +00:00
commit 48272cfd28
2 changed files with 83 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
apps/metrics/backend/services/starrocks_metrics_service.py
View File

@ -17,17 +17,29 @@ class StarRocksMetricsService:
# Global dictionary mapping metric names to their corresponding SQL queries # Global dictionary mapping metric names to their corresponding SQL queries
METRIC_SQL_MAP: Dict[str, Dict[str, str]] = { METRIC_SQL_MAP: Dict[str, Dict[str, str]] = {
"freeleaps": { "freeleaps": {
"daily_registered_users": """ "dru": """
SELECT SELECT
date_id, date,
product_id, product_id,
registered_cnt, value,
updated_at updated_date
FROM dws_daily_registered_users FROM dws_dru
WHERE date_id >= %s WHERE date >= %s
AND date_id < %s AND date < %s
AND product_id = %s AND product_id = %s
ORDER BY date_id ASC ORDER BY date ASC
""",
"mru": """
SELECT
date,
product_id,
value,
updated_date
FROM dws_mru
WHERE date >= %s
AND date < %s
AND product_id = %s
ORDER BY date ASC
""", """,
}, },
"magicleaps": { "magicleaps": {
@ -86,6 +98,9 @@ class StarRocksMetricsService:
""" """
Query metric data for a specific date range. Query metric data for a specific date range.
This method will fill missing dates in the range with 0 values to ensure
a complete time series with no gaps.
Args: Args:
product_id: Product ID to identify which product's metrics to query product_id: Product ID to identify which product's metrics to query
metric_name: Name of the metric to query metric_name: Name of the metric to query
@ -93,7 +108,8 @@ class StarRocksMetricsService:
end_date: End date for the query (ISO string or date) end_date: End date for the query (ISO string or date)
Returns: Returns:
List of dictionaries with 'date' and 'value' keys List of dictionaries with 'date' and 'value' keys. Missing dates
in the range will be filled with 0 values.
Raises: Raises:
ValueError: If product_id or metric_name is not found in the SQL mapping ValueError: If product_id or metric_name is not found in the SQL mapping
@ -106,7 +122,8 @@ class StarRocksMetricsService:
start_date=date.today() - timedelta(days=30), start_date=date.today() - timedelta(days=30),
end_date=date.today() end_date=date.today()
) )
# Returns: [{"date": "2024-01-01", "value": 45, "labels": {...}}, ...] # Returns: [{"date": "2024-01-01", "value": 45, "labels": {...}},
# {"date": "2024-01-02", "value": 0, "labels": {...}}, ...]
""" """
# Check if product_id exists in the mapping # Check if product_id exists in the mapping
if product_id not in self.METRIC_SQL_MAP: if product_id not in self.METRIC_SQL_MAP:
@ -125,22 +142,22 @@ class StarRocksMetricsService:
# Parse date strings if they are strings # Parse date strings if they are strings
if isinstance(start_date, str): if isinstance(start_date, str):
try: try:
start_dt = datetime.strptime(start_date, '%Y-%m-%d').date() start_dt = datetime.strptime(start_date, '%Y-%m-%d %H:%M:%S')
except ValueError: except ValueError:
raise HTTPException( raise HTTPException(
status_code=400, status_code=400,
detail="Invalid start_date format. Expected YYYY-MM-DD" detail="Invalid start_date format. Expected YYYY-MM-DD HH:MM:SS"
) )
else: else:
start_dt = start_date start_dt = start_date
if isinstance(end_date, str): if isinstance(end_date, str):
try: try:
end_dt = datetime.strptime(end_date, '%Y-%m-%d').date() end_dt = datetime.strptime(end_date, '%Y-%m-%d %H:%M:%S')
except ValueError: except ValueError:
raise HTTPException( raise HTTPException(
status_code=400, status_code=400,
detail="Invalid end_date format. Expected YYYY-MM-DD" detail="Invalid start_date format. Expected YYYY-MM-DD HH:MM:SS"
) )
else: else:
end_dt = end_date end_dt = end_date
@ -174,7 +191,7 @@ class StarRocksMetricsService:
) )
# Parse the result and format it # Parse the result and format it
formatted_data = self._format_query_result(result, metric_name, product_id) formatted_data = self._format_query_result(result, metric_name, product_id, start_dt, end_dt)
await self.module_logger.log_info( await self.module_logger.log_info(
f"Successfully queried metric '{metric_name}' with {len(formatted_data)} data points") f"Successfully queried metric '{metric_name}' with {len(formatted_data)} data points")
@ -184,33 +201,45 @@ class StarRocksMetricsService:
await self.module_logger.log_error(f"Failed to query metric '{metric_name}': {e}") await self.module_logger.log_error(f"Failed to query metric '{metric_name}': {e}")
raise raise
def _format_query_result(self, starrocks_result: List[Dict[str, Any]], metric_name: str, product_id: str) -> List[Dict[str, Any]]: def _format_query_result(self, starrocks_result: List[Dict[str, Any]], metric_name: str, product_id: str, start_date: datetime, end_date: datetime) -> List[Dict[str, Any]]:
""" """
Format StarRocks query result into the required format. Format StarRocks query result into the required format and fill missing dates with 0 values.
Args: Args:
starrocks_result: Raw result from StarRocks query starrocks_result: Raw result from StarRocks query
metric_name: Name of the metric being queried metric_name: Name of the metric being queried
product_id: Product ID for the metric product_id: Product ID for the metric
start_date: Start date of the query range
end_date: End date of the query range
Returns: Returns:
List of dictionaries with 'date' and 'value' keys List of dictionaries with 'date' and 'value' keys, with missing dates filled with 0
""" """
formatted_data = [] # First, process the query results and create a dictionary for quick lookup
result_dict = {}
for row in starrocks_result: for row in starrocks_result:
# Format the date # Format the date
date_value = row.get("date_id") date_value = row.get("date")
if date_value: if date_value:
if isinstance(date_value, str): if isinstance(date_value, str):
date_str = date_value date_str = date_value
else: else:
date_str = str(date_value) # If it's a datetime object, format it as a string
if hasattr(date_value, 'strftime'):
# Convert to date first, then format consistently
if hasattr(date_value, 'date'):
date_obj = date_value.date() if hasattr(date_value, 'date') else date_value
else:
date_obj = date_value
date_str = date_obj.strftime('%Y-%m-%d') + ' 00:00:00'
else:
date_str = str(date_value)
else: else:
continue continue
# Get the value # Get the value
value = row.get("registered_cnt", 0) value = row.get("value", 0)
if value is None: if value is None:
value = 0 value = 0
@ -220,15 +249,39 @@ class StarRocksMetricsService:
"metric_type": metric_name "metric_type": metric_name
} }
formatted_data.append({ result_dict[date_str] = {
"date": date_str, "date": date_str,
"value": int(value) if value is not None else 0, "value": int(value) if value is not None else 0,
"metric": metric_name, "metric": metric_name,
"labels": labels "labels": labels
}) }
# Sort by date # Generate complete date range and fill missing dates with 0
formatted_data.sort(key=lambda x: x["date"]) formatted_data = []
current_date = start_date.date()
end_date_only = end_date.date()
while current_date < end_date_only:
date_str = current_date.strftime('%Y-%m-%d') + ' 00:00:00'
if date_str in result_dict:
# Use existing data
formatted_data.append(result_dict[date_str])
else:
# Fill missing date with 0 value
labels = {
"product_id": product_id,
"metric_type": metric_name
}
formatted_data.append({
"date": date_str,
"value": 0,
"metric": metric_name,
"labels": labels
})
current_date += timedelta(days=1)
return formatted_data return formatted_data

4
apps/metrics/webapi/routes/starrocks_metrics/metrics_query.py
View File

@ -26,8 +26,8 @@ class MetricQueryRequest(BaseModel):
"""Request model for metric query.""" """Request model for metric query."""
product_id: str = Field(..., description="Product ID to identify which product's data to query") product_id: str = Field(..., description="Product ID to identify which product's data to query")
metric_name: str = Field(..., description="Name of the metric to query") metric_name: str = Field(..., description="Name of the metric to query")
start_date: str = Field(..., description="Start date in YYYY-MM-DD format") start_date: str = Field(..., description="Start date in YYYY-MM-DD HH:MM:SS format")
end_date: str = Field(..., description="End date in YYYY-MM-DD format") end_date: str = Field(..., description="End date in YYYY-MM-DD HH:MM:SS format")
router = APIRouter() router = APIRouter()