some change
This commit is contained in:
weicao
parent
7882f77bef
commit
a1a7624637
@ -17,25 +17,25 @@ class StarRocksMetricsService:
|
||||
# Global dictionary mapping metric names to their corresponding SQL queries
|
||||
METRIC_SQL_MAP: Dict[str, Dict[str, str]] = {
|
||||
"freeleaps": {
|
||||
"daily_registered_users": """
|
||||
"dru": """
|
||||
SELECT
|
||||
date,
|
||||
product_id,
|
||||
value,
|
||||
updated_date
|
||||
FROM dws_daily_registered_users_test
|
||||
FROM dws_dru
|
||||
WHERE date >= %s
|
||||
AND date < %s
|
||||
AND product_id = %s
|
||||
ORDER BY date ASC
|
||||
""",
|
||||
"monthly_registered_users": """
|
||||
"mru": """
|
||||
SELECT
|
||||
date,
|
||||
product_id,
|
||||
value,
|
||||
updated_date
|
||||
FROM dws_monthly_registered_users_test
|
||||
FROM dws_dru
|
||||
WHERE date >= %s
|
||||
AND date < %s
|
||||
AND product_id = %s
|
||||
@ -98,6 +98,9 @@ class StarRocksMetricsService:
|
||||
"""
|
||||
Query metric data for a specific date range.
|
||||
|
||||
This method will fill missing dates in the range with 0 values to ensure
|
||||
a complete time series with no gaps.
|
||||
|
||||
Args:
|
||||
product_id: Product ID to identify which product's metrics to query
|
||||
metric_name: Name of the metric to query
|
||||
@ -105,7 +108,8 @@ class StarRocksMetricsService:
|
||||
end_date: End date for the query (ISO string or date)
|
||||
|
||||
Returns:
|
||||
List of dictionaries with 'date' and 'value' keys
|
||||
List of dictionaries with 'date' and 'value' keys. Missing dates
|
||||
in the range will be filled with 0 values.
|
||||
|
||||
Raises:
|
||||
ValueError: If product_id or metric_name is not found in the SQL mapping
|
||||
@ -118,7 +122,8 @@ class StarRocksMetricsService:
|
||||
start_date=date.today() - timedelta(days=30),
|
||||
end_date=date.today()
|
||||
)
|
||||
# Returns: [{"date": "2024-01-01", "value": 45, "labels": {...}}, ...]
|
||||
# Returns: [{"date": "2024-01-01", "value": 45, "labels": {...}},
|
||||
# {"date": "2024-01-02", "value": 0, "labels": {...}}, ...]
|
||||
"""
|
||||
# Check if product_id exists in the mapping
|
||||
if product_id not in self.METRIC_SQL_MAP:
|
||||
@ -186,7 +191,7 @@ class StarRocksMetricsService:
|
||||
)
|
||||
|
||||
# Parse the result and format it
|
||||
formatted_data = self._format_query_result(result, metric_name, product_id)
|
||||
formatted_data = self._format_query_result(result, metric_name, product_id, start_dt, end_dt)
|
||||
|
||||
await self.module_logger.log_info(
|
||||
f"Successfully queried metric '{metric_name}' with {len(formatted_data)} data points")
|
||||
@ -196,19 +201,22 @@ class StarRocksMetricsService:
|
||||
await self.module_logger.log_error(f"Failed to query metric '{metric_name}': {e}")
|
||||
raise
|
||||
|
||||
def _format_query_result(self, starrocks_result: List[Dict[str, Any]], metric_name: str, product_id: str) -> List[Dict[str, Any]]:
|
||||
def _format_query_result(self, starrocks_result: List[Dict[str, Any]], metric_name: str, product_id: str, start_date: datetime, end_date: datetime) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Format StarRocks query result into the required format.
|
||||
Format StarRocks query result into the required format and fill missing dates with 0 values.
|
||||
|
||||
Args:
|
||||
starrocks_result: Raw result from StarRocks query
|
||||
metric_name: Name of the metric being queried
|
||||
product_id: Product ID for the metric
|
||||
start_date: Start date of the query range
|
||||
end_date: End date of the query range
|
||||
|
||||
Returns:
|
||||
List of dictionaries with 'date' and 'value' keys
|
||||
List of dictionaries with 'date' and 'value' keys, with missing dates filled with 0
|
||||
"""
|
||||
formatted_data = []
|
||||
# First, process the query results and create a dictionary for quick lookup
|
||||
result_dict = {}
|
||||
|
||||
for row in starrocks_result:
|
||||
# Format the date
|
||||
@ -219,7 +227,12 @@ class StarRocksMetricsService:
|
||||
else:
|
||||
# If it's a datetime object, format it as a string
|
||||
if hasattr(date_value, 'strftime'):
|
||||
date_str = date_value.strftime('%Y-%m-%d %H:%M:%S')
|
||||
# Convert to date first, then format consistently
|
||||
if hasattr(date_value, 'date'):
|
||||
date_obj = date_value.date() if hasattr(date_value, 'date') else date_value
|
||||
else:
|
||||
date_obj = date_value
|
||||
date_str = date_obj.strftime('%Y-%m-%d') + ' 00:00:00'
|
||||
else:
|
||||
date_str = str(date_value)
|
||||
else:
|
||||
@ -236,15 +249,39 @@ class StarRocksMetricsService:
|
||||
"metric_type": metric_name
|
||||
}
|
||||
|
||||
formatted_data.append({
|
||||
result_dict[date_str] = {
|
||||
"date": date_str,
|
||||
"value": int(value) if value is not None else 0,
|
||||
"metric": metric_name,
|
||||
"labels": labels
|
||||
})
|
||||
}
|
||||
|
||||
# Sort by date
|
||||
formatted_data.sort(key=lambda x: x["date"])
|
||||
# Generate complete date range and fill missing dates with 0
|
||||
formatted_data = []
|
||||
current_date = start_date.date()
|
||||
end_date_only = end_date.date()
|
||||
|
||||
while current_date < end_date_only:
|
||||
date_str = current_date.strftime('%Y-%m-%d') + ' 00:00:00'
|
||||
|
||||
if date_str in result_dict:
|
||||
# Use existing data
|
||||
formatted_data.append(result_dict[date_str])
|
||||
else:
|
||||
# Fill missing date with 0 value
|
||||
labels = {
|
||||
"product_id": product_id,
|
||||
"metric_type": metric_name
|
||||
}
|
||||
|
||||
formatted_data.append({
|
||||
"date": date_str,
|
||||
"value": 0,
|
||||
"metric": metric_name,
|
||||
"labels": labels
|
||||
})
|
||||
|
||||
current_date += timedelta(days=1)
|
||||
|
||||
return formatted_data
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user