icecheng/freeleaps-service-hub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request 'dev' (#64) from dev into master

Browse Source 
Reviewed-on: freeleaps/freeleaps-service-hub#64
This commit is contained in:
icecheng 2025-09-22 08:26:42 +00:00
commit d9c08d9c79
5 changed files with 92 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
apps/metrics/backend/services/starrocks_metrics_service.py
View File

@ -17,17 +17,29 @@ class StarRocksMetricsService:
# Global dictionary mapping metric names to their corresponding SQL queries
METRIC_SQL_MAP: Dict[str, Dict[str, str]] = {
"freeleaps": {
"daily_registered_users": """
"dru": """
SELECT
date_id,
date,
product_id,
registered_cnt,
updated_at
FROM dws_daily_registered_users
WHERE date_id >= %s
AND date_id < %s
AND product_id = %s
ORDER BY date_id ASC
value,
updated_date
FROM dws_dru
WHERE date >= %s
AND date < %s
AND product_id = %s
ORDER BY date ASC
""",
"mru": """
SELECT
date,
product_id,
value,
updated_date
FROM dws_mru
WHERE date >= %s
AND date < %s
AND product_id = %s
ORDER BY date ASC
""",
},
"magicleaps": {
@ -86,6 +98,9 @@ class StarRocksMetricsService:
"""
Query metric data for a specific date range.
This method will fill missing dates in the range with 0 values to ensure
a complete time series with no gaps.
Args:
product_id: Product ID to identify which product's metrics to query
metric_name: Name of the metric to query
@ -93,7 +108,8 @@ class StarRocksMetricsService:
end_date: End date for the query (ISO string or date)
Returns:
List of dictionaries with 'date' and 'value' keys
List of dictionaries with 'date' and 'value' keys. Missing dates
in the range will be filled with 0 values.
Raises:
ValueError: If product_id or metric_name is not found in the SQL mapping
@ -106,7 +122,8 @@ class StarRocksMetricsService:
start_date=date.today() - timedelta(days=30),
end_date=date.today()
)
# Returns: [{"date": "2024-01-01", "value": 45, "labels": {...}}, ...]
# Returns: [{"date": "2024-01-01", "value": 45, "labels": {...}},
# {"date": "2024-01-02", "value": 0, "labels": {...}}, ...]
"""
# Check if product_id exists in the mapping
if product_id not in self.METRIC_SQL_MAP:
@ -125,22 +142,22 @@ class StarRocksMetricsService:
# Parse date strings if they are strings
if isinstance(start_date, str):
try:
start_dt = datetime.strptime(start_date, '%Y-%m-%d').date()
start_dt = datetime.strptime(start_date, '%Y-%m-%d %H:%M:%S')
except ValueError:
raise HTTPException(
status_code=400,
detail="Invalid start_date format. Expected YYYY-MM-DD"
detail="Invalid start_date format. Expected YYYY-MM-DD HH:MM:SS"
)
else:
start_dt = start_date
if isinstance(end_date, str):
try:
end_dt = datetime.strptime(end_date, '%Y-%m-%d').date()
end_dt = datetime.strptime(end_date, '%Y-%m-%d %H:%M:%S')
except ValueError:
raise HTTPException(
status_code=400,
detail="Invalid end_date format. Expected YYYY-MM-DD"
detail="Invalid start_date format. Expected YYYY-MM-DD HH:MM:SS"
)
else:
end_dt = end_date
@ -174,7 +191,7 @@ class StarRocksMetricsService:
)
# Parse the result and format it
formatted_data = self._format_query_result(result, metric_name, product_id)
formatted_data = self._format_query_result(result, metric_name, product_id, start_dt, end_dt)
await self.module_logger.log_info(
f"Successfully queried metric '{metric_name}' with {len(formatted_data)} data points")
@ -184,33 +201,45 @@ class StarRocksMetricsService:
await self.module_logger.log_error(f"Failed to query metric '{metric_name}': {e}")
raise
def _format_query_result(self, starrocks_result: List[Dict[str, Any]], metric_name: str, product_id: str) -> List[Dict[str, Any]]:
def _format_query_result(self, starrocks_result: List[Dict[str, Any]], metric_name: str, product_id: str, start_date: datetime, end_date: datetime) -> List[Dict[str, Any]]:
"""
Format StarRocks query result into the required format.
Format StarRocks query result into the required format and fill missing dates with 0 values.
Args:
starrocks_result: Raw result from StarRocks query
metric_name: Name of the metric being queried
product_id: Product ID for the metric
start_date: Start date of the query range
end_date: End date of the query range
Returns:
List of dictionaries with 'date' and 'value' keys
List of dictionaries with 'date' and 'value' keys, with missing dates filled with 0
"""
formatted_data = []
# First, process the query results and create a dictionary for quick lookup
result_dict = {}
for row in starrocks_result:
# Format the date
date_value = row.get("date_id")
date_value = row.get("date")
if date_value:
if isinstance(date_value, str):
date_str = date_value
else:
date_str = str(date_value)
# If it's a datetime object, format it as a string
if hasattr(date_value, 'strftime'):
# Convert to date first, then format consistently
if hasattr(date_value, 'date'):
date_obj = date_value.date() if hasattr(date_value, 'date') else date_value
else:
date_obj = date_value
date_str = date_obj.strftime('%Y-%m-%d') + ' 00:00:00'
else:
date_str = str(date_value)
else:
continue
# Get the value
value = row.get("registered_cnt", 0)
value = row.get("value", 0)
if value is None:
value = 0
@ -220,15 +249,39 @@ class StarRocksMetricsService:
"metric_type": metric_name
}
formatted_data.append({
result_dict[date_str] = {
"date": date_str,
"value": int(value) if value is not None else 0,
"metric": metric_name,
"labels": labels
})
}
# Sort by date
formatted_data.sort(key=lambda x: x["date"])
# Generate complete date range and fill missing dates with 0
formatted_data = []
current_date = start_date.date()
end_date_only = end_date.date()
while current_date < end_date_only:
date_str = current_date.strftime('%Y-%m-%d') + ' 00:00:00'
if date_str in result_dict:
# Use existing data
formatted_data.append(result_dict[date_str])
else:
# Fill missing date with 0 value
labels = {
"product_id": product_id,
"metric_type": metric_name
}
formatted_data.append({
"date": date_str,
"value": 0,
"metric": metric_name,
"labels": labels
})
current_date += timedelta(days=1)
return formatted_data

2
apps/metrics/common/config/app_settings.py
View File

@ -24,7 +24,7 @@ class AppSettings(BaseSettings):
# Prometheus settings
PROMETHEUS_ENDPOINT: str = "http://localhost:9090"
METRICS_ENABLED: bool = False
METRICS_ENABLED: bool = True
PROBES_ENABLED: bool = True

6
apps/metrics/local.env
View File

@ -3,7 +3,7 @@ SERVER_HOST=0.0.0.0
SERVER_PORT=8009
SERVICE_API_ACCESS_PORT=8009
SERVICE_API_ACCESS_HOST=0.0.0.0
APP_NAME=1
# starrocks settings
STARROCKS_HOST=freeleaps-starrocks-cluster-fe-service.freeleaps-data-platform.svc
STARROCKS_PORT=9030
@ -16,4 +16,6 @@ LOG_BASE_PATH=./logs
BACKEND_LOG_FILE_NAME=metrics
APPLICATION_ACTIVITY_LOG=metrics-activity
PROMETHEUS_ENDPOINT=http://localhost:9090
PROMETHEUS_ENDPOINT=http://localhost:9090
METRICS_ENABLED=True

6
apps/metrics/webapi/providers/metrics.py
View File

@ -1,13 +1,15 @@
import logging
from prometheus_fastapi_instrumentator import Instrumentator
from common.config.app_settings import app_settings
from common.config.site_settings import site_settings
def register(app):
instrumentator = (
Instrumentator().instrument(
app,
metric_namespace="freeleaps-mertics",
metric_subsystem=app_settings.APP_NAME)
metric_subsystem=site_settings.NAME)
)
@app.on_event("startup")

4
apps/metrics/webapi/routes/starrocks_metrics/metrics_query.py
View File

@ -26,8 +26,8 @@ class MetricQueryRequest(BaseModel):
"""Request model for metric query."""
product_id: str = Field(..., description="Product ID to identify which product's data to query")
metric_name: str = Field(..., description="Name of the metric to query")
start_date: str = Field(..., description="Start date in YYYY-MM-DD format")
end_date: str = Field(..., description="End date in YYYY-MM-DD format")
start_date: str = Field(..., description="Start date in YYYY-MM-DD HH:MM:SS format")
end_date: str = Field(..., description="End date in YYYY-MM-DD HH:MM:SS format")
router = APIRouter()