diff --git a/apps/metrics/.env b/apps/metrics/.env deleted file mode 100644 index f88d5c1..0000000 --- a/apps/metrics/.env +++ /dev/null @@ -1,16 +0,0 @@ -# Local environment configuration for Metrics service -SERVICE_API_ACCESS_PORT=8009 -SERVICE_API_ACCESS_HOST=0.0.0.0 - -# Local database settings -MONGODB_URI=mongodb://localhost:27017/ -MONGODB_NAME=freeleaps2 - -# Local service URLs -DEVSVC_WEBAPI_URL_BASE=http://localhost:8007/api/devsvc -NOTIFICATION_WEBAPI_URL_BASE=http://localhost:8003/api/notification/ - -# Log settings -LOG_BASE_PATH=./logs -BACKEND_LOG_FILE_NAME=metrics -APPLICATION_ACTIVITY_LOG=metrics-activity \ No newline at end of file diff --git a/apps/metrics/.gitignore b/apps/metrics/.gitignore index 365d8bf..0cf956f 100644 --- a/apps/metrics/.gitignore +++ b/apps/metrics/.gitignore @@ -73,3 +73,12 @@ coverage.xml .env.development.local .env.test.local .env.production.local + + + + + + + + + diff --git a/apps/metrics/Dockerfile b/apps/metrics/Dockerfile index 4c40152..b090349 100644 --- a/apps/metrics/Dockerfile +++ b/apps/metrics/Dockerfile @@ -1,37 +1,20 @@ -FROM python:3.10-slim-bullseye +# download image here: https://docker.aityp.com/image/docker.io/python:3.12-slim +FROM swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/python:3.12-slim -# docker settings -ARG CONTAINER_APP_ROOT="/app" -ENV APP_NAME="metrics" +# Set working directory +WORKDIR /app -# Service dependencies -ENV DEVSVC_WEBAPI_URL_BASE="http://devsvc:8007/api/devsvc" -ENV NOTIFICATION_WEBAPI_URL_BASE="http://notification:8003/api/notification/" - -# JWT settings -ENV JWT_SECRET_KEY="8f87ca8c3c9c3df09a9c78e0adb0927855568f6072d9efc892534aee35f5867b" -ENV JWT_ALGORITHM="HS256" - -# Site settings -ENV SERVICE_API_ACCESS_HOST=0.0.0.0 -ENV SERVICE_API_ACCESS_PORT=8009 -ENV MONGODB_NAME=freeleaps2 -ENV MONGODB_PORT=27017 -ENV MONGODB_URI="mongodb://localhost:27017/" - -# Log settings -ENV LOG_BASE_PATH=$CONTAINER_APP_ROOT/log/$APP_NAME -ENV BACKEND_LOG_FILE_NAME=$APP_NAME -ENV APPLICATION_ACTIVITY_LOG=$APP_NAME-activity - -WORKDIR ${CONTAINER_APP_ROOT} +# Copy requirements file COPY requirements.txt . -RUN pip install --upgrade pip +# Install dependencies RUN pip install --no-cache-dir -r requirements.txt -COPY . ${CONTAINER_APP_ROOT} +# Copy application code +COPY . . -EXPOSE ${SERVICE_API_ACCESS_PORT} -# Using shell to expand environment to ensure pass the actual environment value to uvicorn -CMD uvicorn webapi.main:app --reload --port=$SERVICE_API_ACCESS_PORT --host=$SERVICE_API_ACCESS_HOST +# Expose port +EXPOSE 8009 + +# Start command +CMD ["uvicorn", "webapi.main:app", "--host", "0.0.0.0", "--port", "8009"] diff --git a/apps/metrics/README.md b/apps/metrics/README.md new file mode 100644 index 0000000..ac893d0 --- /dev/null +++ b/apps/metrics/README.md @@ -0,0 +1,335 @@ +# Metrics Service + +A FastAPI microservice for user registration statistics and data analytics in the Freeleaps platform. + +## ๐Ÿš€ Overview + +The Metrics service provides comprehensive APIs for querying and analyzing user registration data from the StarRocks database. It offers flexible querying options and statistical summaries for better insights into user growth patterns. + +## โœจ Features + +### ๐Ÿ“Š User Registration Statistics APIs +- **Date Range Query** - Query registration data for specific date ranges +- **Recent N Days Query** - Get registration data for the last N days +- **Start Date + Days Query** - Query N days starting from a specified date +- **Statistics Summary** - Get comprehensive statistics and analytics +- **POST Method Support** - JSON request body support for complex queries + +### ๐Ÿ—„๏ธ Database Integration +- **StarRocks Database Connection** + - Host: `freeleaps-starrocks-cluster-fe-service.freeleaps-data-platform.svc` + - Port: `9030` + - Database: `freeleaps` + - Table: `dws_daily_registered_users` + +### ๐Ÿ”ง Technical Features +- **Data Models**: Pydantic validation for data integrity +- **Connection Management**: Automatic database connection and disconnection +- **Error Handling**: Comprehensive exception handling with user-friendly error messages +- **Logging**: Structured logging using Loguru +- **API Documentation**: Auto-generated Swagger/OpenAPI documentation + +## ๐Ÿ“ Project Structure + +``` +apps/metrics/ +โ”œโ”€โ”€ backend/ +โ”‚ โ”œโ”€โ”€ infra/starrocks_client.py # StarRocks database client +โ”‚ โ”œโ”€โ”€ models/registered_users.py # Data model definitions +โ”‚ โ””โ”€โ”€ services/registration_service.py # Business logic service +โ”œโ”€โ”€ webapi/ +โ”‚ โ”œโ”€โ”€ routes/registration.py # API route definitions +โ”‚ โ”œโ”€โ”€ config/site_settings.py # Configuration management +โ”‚ โ”œโ”€โ”€ bootstrap/application.py # Application initialization +โ”‚ โ””โ”€โ”€ main.py # FastAPI main application +โ”œโ”€โ”€ common/ # Shared utilities +โ”œโ”€โ”€ tests/ # Test files +โ”œโ”€โ”€ requirements.txt # Python dependencies +โ”œโ”€โ”€ Dockerfile # Simplified Docker configuration (Python 3.12-slim) +โ”œโ”€โ”€ start_fastapi.sh # Startup script +โ”œโ”€โ”€ test_registration_api.py # API test script +โ”œโ”€โ”€ .env # Environment configuration +โ””โ”€โ”€ README.md # This file +``` + +## ๐Ÿš€ Quick Start + +### Prerequisites +- Python 3.12+ +- Access to StarRocks database +- pip package manager + +### Installation + +1. **Clone the repository** + ```bash + git clone + cd freeleaps-service-hub/apps/metrics + ``` + +2. **Install dependencies** + ```bash + pip install -r requirements.txt + ``` + +3. **Configure environment** + ```bash + cp .env.example .env # If available + # Edit .env with your configuration + ``` + +4. **Start the service** + + **Option A: Direct Python execution** + ```bash + python3 -m uvicorn webapi.main:app --host 0.0.0.0 --port 8009 --reload + ``` + + **Option B: Using Docker** + ```bash + # Build Docker image + docker build -t metrics:latest . + + # Run container + docker run --rm -p 8009:8009 metrics:latest + ``` + +5. **Access API documentation** + ``` + http://localhost:8009/docs + ``` + +## ๐Ÿ“Š API Endpoints + +### 1. Daily Registered Users (Date Range) +```http +GET /api/metrics/daily-registered-users?start_date=2024-09-10&end_date=2024-09-20&product_id=freeleaps +``` + +**Parameters:** +- `start_date` (required): Start date in YYYY-MM-DD format +- `end_date` (required): End date in YYYY-MM-DD format +- `product_id` (optional): Product identifier (default: "freeleaps") + +### 2. Recent Registered Users +```http +GET /api/metrics/recent-registered-users?days=7&product_id=freeleaps +``` + +**Parameters:** +- `days` (optional): Number of recent days (default: 7, max: 365) +- `product_id` (optional): Product identifier (default: "freeleaps") + +### 3. Registered Users by Days +```http +GET /api/metrics/registered-users-by-days?start_date=2024-09-01&days=5&product_id=freeleaps +``` + +**Parameters:** +- `start_date` (required): Start date in YYYY-MM-DD format +- `days` (required): Number of days from start date (max: 365) +- `product_id` (optional): Product identifier (default: "freeleaps") + +### 4. Registration Summary +```http +GET /api/metrics/registration-summary?start_date=2024-09-10&end_date=2024-09-20&product_id=freeleaps +``` + +**Parameters:** +- `start_date` (required): Start date in YYYY-MM-DD format +- `end_date` (required): End date in YYYY-MM-DD format +- `product_id` (optional): Product identifier (default: "freeleaps") + +### 5. Daily Registered Users (POST) +```http +POST /api/metrics/daily-registered-users +Content-Type: application/json + +{ + "start_date": "2024-09-10", + "end_date": "2024-09-20", + "product_id": "freeleaps" +} +``` + +## ๐Ÿ“ˆ Response Format + +### Standard Response +```json +{ + "dates": ["2024-09-10", "2024-09-11", "2024-09-12"], + "counts": [39, 38, 31], + "total_registrations": 108, + "query_period": "2024-09-10 to 2024-09-12" +} +``` + +### Summary Response +```json +{ + "total_registrations": 282, + "average_daily": 25.64, + "max_daily": 39, + "min_daily": 8, + "days_with_registrations": 10, + "total_days": 11 +} +``` + +## ๐Ÿงช Testing + +### Run API Tests +```bash +python test_registration_api.py +``` + +### Manual Testing Examples +```bash +# Test recent 7 days +curl "http://localhost:8009/api/metrics/recent-registered-users?days=7" + +# Test date range +curl "http://localhost:8009/api/metrics/daily-registered-users?start_date=2024-09-10&end_date=2024-09-20" + +# Test summary +curl "http://localhost:8009/api/metrics/registration-summary?start_date=2024-09-10&end_date=2024-09-20" +``` + +## โš™๏ธ Configuration + +### Environment Variables +```bash +# Server settings +SERVICE_API_ACCESS_HOST=0.0.0.0 +SERVICE_API_ACCESS_PORT=8009 + +# Database settings +MONGODB_URI=mongodb://localhost:27017/ +MONGODB_NAME=freeleaps2 + +# StarRocks settings +STARROCKS_HOST=freeleaps-starrocks-cluster-fe-service.freeleaps-data-platform.svc +STARROCKS_PORT=9030 +STARROCKS_USER=root +STARROCKS_PASSWORD= +STARROCKS_DATABASE=freeleaps + +# Log settings +LOG_BASE_PATH=./logs +BACKEND_LOG_FILE_NAME=metrics +APPLICATION_ACTIVITY_LOG=metrics-activity +``` + +### Docker Deployment + +**Prerequisites:** +- Docker installed and running +- Local Python 3.12+ image available + +**Build and Run:** +```bash +# Build image (using local Python 3.12-slim image) +docker build -t metrics:latest . + +# Run container with port mapping +docker run --rm -p 8009:8009 metrics:latest + +# Run with environment file (if available) +docker run --rm -p 8009:8009 --env-file .env metrics:latest + +# Run in background +docker run -d --name metrics-service -p 8009:8009 metrics:latest +``` + +**Docker Image Details:** +- Base Image: `swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/python:3.12-slim` +- Port: 8009 +- Working Directory: `/app` +- Auto-reload: Disabled (production mode) + +## ๐Ÿ”ง Development + +### Project Setup +```bash +# Create virtual environment +python -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate + +# Install dependencies +pip install -r requirements.txt + +# Run with auto-reload +python -m uvicorn webapi.main:app --reload +``` + +### Code Structure +- **Backend**: Business logic and data access layer +- **WebAPI**: FastAPI routes and application setup +- **Models**: Pydantic data models for validation +- **Services**: Business logic services +- **Infra**: Database clients and infrastructure code + +## ๐Ÿ“ API Documentation + +Once the service is running, you can access the interactive API documentation at: +- **Swagger UI**: `http://localhost:8009/docs` +- **ReDoc**: `http://localhost:8009/redoc` +- **OpenAPI JSON**: `http://localhost:8009/openapi.json` + +## โš ๏ธ Important Notes + +- All APIs support `product_id` parameter (defaults to "freeleaps") +- Date format must be `YYYY-MM-DD` +- Query range is limited to 365 days maximum +- Service requires network access to StarRocks cluster +- Logs are written to `./logs/` directory + +## ๐Ÿ› Troubleshooting + +### Common Issues + +1. **Database Connection Failed** + - Verify StarRocks cluster accessibility + - Check network connectivity + - Validate database credentials + +2. **Import Errors** + - Ensure all dependencies are installed + - Check Python path configuration + - Verify virtual environment activation + +3. **Port Already in Use** + - Change port in configuration + - Stop existing service on port 8009 + - For Docker: `docker stop $(docker ps -q --filter ancestor=metrics:latest)` + +4. **Docker Build Issues** + - Ensure local Python image is available: `docker images | grep python` + - Check Docker daemon is running: `docker ps` + - Verify Dockerfile syntax and paths + +5. **Docker Container Issues** + - Check container logs: `docker logs ` + - Verify port mapping: `docker ps` + - Check container status: `docker ps -a` + +### Logs +Check application logs in the `./logs/` directory for detailed error information. + +## ๐Ÿ“ž Support + +For issues and questions: +- Check the API documentation at `/docs` +- Review logs for error details +- Contact the development team + +## ๐Ÿ“„ License + +This project is part of the Freeleaps platform. + + + + + + + diff --git a/apps/metrics/backend/annotation/__init__.py b/apps/metrics/backend/annotation/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/apps/metrics/backend/application/__init__.py b/apps/metrics/backend/application/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/apps/metrics/backend/business/__init__.py b/apps/metrics/backend/business/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/apps/metrics/backend/infra/starrocks_client.py b/apps/metrics/backend/infra/starrocks_client.py index af7fb97..db58f97 100644 --- a/apps/metrics/backend/infra/starrocks_client.py +++ b/apps/metrics/backend/infra/starrocks_client.py @@ -88,3 +88,12 @@ class StarRocksClient: def __exit__(self, exc_type, exc_val, exc_tb): """Context manager exit""" self.disconnect() + + + + + + + + + diff --git a/apps/metrics/backend/services/registration_service.py b/apps/metrics/backend/services/registration_service.py index 29432c4..8d786e9 100644 --- a/apps/metrics/backend/services/registration_service.py +++ b/apps/metrics/backend/services/registration_service.py @@ -123,3 +123,12 @@ class RegistrationService: except Exception as e: logger.error(f"Failed to get registration summary: {e}") raise e + + + + + + + + + diff --git a/apps/metrics/local.env b/apps/metrics/local.env new file mode 100644 index 0000000..616515e --- /dev/null +++ b/apps/metrics/local.env @@ -0,0 +1,17 @@ +# Local environment configuration for Metrics service +SERVER_HOST=0.0.0.0 +SERVER_PORT=8009 +SERVICE_API_ACCESS_PORT=8009 +SERVICE_API_ACCESS_HOST=0.0.0.0 + +# starrocks settings +STARROCKS_HOST=freeleaps-starrocks-cluster-fe-service.freeleaps-data-platform.svc +STARROCKS_PORT=9030 +STARROCKS_USER=root +STARROCKS_PASSWORD="" +STARROCKS_DATABASE=freeleaps + +# log settings +LOG_BASE_PATH=./logs +BACKEND_LOG_FILE_NAME=metrics +APPLICATION_ACTIVITY_LOG=metrics-activity \ No newline at end of file diff --git a/apps/metrics/start_fastapi.sh b/apps/metrics/start_fastapi.sh deleted file mode 100755 index 7c469ba..0000000 --- a/apps/metrics/start_fastapi.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -rp=$(dirname "$(realpath '$1')") -pushd $rp - -APP_NAME=metrics -APP_PARENT_FOLDER=apps - -GIT_REPO_ROOT=$(git rev-parse --show-toplevel) -CODEBASE_ROOT=$GIT_REPO_ROOT/$APP_PARENT_FOLDER/$APP_NAME -SITE_DEPLOY_FOLDER=$GIT_REPO_ROOT/sites/$APP_NAME/deploy - -echo APP_NAME=$APP_NAME > .env -cat $SITE_DEPLOY_FOLDER/common/.env >> .env -echo GIT_REPO_ROOT=$(git rev-parse --show-toplevel) >> .env -echo CODEBASE_ROOT=$GIT_REPO_ROOT/$APP_PARENT_FOLDER/$APP_NAME >> .env -echo SITE_DEPLOY_FOLDER=$GIT_REPO_ROOT/sites/$APP_NAME/deploy >> .env -cat $SITE_DEPLOY_FOLDER/common/.host.env >> .env -cat $SITE_DEPLOY_FOLDER/local/.env >> .env - -. .env - -if [ -d "$VENV_DIR" ] -then - echo "Folder $VENV_DIR exists. Proceed to next steps" -else - echo "Folder $VENV_DIR doesn't exist. create it" - sudo apt install python3-pip - python3 -m pip install virtualenv - python3 -m virtualenv $VENV_DIR -fi - -source $VENV_DIR/bin/activate -pip install --upgrade pip -pip install -r requirements.txt - -uvicorn webapi.main:app --reload --host 0.0.0.0 --port $SERVICE_API_ACCESS_PORT - -popd diff --git a/apps/metrics/test_registration_api.py b/apps/metrics/test_registration_api.py deleted file mode 100755 index a181b0b..0000000 --- a/apps/metrics/test_registration_api.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python3 -""" -Test script for registration API endpoints -""" -import requests -import json -from datetime import date, timedelta - -# API base URL -BASE_URL = "http://localhost:8009" - -def test_daily_registered_users(): - """Test the daily registered users endpoint""" - print("Testing daily registered users endpoint...") - - # Test with last 7 days - end_date = date.today() - start_date = end_date - timedelta(days=6) - - url = f"{BASE_URL}/api/metrics/daily-registered-users" - params = { - "start_date": str(start_date), - "end_date": str(end_date), - "product_id": "freeleaps" - } - - try: - response = requests.get(url, params=params) - print(f"Status Code: {response.status_code}") - - if response.status_code == 200: - data = response.json() - print(f"Response: {json.dumps(data, indent=2)}") - print(f"Number of days: {len(data['dates'])}") - print(f"Total registrations: {data['total_registrations']}") - else: - print(f"Error: {response.text}") - - except Exception as e: - print(f"Request failed: {e}") - -def test_registration_summary(): - """Test the registration summary endpoint""" - print("\nTesting registration summary endpoint...") - - end_date = date.today() - start_date = end_date - timedelta(days=6) - - url = f"{BASE_URL}/api/metrics/registration-summary" - params = { - "start_date": str(start_date), - "end_date": str(end_date), - "product_id": "freeleaps" - } - - try: - response = requests.get(url, params=params) - print(f"Status Code: {response.status_code}") - - if response.status_code == 200: - data = response.json() - print(f"Summary: {json.dumps(data, indent=2)}") - else: - print(f"Error: {response.text}") - - except Exception as e: - print(f"Request failed: {e}") - -def test_post_method(): - """Test the POST method for daily registered users""" - print("\nTesting POST method for daily registered users...") - - end_date = date.today() - start_date = end_date - timedelta(days=6) - - url = f"{BASE_URL}/api/metrics/daily-registered-users" - payload = { - "start_date": str(start_date), - "end_date": str(end_date), - "product_id": "freeleaps" - } - - try: - response = requests.post(url, json=payload) - print(f"Status Code: {response.status_code}") - - if response.status_code == 200: - data = response.json() - print(f"Response: {json.dumps(data, indent=2)}") - else: - print(f"Error: {response.text}") - - except Exception as e: - print(f"Request failed: {e}") - -if __name__ == "__main__": - print("Starting registration API tests...") - print(f"Testing against: {BASE_URL}") - print("=" * 50) - - test_daily_registered_users() - test_registration_summary() - test_post_method() - - print("\n" + "=" * 50) - print("Tests completed!") diff --git a/apps/metrics/tests/__init__.py b/apps/metrics/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/apps/metrics/webapi/config/site_settings.py b/apps/metrics/webapi/config/site_settings.py index 098d36f..fac31e6 100644 --- a/apps/metrics/webapi/config/site_settings.py +++ b/apps/metrics/webapi/config/site_settings.py @@ -9,24 +9,11 @@ class SiteSettings(BaseSettings): SERVICE_API_ACCESS_HOST: str = "0.0.0.0" SERVICE_API_ACCESS_PORT: int = 8009 - # Database settings - MONGODB_URI: str = "mongodb://localhost:27017/" - MONGODB_NAME: str = "freeleaps2" - MONGODB_PORT: int = 27017 - - # JWT settings - JWT_SECRET_KEY: str = "8f87ca8c3c9c3df09a9c78e0adb0927855568f6072d9efc892534aee35f5867b" - JWT_ALGORITHM: str = "HS256" - # Log settings LOG_BASE_PATH: str = "./logs" BACKEND_LOG_FILE_NAME: str = "metrics" APPLICATION_ACTIVITY_LOG: str = "metrics-activity" - # Service dependencies - DEVSVC_WEBAPI_URL_BASE: str = "http://devsvc:8007/api/devsvc" - NOTIFICATION_WEBAPI_URL_BASE: str = "http://notification:8003/api/notification/" - # StarRocks database settings STARROCKS_HOST: str = "freeleaps-starrocks-cluster-fe-service.freeleaps-data-platform.svc" STARROCKS_PORT: int = 9030 @@ -35,7 +22,7 @@ class SiteSettings(BaseSettings): STARROCKS_DATABASE: str = "freeleaps" class Config: - env_file = ".env" + env_file = "local.env" site_settings = SiteSettings() diff --git a/apps/metrics/webapi/providers/__init__.py b/apps/metrics/webapi/providers/__init__.py deleted file mode 100644 index e69de29..0000000