diff --git a/apps/authentication/Dockerfile b/apps/authentication/Dockerfile index 1439b4b..0b97a04 100644 --- a/apps/authentication/Dockerfile +++ b/apps/authentication/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim-bullseye +FROM python:3.12-slim # docker settings ARG CONTAINER_APP_ROOT="/app" diff --git a/apps/central_storage/Dockerfile b/apps/central_storage/Dockerfile index fbf07c1..f8f2596 100644 --- a/apps/central_storage/Dockerfile +++ b/apps/central_storage/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim-buster +FROM python:3.12-slim # docker settings ARG CONTAINER_APP_ROOT="/app" diff --git a/apps/central_storage/backend/application/document_hub.py b/apps/central_storage/backend/application/document_hub.py index d50ac39..c2e746e 100644 --- a/apps/central_storage/backend/application/document_hub.py +++ b/apps/central_storage/backend/application/document_hub.py @@ -1,6 +1,7 @@ from backend.business.document_manager import ( DocumentManager, ) +from common.log.module_logger import ModuleLogger class DocumentHub: @@ -8,15 +9,22 @@ class DocumentHub: self, ): self.document_manager = DocumentManager() + self.module_logger = ModuleLogger(sender_id="DocumentHub") return async def retrieve_document_info(self, document_id: str): - return await self.document_manager.retrieve_document_info(document_id) + await self.module_logger.log_info(f"Retrieving document info for ID: {document_id}") + result = await self.document_manager.retrieve_document_info(document_id) + await self.module_logger.log_info(f"Document info retrieved successfully for ID: {document_id}") + return result async def read_document_file_as_http_media_data(self, document_id: str): - return await self.document_manager.read_document_file_as_http_media_data( + await self.module_logger.log_info(f"Reading document as HTTP media for ID: {document_id}") + result = await self.document_manager.read_document_file_as_http_media_data( document_id ) + await self.module_logger.log_info(f"Document media data read successfully for ID: {document_id}") + return result async def upload_document( self, associated_with: str, file_name: str, file_data: bytes @@ -26,11 +34,16 @@ class DocumentHub: file_name: the name of the file file (bytes): the file to be uploaded """ - return await self.document_manager.upload_file( + await self.module_logger.log_info(f"Uploading document {file_name} for user {associated_with}") + result = await self.document_manager.upload_file( associated_with, file_name, file_data ) + await self.module_logger.log_info(f"Document upload completed for {file_name}, result: {result}") + return result async def delete_documents(self, document_ids: list): + await self.module_logger.log_info(f"Deleting documents: {document_ids}") for document_id in document_ids: await self.document_manager.delete_document(document_id) + await self.module_logger.log_info(f"Documents deleted successfully: {document_ids}") return diff --git a/apps/central_storage/backend/business/document_manager.py b/apps/central_storage/backend/business/document_manager.py index 08b212e..7e46819 100644 --- a/apps/central_storage/backend/business/document_manager.py +++ b/apps/central_storage/backend/business/document_manager.py @@ -1,10 +1,12 @@ from backend.services.document_service import DocumentService from backend.models.models import MediaType, DataFormat +from common.log.module_logger import ModuleLogger class DocumentManager: def __init__(self) -> None: self.document_service = DocumentService() + self.module_logger = ModuleLogger(sender_id="DocumentManager") async def retrieve_document_info(self, document_id: str): await self.document_service.load_document(document_id=document_id) diff --git a/apps/central_storage/webapi/bootstrap/application.py b/apps/central_storage/webapi/bootstrap/application.py index aae7a72..575d41d 100644 --- a/apps/central_storage/webapi/bootstrap/application.py +++ b/apps/central_storage/webapi/bootstrap/application.py @@ -12,6 +12,7 @@ from webapi.providers import probes from webapi.providers import metrics from .freeleaps_app import FreeleapsApp from common.config.app_settings import app_settings +from common.log.module_logger import ModuleLogger def create_app() -> FastAPI: @@ -20,6 +21,10 @@ def create_app() -> FastAPI: app = FreeleapsApp() register_logger() + + # Create application logger for startup logging + app_logger = ModuleLogger(sender_id="ApplicationBootstrap") + register(app, exception_handler) register(app, database) register(app, router) @@ -37,6 +42,14 @@ def create_app() -> FastAPI: if app_settings.METRICS_ENABLED: register(app, metrics) + # Log application startup completion + import asyncio + async def log_startup(): + await app_logger.log_info("Central Storage application initialized successfully") + + # Run the async logging + asyncio.create_task(log_startup()) + return app diff --git a/apps/content/Dockerfile b/apps/content/Dockerfile index bfad1fd..efc13d4 100644 --- a/apps/content/Dockerfile +++ b/apps/content/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim-buster +FROM python:3.12-slim # docker settings ARG CONTAINER_APP_ROOT="/app" diff --git a/apps/content/backend/content/content_service.py b/apps/content/backend/content/content_service.py index b7c16ac..2179958 100644 --- a/apps/content/backend/content/content_service.py +++ b/apps/content/backend/content/content_service.py @@ -6,15 +6,18 @@ from common.constants.region import UserRegion from backend.document.document_manager import DocumentManager from .content_sharepoint_manager import ContentSharePointManager from backend.content.constants import ContentSource +from common.log.module_logger import ModuleLogger class ContentService: def __init__(self) -> None: - pass + self.module_logger = ModuleLogger(sender_id="ContentService") async def retrieve_content_directories_for_folder( self, folder_name: str, region: UserRegion ) -> List[ContentDirectory]: + await self.module_logger.log_info(f"Retrieving content directories for folder: {folder_name}, region: {region.name}") + folder = ( await ContentFolderDoc.find( ContentFolderDoc.folder_name == folder_name, @@ -26,12 +29,18 @@ class ContentService: if folder is None or folder.valid_thru.replace( tzinfo=timezone.utc ) < datetime.now(timezone.utc): + await self.module_logger.log_info(f"Folder cache expired or not found, fetching from SharePoint: {folder_name}") folder = await ContentSharePointManager().retrieve_directories_for_folder( folder_name=folder_name, region=region ) - return folder.content_directories if folder else None + result = folder.content_directories if folder else None + await self.module_logger.log_info(f"Successfully retrieved {len(result) if result else 0} content directories for folder: {folder_name}") + return result async def retrieve_content_as_media_data(self, document_id: str) -> Optional[str]: + await self.module_logger.log_info(f"Retrieving content as media data for document ID: {document_id}") document_manager = DocumentManager() - return await document_manager.retrieve_document_as_http_media(document_id) + result = await document_manager.retrieve_document_as_http_media(document_id) + await self.module_logger.log_info(f"Successfully retrieved media data for document ID: {document_id}") + return result diff --git a/apps/content/webapi/bootstrap/application.py b/apps/content/webapi/bootstrap/application.py index 038f61b..4c076d1 100644 --- a/apps/content/webapi/bootstrap/application.py +++ b/apps/content/webapi/bootstrap/application.py @@ -13,6 +13,7 @@ from webapi.providers import probes from webapi.providers import metrics from .freeleaps_app import FreeleapsApp from common.config.app_settings import app_settings +from common.log.module_logger import ModuleLogger def create_app() -> FastAPI: @@ -21,6 +22,10 @@ def create_app() -> FastAPI: app = FreeleapsApp() register_logger() + + # Create application logger for startup logging + app_logger = ModuleLogger(sender_id="ApplicationBootstrap") + register(app, exception_handler) register(app, database) register(app, router) @@ -37,6 +42,15 @@ def create_app() -> FastAPI: # Register metrics APIs if enabled if app_settings.METRICS_ENABLED: register(app, metrics) + + # Log application startup completion + import asyncio + async def log_startup(): + await app_logger.log_info("Content application initialized successfully") + + # Run the async logging + asyncio.create_task(log_startup()) + return app diff --git a/apps/devops/Dockerfile b/apps/devops/Dockerfile index 1d96106..6b07b97 100644 --- a/apps/devops/Dockerfile +++ b/apps/devops/Dockerfile @@ -15,7 +15,7 @@ RUN pip install --no-cache-dir -r requirements.txt COPY . . # Set environment variables -ENV LOG_BASE_PATH=/app/log/devsvc +ENV LOG_BASE_PATH=/app/log/devops # Create necessary directories RUN mkdir -p /app/log/devops diff --git a/apps/devops/app/bootstrap/application.py b/apps/devops/app/bootstrap/application.py index 3cf635b..0528ed0 100644 --- a/apps/devops/app/bootstrap/application.py +++ b/apps/devops/app/bootstrap/application.py @@ -11,6 +11,7 @@ from app.providers import probes from app.providers import exception_handler from app.providers import message_queue from app.common.config.app_settings import app_settings +from app.common.log.module_logger import ModuleLogger def create_app() -> FastAPI: logging.info("App initializing") @@ -33,6 +34,16 @@ def create_app() -> FastAPI: # Register metrics APIs if enabled if app_settings.METRICS_ENABLED: register(app, metrics) + + # Add startup logging + @app.on_event("startup") + async def startup_logging(): + module_logger = ModuleLogger(sender_id="ApplicationBootstrap") + await module_logger.log_info( + text=f"DevOps service started successfully in {app_settings.APP_ENV} environment", + data={"app_name": app_settings.APP_NAME, "environment": app_settings.APP_ENV} + ) + return app diff --git a/apps/devops/app/common/config/app_settings.py b/apps/devops/app/common/config/app_settings.py index d7b3c7d..8717b20 100644 --- a/apps/devops/app/common/config/app_settings.py +++ b/apps/devops/app/common/config/app_settings.py @@ -2,7 +2,7 @@ from pydantic_settings import BaseSettings # NOTE: The values fall backs to your environment variables when not set here class AppSettings(BaseSettings): - NAME: str = "YOUR_APP_NAME" + NAME: str = "devops" APP_NAME: str = NAME APP_ENV: str = "alpha" @@ -17,8 +17,8 @@ class AppSettings(BaseSettings): APP_MONGODB_NAME: str = "testdb" LOG_BASE_PATH: str = "./log" - BACKEND_LOG_FILE_NAME: str = APP_NAME - APPLICATION_ACTIVITY_LOG: str = APP_NAME + "-application-activity"\ + BACKEND_LOG_FILE_NAME: str = "devops" + APPLICATION_ACTIVITY_LOG: str = "devops-application-activity"\ RABBITMQ_HOST: str = "localhost" RABBITMQ_PORT: int = 5672 diff --git a/apps/notification/Dockerfile b/apps/notification/Dockerfile index dce32fc..810fa6d 100644 --- a/apps/notification/Dockerfile +++ b/apps/notification/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim-buster +FROM python:3.12-slim # docker settings ARG CONTAINER_APP_ROOT=/app diff --git a/apps/notification/webapi/bootstrap/application.py b/apps/notification/webapi/bootstrap/application.py index 65adca9..130ff17 100644 --- a/apps/notification/webapi/bootstrap/application.py +++ b/apps/notification/webapi/bootstrap/application.py @@ -14,6 +14,7 @@ from webapi.providers import metrics from webapi.providers import middleware from .freeleaps_app import FreeleapsApp from common.config.app_settings import app_settings +from common.log.module_logger import ModuleLogger from prometheus_fastapi_instrumentator import Instrumentator diff --git a/apps/notification/webapi/bootstrap/freeleaps_app.py b/apps/notification/webapi/bootstrap/freeleaps_app.py index dca6d6e..bc6b03b 100644 --- a/apps/notification/webapi/bootstrap/freeleaps_app.py +++ b/apps/notification/webapi/bootstrap/freeleaps_app.py @@ -21,8 +21,14 @@ class FreeleapsApp(FastAPI): @self.on_event("startup") async def start_consumers(): print("starting up!") + + # Consumer registration first await self.sms_handler.register_consumer() await self.email_handler.register_consumer() + + # Note: If we want startup logging for audit purposes, + # it should be done AFTER all critical startup is complete + # and database is ready. Currently keeping minimal startup. @self.on_event("shutdown") async def stop_consumers(): diff --git a/apps/notification/webapi/middleware/__init__.py b/apps/notification/webapi/middleware/__init__.py index e607c44..d1b6f8b 100644 --- a/apps/notification/webapi/middleware/__init__.py +++ b/apps/notification/webapi/middleware/__init__.py @@ -1,4 +1,4 @@ from .freeleaps_auth_middleware import FreeleapsAuthMiddleware -from .tenant_DBConnection_middleware import TenantDBConnectionMiddleware +from .database_middleware import DatabaseMiddleware -__all__ = ['FreeleapsAuthMiddleware', 'TenantDBConnectionMiddleware'] \ No newline at end of file +__all__ = ['FreeleapsAuthMiddleware', 'DatabaseMiddleware'] \ No newline at end of file diff --git a/apps/notification/webapi/middleware/tenant_DBConnection_middleware.py b/apps/notification/webapi/middleware/database_middleware.py similarity index 96% rename from apps/notification/webapi/middleware/tenant_DBConnection_middleware.py rename to apps/notification/webapi/middleware/database_middleware.py index d568cd9..e9f12db 100644 --- a/apps/notification/webapi/middleware/tenant_DBConnection_middleware.py +++ b/apps/notification/webapi/middleware/database_middleware.py @@ -4,10 +4,10 @@ from webapi.middleware.freeleaps_auth_middleware import request_context_var from common.log.module_logger import ModuleLogger -class TenantDBConnectionMiddleware: +class DatabaseMiddleware: def __init__(self, app): self.app = app - self.module_logger = ModuleLogger(sender_id=TenantDBConnectionMiddleware) + self.module_logger = ModuleLogger(sender_id=DatabaseMiddleware) async def __call__(self, scope, receive, send): if scope["type"] != "http": diff --git a/apps/notification/webapi/providers/database.py b/apps/notification/webapi/providers/database.py index 7f6fc01..1dc1843 100644 --- a/apps/notification/webapi/providers/database.py +++ b/apps/notification/webapi/providers/database.py @@ -32,15 +32,16 @@ tenant_document_models = [ class TenantDBCache: """ - Enhanced tenant database cache that includes Beanie initialization. - product_id -> (AsyncIOMotorClient, AsyncIOMotorDatabase, beanie_initialized: bool) - Uses main_db.tenant_doc to resolve mongodb_uri; caches clients/dbs with LRU and Beanie state. + Enhanced tenant database cache that caches only clients, not databases. + product_id -> AsyncIOMotorClient + Uses main_db.tenant_doc to resolve mongodb_uri; caches clients with LRU. + Database instances are created fresh each time from cached clients. """ def __init__(self, main_db: AsyncIOMotorDatabase, max_size: int = 64): self.main_db = main_db self.max_size = max_size - self._cache: "OrderedDict[str, tuple[AsyncIOMotorClient, AsyncIOMotorDatabase, bool]]" = OrderedDict() + self._cache: "OrderedDict[str, AsyncIOMotorClient]" = OrderedDict() self._locks: dict[str, asyncio.Lock] = {} self._global_lock = asyncio.Lock() self.module_logger = ModuleLogger(sender_id="TenantDBCache") @@ -48,43 +49,43 @@ class TenantDBCache: async def get_initialized_db(self, product_id: str) -> AsyncIOMotorDatabase: """Get tenant database with Beanie already initialized""" - # fast-path - cached = self._cache.get(product_id) - if cached: - client, db, beanie_initialized = cached - await self.module_logger.log_info(f"Found cached database for {product_id}, beanie_initialized: {beanie_initialized}") + # fast-path: check if client is cached + cached_client = self._cache.get(product_id) + if cached_client: + await self.module_logger.log_info(f"Found cached client for {product_id}") self._cache.move_to_end(product_id) - if beanie_initialized: + # Get fresh database instance from cached client + db = cached_client.get_default_database() + if db is not None: + # Initialize Beanie for this fresh database instance + await init_beanie(database=db, document_models=tenant_document_models) + await self.module_logger.log_info(f"Beanie initialization completed for {product_id} using cached client") return db else: - # Initialize Beanie if not done yet - await init_beanie(database=db, document_models=tenant_document_models) - await self.module_logger.log_info(f"Beanie initialization completed for {product_id}") - - # Update cache with beanie_initialized = True - self._cache[product_id] = (client, db, True) - return db + await self.module_logger.log_error(f"No default database found for cached client {product_id}") + # Remove invalid cached client + del self._cache[product_id] # double-checked under per-tenant lock lock = self._locks.setdefault(product_id, asyncio.Lock()) async with lock: - cached = self._cache.get(product_id) - if cached: - client, db, beanie_initialized = cached - await self.module_logger.log_info(f"Double-check found cached database for {product_id}, beanie_initialized: {beanie_initialized}") + cached_client = self._cache.get(product_id) + if cached_client: + await self.module_logger.log_info(f"Double-check found cached client for {product_id}") self._cache.move_to_end(product_id) - if beanie_initialized: + # Get fresh database instance from cached client + db = cached_client.get_default_database() + if db is not None: + # Initialize Beanie for this fresh database instance + await init_beanie(database=db, document_models=tenant_document_models) + await self.module_logger.log_info(f"Beanie initialization completed for {product_id} using cached client (double-check)") return db else: - # Initialize Beanie if not done yet - await init_beanie(database=db, document_models=tenant_document_models) - await self.module_logger.log_info(f"Beanie initialization completed for {product_id} (double-check)") - - # Update cache with beanie_initialized = True - self._cache[product_id] = (client, db, True) - return db + await self.module_logger.log_error(f"No default database found for cached client {product_id}") + # Remove invalid cached client + del self._cache[product_id] # Create new tenant connection - use raw MongoDB query since we don't have TenantDoc model """ @@ -127,13 +128,14 @@ class TenantDBCache: detail=f"No default database found for tenant {product_id}", headers={"X-Error-Message": f"No default database found for tenant {product_id}"} ) + # Initialize Beanie for this tenant database await init_beanie(database=db, document_models=tenant_document_models) await self.module_logger.log_info(f"Beanie initialization completed for new tenant database {product_id}") - # LRU put with beanie_initialized = True - await self._lru_put(product_id, (client, db, True)) - await self.module_logger.log_info(f"Tenant database {product_id} cached successfully with Beanie initialized") + # Cache only the client + await self._lru_put(product_id, client) + await self.module_logger.log_info(f"Tenant client {product_id} cached successfully") return db async def get_main_db_initialized(self) -> AsyncIOMotorDatabase: @@ -143,12 +145,12 @@ class TenantDBCache: await self.module_logger.log_info("Beanie initialization completed for main database") return self.main_db - async def _lru_put(self, key: str, value: tuple[AsyncIOMotorClient, AsyncIOMotorDatabase, bool]): + async def _lru_put(self, key: str, client: AsyncIOMotorClient): async with self._global_lock: - self._cache[key] = value + self._cache[key] = client self._cache.move_to_end(key) if len(self._cache) > self.max_size: - old_key, (old_client, _, _) = self._cache.popitem(last=False) + old_key, old_client = self._cache.popitem(last=False) await self.module_logger.log_info(f"Cache full, removing LRU tenant: {old_key}") try: old_client.close() @@ -159,7 +161,7 @@ class TenantDBCache: async def aclose(self): async with self._global_lock: - for key, (client, _, _) in self._cache.items(): + for key, client in self._cache.items(): try: client.close() await self.module_logger.log_info(f"Closed connection for tenant: {key}") diff --git a/apps/notification/webapi/providers/middleware.py b/apps/notification/webapi/providers/middleware.py index ee47f3e..f11c216 100644 --- a/apps/notification/webapi/providers/middleware.py +++ b/apps/notification/webapi/providers/middleware.py @@ -1,5 +1,5 @@ from webapi.middleware.freeleaps_auth_middleware import FreeleapsAuthMiddleware -from webapi.middleware.tenant_DBConnection_middleware import TenantDBConnectionMiddleware +from webapi.middleware.database_middleware import DatabaseMiddleware def register(app): @@ -7,5 +7,5 @@ def register(app): Register middleware to FastAPI application """ # Register middlewares - app.add_middleware(TenantDBConnectionMiddleware) + app.add_middleware(DatabaseMiddleware) app.add_middleware(FreeleapsAuthMiddleware) \ No newline at end of file diff --git a/apps/payment/Dockerfile b/apps/payment/Dockerfile index f5ac56d..d5e55d5 100644 --- a/apps/payment/Dockerfile +++ b/apps/payment/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim-buster +FROM python:3.12-slim # docker settings ARG CONTAINER_APP_ROOT=/app