From c25252ceda4a73c162a908a0c01e5c97e0a36ff3 Mon Sep 17 00:00:00 2001 From: Jet Li Date: Sat, 19 Oct 2024 05:37:34 +0000 Subject: [PATCH] Initial check-in for central_storage --- app/central_storage/Dockerfile | 19 ++ .../backend/application/document_app.py | 15 ++ .../backend/business/document_manager.py | 19 ++ .../infra/azure_storage/blob_manager.py | 124 +++++++++++++ .../backend/models/constants.py | 13 ++ app/central_storage/backend/models/models.py | 22 +++ .../backend/services/document_service.py | 168 ++++++++++++++++++ .../common/config/app_settings.py | 18 ++ .../common/config/log_settings.py | 17 ++ app/central_storage/requirements.txt | 3 + app/central_storage/start_central_storage.sh | 4 + .../webapi/bootstrap/application.py | 75 ++++++++ .../webapi/bootstrap/freeleaps_app.py | 6 + .../webapi/config/site_settings.py | 25 +++ app/central_storage/webapi/main.py | 30 ++++ .../webapi/providers/common.py | 31 ++++ .../webapi/providers/database.py | 17 ++ .../webapi/providers/exception_handler.py | 39 ++++ .../webapi/providers/logger.py | 60 +++++++ .../webapi/providers/router.py | 34 ++++ .../webapi/providers/scheduler.py | 8 + app/central_storage/webapi/routes/__init__.py | 5 + app/central_storage/webapi/routes/api.py | 15 ++ .../webapi/routes/get_document_by_id.py | 34 ++++ docker-compose.yaml | 50 +++--- infra/config/app_settings.py | 15 ++ infra/config/log_settings.py | 17 ++ infra/exception/__init__.py | 0 infra/exception/exceptions.py | 23 +++ infra/log/__init__.py | 0 infra/log/application_logger.py | 14 ++ infra/log/base_logger.py | 139 +++++++++++++++ infra/log/business_metric_logger.py | 25 +++ infra/log/function_logger.py | 50 ++++++ infra/log/log_utils.py | 25 +++ infra/log/module_logger.py | 46 +++++ infra/log/user_logger.py | 14 ++ requirements.txt | 1 - sites/central_storage/.env | 1 + 39 files changed, 1196 insertions(+), 25 deletions(-) create mode 100644 app/central_storage/Dockerfile create mode 100644 app/central_storage/backend/application/document_app.py create mode 100644 app/central_storage/backend/business/document_manager.py create mode 100644 app/central_storage/backend/infra/azure_storage/blob_manager.py create mode 100644 app/central_storage/backend/models/constants.py create mode 100644 app/central_storage/backend/models/models.py create mode 100644 app/central_storage/backend/services/document_service.py create mode 100644 app/central_storage/common/config/app_settings.py create mode 100644 app/central_storage/common/config/log_settings.py create mode 100644 app/central_storage/requirements.txt create mode 100755 app/central_storage/start_central_storage.sh create mode 100644 app/central_storage/webapi/bootstrap/application.py create mode 100644 app/central_storage/webapi/bootstrap/freeleaps_app.py create mode 100644 app/central_storage/webapi/config/site_settings.py create mode 100755 app/central_storage/webapi/main.py create mode 100644 app/central_storage/webapi/providers/common.py create mode 100644 app/central_storage/webapi/providers/database.py create mode 100644 app/central_storage/webapi/providers/exception_handler.py create mode 100644 app/central_storage/webapi/providers/logger.py create mode 100644 app/central_storage/webapi/providers/router.py create mode 100644 app/central_storage/webapi/providers/scheduler.py create mode 100644 app/central_storage/webapi/routes/__init__.py create mode 100644 app/central_storage/webapi/routes/api.py create mode 100644 app/central_storage/webapi/routes/get_document_by_id.py create mode 100644 infra/config/app_settings.py create mode 100644 infra/config/log_settings.py create mode 100644 infra/exception/__init__.py create mode 100644 infra/exception/exceptions.py create mode 100644 infra/log/__init__.py create mode 100644 infra/log/application_logger.py create mode 100644 infra/log/base_logger.py create mode 100644 infra/log/business_metric_logger.py create mode 100644 infra/log/function_logger.py create mode 100644 infra/log/log_utils.py create mode 100644 infra/log/module_logger.py create mode 100644 infra/log/user_logger.py delete mode 100644 requirements.txt create mode 100644 sites/central_storage/.env diff --git a/app/central_storage/Dockerfile b/app/central_storage/Dockerfile new file mode 100644 index 0000000..1ae22ec --- /dev/null +++ b/app/central_storage/Dockerfile @@ -0,0 +1,19 @@ +# Dockerfile for Python Service +FROM python:3.11-slim + +# Set the working directory inside the container +WORKDIR /app + +# Copy the requirements.txt to the working directory and install dependencies +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +# Copy the application code to the working directory +COPY . /app + +# Expose the port used by the FastAPI app +EXPOSE 8005 + + +# Run the application using the start script +CMD ["uvicorn", "webapi.main:app", "--reload", "--port=8003", "--host=0.0.0.0"] diff --git a/app/central_storage/backend/application/document_app.py b/app/central_storage/backend/application/document_app.py new file mode 100644 index 0000000..9738a2b --- /dev/null +++ b/app/central_storage/backend/application/document_app.py @@ -0,0 +1,15 @@ +from app.central_storage.backend.business.document_manager import ( + DocumentBusinessManager, +) + + +class DocumentHub: + def __init__(self, user_id: str): + self.user_id = user_id + self.document_business_manager = DocumentBusinessManager(self.user_id) + return + + async def get_document_by_id(self, document_id: str): + return await self.document_business_manager.get_document_details_by_id( + document_id + ) diff --git a/app/central_storage/backend/business/document_manager.py b/app/central_storage/backend/business/document_manager.py new file mode 100644 index 0000000..88e78b1 --- /dev/null +++ b/app/central_storage/backend/business/document_manager.py @@ -0,0 +1,19 @@ +from app.central_storage.backend.services.document_service import DocumentService + + +class DocumentBusinessManager: + def __init__(self, user_id) -> None: + self.user_id = user_id + self.document_service = DocumentService() + + async def get_document_details_by_id(self, document_id: str): + await self.document_service.load_document(document_id=document_id) + + download_link = ( + await self.document_service.fetch_document_file_as_http_download() + ) + file_name = self.document_service.get_file_name() + return { + "file_name": file_name, + "file_download_url": download_link, + } diff --git a/app/central_storage/backend/infra/azure_storage/blob_manager.py b/app/central_storage/backend/infra/azure_storage/blob_manager.py new file mode 100644 index 0000000..943f2c0 --- /dev/null +++ b/app/central_storage/backend/infra/azure_storage/blob_manager.py @@ -0,0 +1,124 @@ +from app.central_storage.common.config.app_settings import app_settings +import datetime + +import asyncio +from azure.storage.blob.aio import ( + BlobServiceClient, + ContainerClient, + BlobClient, +) +from azure.storage.blob import ( + BlobSasPermissions, + UserDelegationKey, + generate_container_sas, + generate_blob_sas, +) +from azure.core.exceptions import ResourceExistsError + + +class AzureBlobManager: + def __init__(self) -> None: + pass + + def __create_user_delegation_sas_token__( + self, + blob_client: BlobClient, + user_delegation_key: UserDelegationKey, + valid_minutes: int, + ): + # Create a SAS token that's valid for one day, as an example + start_time = datetime.datetime.now(datetime.timezone.utc) + expiry_time = start_time + datetime.timedelta(minutes=valid_minutes) + + sas_token = generate_blob_sas( + account_name=blob_client.account_name, + container_name=blob_client.container_name, + blob_name=blob_client.blob_name, + user_delegation_key=user_delegation_key, + permission=BlobSasPermissions(read=True), + expiry=expiry_time, + start=start_time, + ) + return sas_token + + async def create_container(self, container_name: str): + try: + async with BlobServiceClient( + app_settings.AZURE_STORAGE_DOCUMENT_API_ENDPOINT, + credential=app_settings.AZURE_STORAGE_DOCUMENT_API_KEY, + ) as blob_service_client: + await blob_service_client.create_container(name=container_name) + except ResourceExistsError: + # swallow the error for the resource existing error + pass + + async def delete_container(self, container_name: str): + raise NotImplementedError("Deleting container is not supported") + + async def upload_blob(self, container_name: str, blob_name: str, blob_data: bytes): + async with ContainerClient( + app_settings.AZURE_STORAGE_DOCUMENT_API_ENDPOINT, + credential=app_settings.AZURE_STORAGE_DOCUMENT_API_KEY, + container_name=container_name, + ) as container_client: + if not await container_client.exists(): + await container_client.create_container() + + await container_client.upload_blob( + name=blob_name, + data=blob_data, + length=len(blob_data), + blob_type="BlockBlob", + overwrite=True, + ) + + async def delete_blob(self, container_name: str, blob_name: str): + async with BlobServiceClient( + app_settings.AZURE_STORAGE_DOCUMENT_API_ENDPOINT, + credential=app_settings.AZURE_STORAGE_DOCUMENT_API_KEY, + ) as blob_service_client: + blob_client = await blob_service_client.get_blob_client( + container=container_name, blob=blob_name + ) + await blob_client.delete_blob() + + async def generate_download_url(self, container_name: str, file_name: str): + token_valid_minutes = 10 + key_start_time = datetime.datetime.now(datetime.timezone.utc) + key_expiry_time = key_start_time + datetime.timedelta( + minutes=token_valid_minutes + ) + async with BlobServiceClient( + app_settings.AZURE_STORAGE_DOCUMENT_API_ENDPOINT, + credential=app_settings.AZURE_STORAGE_DOCUMENT_API_KEY, + ) as blob_service_client: + blob_client = blob_service_client.get_blob_client( + container=container_name, blob=file_name + ) + + sas_token = generate_blob_sas( + account_name=blob_client.account_name, + container_name=blob_client.container_name, + blob_name=blob_client.blob_name, + # user_delegation_key=user_delegation_key, + account_key=app_settings.AZURE_STORAGE_DOCUMENT_API_KEY, + permission=BlobSasPermissions(read=True), + expiry=key_expiry_time, + start=key_start_time, + ) + sas_url = f"{blob_client.url}?{sas_token}" + return sas_url + + async def fetch_blob_data_as_bytes( + self, container_name: str, file_name: str + ) -> bytes: + async with BlobServiceClient( + app_settings.AZURE_STORAGE_DOCUMENT_API_ENDPOINT, + credential=app_settings.AZURE_STORAGE_DOCUMENT_API_KEY, + ) as blob_service_client: + blob_client = blob_service_client.get_blob_client( + container=container_name, blob=file_name + ) + downloader = await blob_client.download_blob(max_concurrency=1) + blob_bytes = await downloader.readall() + return blob_bytes diff --git a/app/central_storage/backend/models/constants.py b/app/central_storage/backend/models/constants.py new file mode 100644 index 0000000..59dcb61 --- /dev/null +++ b/app/central_storage/backend/models/constants.py @@ -0,0 +1,13 @@ +from enum import IntEnum + + +class MediaType(IntEnum): + UNKNOWN = 0 + PDF = 1 + PNG = 2 + TXT = 3 + + +class DataFormat(IntEnum): + RAW = 0 + BASED64 = 1 diff --git a/app/central_storage/backend/models/models.py b/app/central_storage/backend/models/models.py new file mode 100644 index 0000000..c146dc6 --- /dev/null +++ b/app/central_storage/backend/models/models.py @@ -0,0 +1,22 @@ +from datetime import datetime +from typing import Optional + +from beanie import Document +from .constants import MediaType, DataFormat + + +class DocumentDoc(Document): + document_id: Optional[str] + file_name: str + created_by: str + create_time: datetime + updated_by: str + update_time: datetime + location: Optional[str] + version_number: int + media_type: MediaType + data_format: DataFormat + is_deleted: Optional[bool] = False + + class Settings: + name = "document" diff --git a/app/central_storage/backend/services/document_service.py b/app/central_storage/backend/services/document_service.py new file mode 100644 index 0000000..7cd794d --- /dev/null +++ b/app/central_storage/backend/services/document_service.py @@ -0,0 +1,168 @@ +from datetime import datetime as dt, timezone as tz +from app.central_storage.backend.models.models import DocumentDoc +from app.central_storage.backend.models.constants import MediaType, DataFormat + +from infra.exception.exceptions import DoesNotExistError +from app.central_storage.backend.infra.azure_storage.blob_manager import ( + AzureBlobManager, +) +import base64 +import os +from re import match + + +class DocumentService: + def __init__(self) -> None: + self.__document_doc = None + self.blob_manager = AzureBlobManager() + return + + def __normalize_file_name__(file_name: str) -> str: + # There are some restriction on file naming, while we want to save the client file name + # so every time we will generate an internal file name + return file_name.strip("./\\ ") + + def __retrieve_media_type_from_file_name__(file_name) -> MediaType: + extension = os.path.splitext(file_name)[1][1:] + if extension == None or extension == "": + return MediaType.UNKNOWN + elif extension.lower() == "png": + return MediaType.PNG + elif extension.lower() == "pdf": + return MediaType.PDF + elif extension.lower() == "txt": + return MediaType.TXT + else: + return MediaType.UNKNOWN + + def __get_prefix_for_media_data__(self) -> str: + return "data:{};base64,".format(self.get_file_media_type_string()) + + def __validate_document_doc(self) -> None: + if not self.__document_doc: + raise Exception( + status_code=500, + detail="Please call new_document() or load_document() to initialize the internal variables.", + ) + return + + def __get_container_name__(self): + self.__validate_document_doc() + + return self.__document_doc.created_by + + def __get_blob_name__(self) -> str: + self.__validate_document_doc() + + return self.__document_doc.location + "-" + self.__document_doc.file_name + + async def load_document(self, document_id: str) -> None: + document = await DocumentDoc.find_one(DocumentDoc.document_id == document_id) + if not document: + raise DoesNotExistError( + "Cannot find record with document id {document_id}".format( + self.document_id + ), + ) + self.__document_doc = document + return + + async def new_document( + self, + file_name: str, + media_type: MediaType, + data_format: DataFormat, + created_by: str, + ) -> None: + # We just do some simple check here. For detailed requirement, please read + # https://learn.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata + if not bool(match("^[a-z0-9\-]{4,64}$", created_by)): + raise ValueError( + "invalid 'created_by':'{}'. It can contain lower case alphabet, digits, and hyphens with length between 4 and 64".format( + created_by + ) + ) + + document = DocumentDoc( + document_id=None, + location=None, + file_name=DocumentManager.__normalize_file_name__(file_name), + created_by=created_by, + create_time=dt.now(tz.utc), + updated_by=created_by, + update_time=dt.now(tz.utc), + version_number=1, + media_type=( + DocumentManager.__retrieve_media_type_from_file_name__(file_name) + if media_type == MediaType.UNKNOWN + else media_type + ), + data_format=data_format, + ) + + new_document = await document.create() + + # Set document_id and location + location = "{}-{}".format(created_by, new_document.id) + new_document.document_id = str(new_document.id) + new_document.location = location + await new_document.save() + + self.__document_doc = new_document + return + + async def save_document_file(self, file_raw: bytes) -> str: + self.__validate_document_doc() + await self.blob_manager.upload_blob( + container_name=self.__get_container_name__(), + blob_name=self.__get_blob_name__(), + blob_data=file_raw, + ) + return self.__document_doc.document_id + + async def remove_document_file(self) -> None: + self.__validate_document_doc() + + self.__document_doc.is_deleted = True + self.__document_doc.save() + await self.blob_manager.delete_blob( + container_name=self.__get_container_name__(), + blob_name=self.__get_blob_name__(), + ) + + async def read_document_file_as_http_media_data(self) -> str: + self.__validate_document_doc() + raw_data = await self.blob_manager.fetch_blob_data_as_bytes( + container_name=self.__get_container_name__(), + file_name=self.__get_blob_name__(), + ) + base64_data = base64.b64encode(raw_data).decode() + return self.__get_prefix_for_media_data__() + base64_data + + async def fetch_document_file_as_http_download(self) -> str: + self.__validate_document_doc() + + return await self.blob_manager.generate_download_url( + container_name=self.__get_container_name__(), + file_name=self.__get_blob_name__(), + ) + + def get_file_name(self) -> str: + self.__validate_document_doc() + + return self.__document_doc.file_name + + def get_file_media_type_string(self) -> str: + self.__validate_document_doc() + media_type = "application/octet-stream" + if self.__document_doc.media_type == MediaType.PNG: + media_type = "image/png" + elif self.__document_doc.media_type == MediaType.PDF: + media_type = "application/pdf" + elif self.__document_doc.media_type == MediaType.TXT: + media_type = "text/plain" + return media_type + + def get_document_id(self) -> str: + self.__validate_document_doc() + return self.__document_doc.document_id diff --git a/app/central_storage/common/config/app_settings.py b/app/central_storage/common/config/app_settings.py new file mode 100644 index 0000000..f2028e1 --- /dev/null +++ b/app/central_storage/common/config/app_settings.py @@ -0,0 +1,18 @@ +import os +from pydantic_settings import BaseSettings + + +class AppSettings(BaseSettings): + NAME: str = "central_storage" + + AZURE_STORAGE_DOCUMENT_API_ENDPOINT: str = ( + "https://freeleaps1document.blob.core.windows.net/" + ) + AZURE_STORAGE_DOCUMENT_API_KEY: str = "" + + class Config: + env_file = ".myapp.env" + env_file_encoding = "utf-8" + + +app_settings = AppSettings() diff --git a/app/central_storage/common/config/log_settings.py b/app/central_storage/common/config/log_settings.py new file mode 100644 index 0000000..856281d --- /dev/null +++ b/app/central_storage/common/config/log_settings.py @@ -0,0 +1,17 @@ + + +class LogSettings(): + LOG_LEVEL: str = "DEBUG" + LOG_PATH_BASE: str = ( + "./logs" + ) + LOG_PATH: str = LOG_PATH_BASE + '/' + "app" + '.log' + LOG_RETENTION: str = "14 days" + LOG_ROTATION: str = "00:00" # mid night + + class Config: + env_file = ".log.env" + env_file_encoding = "utf-8" + + +log_settings = LogSettings() diff --git a/app/central_storage/requirements.txt b/app/central_storage/requirements.txt new file mode 100644 index 0000000..5301cc4 --- /dev/null +++ b/app/central_storage/requirements.txt @@ -0,0 +1,3 @@ +pika==1.3.2 +fastapi +uvicorn \ No newline at end of file diff --git a/app/central_storage/start_central_storage.sh b/app/central_storage/start_central_storage.sh new file mode 100755 index 0000000..da25767 --- /dev/null +++ b/app/central_storage/start_central_storage.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# This script starts the FastAPI server using uvicorn +uvicorn webapi.bootstrap.application:create_app --reload --host 0.0.0.0 --port 8005 \ No newline at end of file diff --git a/app/central_storage/webapi/bootstrap/application.py b/app/central_storage/webapi/bootstrap/application.py new file mode 100644 index 0000000..7a38aa7 --- /dev/null +++ b/app/central_storage/webapi/bootstrap/application.py @@ -0,0 +1,75 @@ +import logging +from fastapi import FastAPI +from fastapi.openapi.utils import get_openapi + +from webapi.providers import common +from webapi.providers import logger +from webapi.providers import router +from webapi.providers import database +from webapi.providers import scheduler +from webapi.providers import exception_handler +from .freeleaps_app import FreeleapsApp + + +def create_app() -> FastAPI: + logging.info("App initializing") + + app = FreeleapsApp() + + register(app, exception_handler) + register(app, database) + register(app, logger) + register(app, router) + register(app, scheduler) + register(app, common) + + # Call the custom_openapi function to change the OpenAPI version + customize_openapi_security(app) + return app + + +# This function overrides the OpenAPI schema version to 3.0.0 +def customize_openapi_security(app: FastAPI) -> None: + + def custom_openapi(): + if app.openapi_schema: + return app.openapi_schema + + # Generate OpenAPI schema + openapi_schema = get_openapi( + title="FreeLeaps API", + version="3.1.0", + description="FreeLeaps API Documentation", + routes=app.routes, + ) + + # Ensure the components section exists in the OpenAPI schema + if "components" not in openapi_schema: + openapi_schema["components"] = {} + + # Add security scheme to components + openapi_schema["components"]["securitySchemes"] = { + "bearerAuth": { + "type": "http", + "scheme": "bearer", + "bearerFormat": "JWT" + } + } + + # Add security requirement globally + openapi_schema["security"] = [{"bearerAuth": []}] + + app.openapi_schema = openapi_schema + return app.openapi_schema + + app.openapi = custom_openapi + + +def register(app, provider): + logging.info(provider.__name__ + " registering") + provider.register(app) + + +def boot(app, provider): + logging.info(provider.__name__ + " booting") + provider.boot(app) diff --git a/app/central_storage/webapi/bootstrap/freeleaps_app.py b/app/central_storage/webapi/bootstrap/freeleaps_app.py new file mode 100644 index 0000000..496633a --- /dev/null +++ b/app/central_storage/webapi/bootstrap/freeleaps_app.py @@ -0,0 +1,6 @@ +from fastapi import FastAPI + + +class FreeleapsApp(FastAPI): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) diff --git a/app/central_storage/webapi/config/site_settings.py b/app/central_storage/webapi/config/site_settings.py new file mode 100644 index 0000000..a72e6d0 --- /dev/null +++ b/app/central_storage/webapi/config/site_settings.py @@ -0,0 +1,25 @@ +import os + +from pydantic_settings import BaseSettings + + +class SiteSettings(BaseSettings): + NAME: str = "appname" + DEBUG: bool = True + + ENV: str = "dev" + + SERVER_HOST: str = "0.0.0.0" + SERVER_PORT: int = 8005 + + URL: str = "http://localhost" + TIME_ZONE: str = "UTC" + + BASE_PATH: str = os.path.dirname(os.path.dirname((os.path.abspath(__file__)))) + + class Config: + env_file = ".devbase-webapi.env" + env_file_encoding = "utf-8" + + +site_settings = SiteSettings() diff --git a/app/central_storage/webapi/main.py b/app/central_storage/webapi/main.py new file mode 100755 index 0000000..df0c604 --- /dev/null +++ b/app/central_storage/webapi/main.py @@ -0,0 +1,30 @@ +from webapi.bootstrap.application import create_app +from webapi.config.site_settings import site_settings +from fastapi.responses import RedirectResponse +from fastapi.middleware.cors import CORSMiddleware +from strawberry.fastapi import GraphQLRouter +from strawberry.fastapi.handlers import GraphQLTransportWSHandler, GraphQLWSHandler +import uvicorn +from typing import Any + + +app = create_app() + +@app.get("/", status_code=301) +async def root(): + """ + TODO: redirect client to /doc# + """ + return RedirectResponse("docs") + + +if __name__ == "__main__": + uvicorn.run(app="main:app", host=site_settings.SERVER_HOST, port=site_settings.SERVER_PORT) + +def get_context() -> Any: + # Define your context function. This is where you can set up authentication, database connections, etc. + return {} + +def get_root_value() -> Any: + # Define your root value function. This can be used to customize the root value for GraphQL operations. + return {} \ No newline at end of file diff --git a/app/central_storage/webapi/providers/common.py b/app/central_storage/webapi/providers/common.py new file mode 100644 index 0000000..1dd849f --- /dev/null +++ b/app/central_storage/webapi/providers/common.py @@ -0,0 +1,31 @@ +from fastapi.middleware.cors import CORSMiddleware +from webapi.config.site_settings import site_settings + + +def register(app): + app.debug = site_settings.DEBUG + app.title = site_settings.NAME + + add_global_middleware(app) + + # This hook ensures that a connection is opened to handle any queries + # generated by the request. + @app.on_event("startup") + def startup(): + pass + + # This hook ensures that the connection is closed when we've finished + # processing the request. + @app.on_event("shutdown") + def shutdown(): + pass + + +def add_global_middleware(app): + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) diff --git a/app/central_storage/webapi/providers/database.py b/app/central_storage/webapi/providers/database.py new file mode 100644 index 0000000..59ed3ab --- /dev/null +++ b/app/central_storage/webapi/providers/database.py @@ -0,0 +1,17 @@ +from webapi.config.site_settings import site_settings +from beanie import init_beanie +from motor.motor_asyncio import AsyncIOMotorClient + + +def register(app): + app.debug = site_settings.DEBUG + app.title = site_settings.NAME + + @app.on_event("startup") + async def start_database(): + await initiate_database() + + +async def initiate_database(): + #init your database here + pass diff --git a/app/central_storage/webapi/providers/exception_handler.py b/app/central_storage/webapi/providers/exception_handler.py new file mode 100644 index 0000000..21117a5 --- /dev/null +++ b/app/central_storage/webapi/providers/exception_handler.py @@ -0,0 +1,39 @@ +from fastapi import FastAPI, HTTPException +from fastapi.exceptions import RequestValidationError +from starlette.requests import Request +from starlette.responses import JSONResponse +from starlette.status import ( + HTTP_400_BAD_REQUEST, + HTTP_401_UNAUTHORIZED, + HTTP_403_FORBIDDEN, + HTTP_404_NOT_FOUND, + HTTP_422_UNPROCESSABLE_ENTITY, + HTTP_500_INTERNAL_SERVER_ERROR, +) + + +async def custom_http_exception_handler(request: Request, exc: HTTPException): + return JSONResponse( + status_code=exc.status_code, + content={"error": exc.detail}, + ) + + + +async def validation_exception_handler(request: Request, exc: RequestValidationError): + return JSONResponse( + status_code=HTTP_400_BAD_REQUEST, + content={"error": str(exc)}, + ) + +async def exception_handler(request: Request, exc: Exception): + return JSONResponse( + status_code=HTTP_500_INTERNAL_SERVER_ERROR, + content={"error": str(exc)}, + ) + + +def register(app: FastAPI): + app.add_exception_handler(HTTPException, custom_http_exception_handler) + app.add_exception_handler(RequestValidationError, validation_exception_handler) + app.add_exception_handler(Exception, exception_handler) diff --git a/app/central_storage/webapi/providers/logger.py b/app/central_storage/webapi/providers/logger.py new file mode 100644 index 0000000..4a3f1e7 --- /dev/null +++ b/app/central_storage/webapi/providers/logger.py @@ -0,0 +1,60 @@ +import logging +import sys +from loguru import logger +from common.config.log_settings import log_settings + + +def register(app=None): + level = log_settings.LOG_LEVEL + file_path = log_settings.LOG_PATH + retention = log_settings.LOG_RETENTION + rotation = log_settings.LOG_ROTATION + + # intercept everything at the root logger + logging.root.handlers = [InterceptHandler()] + logging.root.setLevel(level) + + # remove every other logger's handlers + # and propagate to root logger + for name in logging.root.manager.loggerDict.keys(): + logging.getLogger(name).handlers = [] + logging.getLogger(name).propagate = True + + # configure loguru + logger.add( + sink=sys.stdout + ) + logger.add( + sink=file_path, + level=level, + retention=retention, + rotation=rotation + ) + + logger.disable("pika.adapters") + logger.disable("pika.connection") + logger.disable("pika.channel") + logger.disable("pika.callback") + logger.disable("pika.frame") + logger.disable("pika.spec") + logger.disable("aiormq.connection") + logger.disable("urllib3.connectionpool") + + +class InterceptHandler(logging.Handler): + def emit(self, record): + # Get corresponding Loguru level if it exists + try: + level = logger.level(record.levelname).name + except ValueError: + level = record.levelno + + # Find caller from where originated the logged message + frame, depth = logging.currentframe(), 2 + while frame.f_code.co_filename == logging.__file__: + frame = frame.f_back + depth += 1 + + logger.opt(depth=depth, exception=record.exc_info).log( + level, record.getMessage() + ) diff --git a/app/central_storage/webapi/providers/router.py b/app/central_storage/webapi/providers/router.py new file mode 100644 index 0000000..3ad11ae --- /dev/null +++ b/app/central_storage/webapi/providers/router.py @@ -0,0 +1,34 @@ +from webapi.routes import api_router + +from starlette import routing + + +def register(app): + app.include_router( + api_router, + prefix="/api", + tags=["api"], + dependencies=[], + responses={404: {"description": "no page found"}}, + ) + + if app.debug: + for route in app.routes: + if not isinstance(route, routing.WebSocketRoute): + print( + { + "path": route.path, + "endpoint": route.endpoint, + "name": route.name, + "methods": route.methods, + } + ) + else: + print( + { + "path": route.path, + "endpoint": route.endpoint, + "name": route.name, + "type": "web socket route", + } + ) diff --git a/app/central_storage/webapi/providers/scheduler.py b/app/central_storage/webapi/providers/scheduler.py new file mode 100644 index 0000000..7ea8d6c --- /dev/null +++ b/app/central_storage/webapi/providers/scheduler.py @@ -0,0 +1,8 @@ +import asyncio + + +def register(app): + @app.on_event("startup") + async def start_scheduler(): + #create your scheduler here + pass diff --git a/app/central_storage/webapi/routes/__init__.py b/app/central_storage/webapi/routes/__init__.py new file mode 100644 index 0000000..3237813 --- /dev/null +++ b/app/central_storage/webapi/routes/__init__.py @@ -0,0 +1,5 @@ +from fastapi import APIRouter + +api_router = APIRouter() + +websocket_router = APIRouter() diff --git a/app/central_storage/webapi/routes/api.py b/app/central_storage/webapi/routes/api.py new file mode 100644 index 0000000..6545552 --- /dev/null +++ b/app/central_storage/webapi/routes/api.py @@ -0,0 +1,15 @@ +from fastapi.routing import APIRoute +from starlette import routing + + +def post_process_router(app) -> None: + """ + Simplify operation IDs so that generated API clients have simpler function + names. + + Should be called only after all routes have been added. + """ + for route in app.routes: + if isinstance(route, APIRoute): + if hasattr(route, "operation_id"): + route.operation_id = route.name # in this case, 'read_items' diff --git a/app/central_storage/webapi/routes/get_document_by_id.py b/app/central_storage/webapi/routes/get_document_by_id.py new file mode 100644 index 0000000..1aaad08 --- /dev/null +++ b/app/central_storage/webapi/routes/get_document_by_id.py @@ -0,0 +1,34 @@ +from fastapi import APIRouter, Security, HTTPException +from fastapi.encoders import jsonable_encoder +from fastapi.responses import JSONResponse +from fastapi_jwt import JwtAuthorizationCredentials +from backend.infra.authentication.auth import access_security +from app.central_storage.backend.application.document_app import DocumentHub + +router = APIRouter() + + +# Web API +# Fetch document by ID +@router.get( + "/get-document-by-id/{document_id}", + operation_id="get-document-by-id", + summary="Fetch a document by its ID", + description="Retrieve a specific document by its document ID and return file name and download URL", + response_description="The document details including file name and download URL", +) +async def get_document_by_id( + document_id: str, + credentials: JwtAuthorizationCredentials = Security(access_security), +): + user_id = credentials["id"] + + # Fetch the document using DocumentHub + document = await DocumentHub(user_id).get_document_by_id(document_id) + + # If document is not found, raise 404 error + if not document: + raise HTTPException(status_code=404, detail="Document not found") + + # Return the document details + return JSONResponse(content=jsonable_encoder({"document": document})) diff --git a/docker-compose.yaml b/docker-compose.yaml index ec341b2..8bf33ab 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,32 +1,34 @@ -version: '3.8' - services: - rabbitmq: - image: rabbitmq:3-management - ports: - - "5672:5672" # RabbitMQ communication port - - "15672:15672" # RabbitMQ management port - networks: - - rabbitmq_network - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:15672"] - interval: 30s - retries: 5 - start_period: 10s - timeout: 10s + # rabbitmq: + # image: rabbitmq:3-management + # ports: + # - "5672:5672" # RabbitMQ communication port + # - "15672:15672" # RabbitMQ management port + # networks: + # - freeleaps_service_hub_network + # healthcheck: + # test: [ "CMD", "curl", "-f", "http://localhost:15672" ] + # interval: 30s + # retries: 5 + # start_period: 10s + # timeout: 10s - python_service: + central_storage: build: - context: . + context: app/central_storage dockerfile: Dockerfile - depends_on: - rabbitmq: - condition: service_healthy + restart: always environment: - RABBITMQ_HOST: rabbitmq + - AZURE_STORAGE_DOCUMENT_API_KEY=${AZURE_STORAGE_DOCUMENT_API_KEY} + ports: + - "8005:8005" # Map the central_storage service port networks: - - rabbitmq_network + - freeleaps_service_hub_network + env_file: + - sites/central_storage/.env + volumes: + - .:/app # Mount the current directory to /app in the container networks: - rabbitmq_network: - driver: bridge \ No newline at end of file + freeleaps_service_hub_network: + driver: bridge diff --git a/infra/config/app_settings.py b/infra/config/app_settings.py new file mode 100644 index 0000000..70bf94e --- /dev/null +++ b/infra/config/app_settings.py @@ -0,0 +1,15 @@ +import os +from pydantic_settings import BaseSettings + +class AppSettings(): + NAME: str = "myapp" + + class Config: + env_file = ".myapp.env" + env_file_encoding = "utf-8" + + APPLICATION_ACTIVITY_LOG: str = "myapp-application-activity" + USER_ACTIVITY_LOG: str = "myapp-user-activity" + BUSINESS_METRIC_LOG: str = "myapp-business-metrics" + +app_settings = AppSettings() diff --git a/infra/config/log_settings.py b/infra/config/log_settings.py new file mode 100644 index 0000000..856281d --- /dev/null +++ b/infra/config/log_settings.py @@ -0,0 +1,17 @@ + + +class LogSettings(): + LOG_LEVEL: str = "DEBUG" + LOG_PATH_BASE: str = ( + "./logs" + ) + LOG_PATH: str = LOG_PATH_BASE + '/' + "app" + '.log' + LOG_RETENTION: str = "14 days" + LOG_ROTATION: str = "00:00" # mid night + + class Config: + env_file = ".log.env" + env_file_encoding = "utf-8" + + +log_settings = LogSettings() diff --git a/infra/exception/__init__.py b/infra/exception/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/infra/exception/exceptions.py b/infra/exception/exceptions.py new file mode 100644 index 0000000..2c489c3 --- /dev/null +++ b/infra/exception/exceptions.py @@ -0,0 +1,23 @@ +class DoesNotExistError(Exception): + def __init__(self, message: str = "Does Not Exist"): + self.message = message + + +class AuthenticationError(Exception): + def __init__(self, message: str = "Unauthorized"): + self.message = message + + +class AuthorizationError(Exception): + def __init__(self, message: str = "Forbidden"): + self.message = message + + +class InvalidOperationError(Exception): + def __init__(self, message: str = "Invalid Operation"): + self.message = message + + +class InvalidDataError(Exception): + def __init__(self, message: str = "Invalid Data"): + self.message = message diff --git a/infra/log/__init__.py b/infra/log/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/infra/log/application_logger.py b/infra/log/application_logger.py new file mode 100644 index 0000000..c1222bb --- /dev/null +++ b/infra/log/application_logger.py @@ -0,0 +1,14 @@ +from .base_logger import LoggerBase +from common.config.app_settings import app_settings +import json + + +class ApplicationLogger(LoggerBase): + def __init__(self, application_activities: dict[str, any] = {}) -> None: + extra_fileds = {} + if application_activities: + extra_fileds.update(application_activities) + super().__init__( + logger_name=app_settings.APPLICATION_ACTIVITY_LOG, + extra_fileds=extra_fileds, + ) diff --git a/infra/log/base_logger.py b/infra/log/base_logger.py new file mode 100644 index 0000000..122dfdb --- /dev/null +++ b/infra/log/base_logger.py @@ -0,0 +1,139 @@ +from loguru import logger as guru_logger +from common.config.log_settings import log_settings +from typing import List +import socket +import json +import logging +import threading + + +class LoggerBase: + binded_loggers = {} + logger_lock = threading.Lock() + + def __init__( + self, logger_name: str, extra_fileds: dict[str, any] + ) -> None: + self.__logger_name = logger_name + self.extra_fileds = extra_fileds + with LoggerBase.logger_lock: + if self.__logger_name in LoggerBase.binded_loggers: + self.logger = LoggerBase.binded_loggers[self.__logger_name] + return + + log_filename = ( + log_settings.LOG_PATH_BASE + "/" + self.__logger_name + ".log" + ) + log_retention = log_settings.LOG_RETENTION + log_rotation = log_settings.LOG_ROTATION + log_level = "INFO" + log_message_format = "{message}" + + guru_logger.add( + sink=log_filename, + level=log_level, + retention=log_retention, + rotation=log_rotation, + format=log_message_format, + serialize=True, + filter=lambda record: "extra" in record + and "topic" in record["extra"] + and record["extra"]["topic"] == self.__logger_name, + ) + host_name = socket.gethostname() + host_ip = socket.gethostbyname(host_name) + self.logger = guru_logger.bind( + topic=self.__logger_name, + host_ip=host_ip, + host_name=host_name, + ) + with LoggerBase.logger_lock: + LoggerBase.binded_loggers[self.__logger_name] = self.logger + + async def log_event( + self, + sender_id: str, + receiver_id: str, + subject: str, + event: str, + properties: dict[str, any], + text: str = "" + ) -> None: + local_logger = self.logger.bind( + sender_id=sender_id, + receiver_id=receiver_id, + subject=subject, + event=event, + properties=properties + ) + local_logger.info(text) + + async def log_exception( + self, + sender_id: str, + receiver_id: str, + subject: str, + exception: Exception, + text: str = "", + properties: dict[str, any] = None, + ) -> None: + local_logger = self.logger.bind( + sender_id=sender_id, + receiver_id=receiver_id, + subject=subject, + event="exception", + properties=properties, + exception=exception + ) + local_logger.exception(text) + + async def log_info( + self, + sender_id: str, + receiver_id: str, + subject: str, + text: str = "", + properties: dict[str, any] = None, + ) -> None: + local_logger = self.logger.bind( + sender_id=sender_id, + receiver_id=receiver_id, + subject=subject, + event="information", + properties=properties, + ) + local_logger.info(text) + + async def log_warning( + self, + sender_id: str, + receiver_id: str, + subject: str, + text: str = "", + properties: dict[str, any] = None, + ) -> None: + local_logger = self.logger.bind( + sender_id=sender_id, + receiver_id=receiver_id, + subject=subject, + event="warning", + properties=properties, + ) + local_logger.warning(text) + + async def log_error( + self, + sender_id: str, + receiver_id: str, + subject: str, + text: str = "", + properties: dict[str, any] = None, + ) -> None: + local_logger = self.logger.bind( + sender_id=sender_id, + receiver_id=receiver_id, + subject=subject, + event="error", + properties=properties, + ) + local_logger.error(text) diff --git a/infra/log/business_metric_logger.py b/infra/log/business_metric_logger.py new file mode 100644 index 0000000..95383ab --- /dev/null +++ b/infra/log/business_metric_logger.py @@ -0,0 +1,25 @@ +from .base_logger import LoggerBase +from common.config.app_settings import app_settings +import json + + +class BusinessMetricLogger(LoggerBase): + def __init__(self, business_metrics: dict[str, any] = {}) -> None: + extra_fileds = {} + if business_metrics: + extra_fileds.update(business_metrics) + super().__init__( + logger_name=app_settings.BUSINESS_METRIC_LOG, + extra_fileds=extra_fileds, + ) + + + async def log_metrics(self, business_metrics: dict[str, any] = {}) -> None: + return await super().log_event( + sender_id="business_metric_manager", + receiver_id="business_metric_logger", + subject="metrics", + event="logging", + properties=business_metrics, + text="business metric logged" + ) diff --git a/infra/log/function_logger.py b/infra/log/function_logger.py new file mode 100644 index 0000000..4388a5e --- /dev/null +++ b/infra/log/function_logger.py @@ -0,0 +1,50 @@ +from .application_logger import ApplicationLogger + + +class FunctionLogger(ApplicationLogger): + def __init__(self, sender_id: str, receiver_id:str) -> None: + super().__init__() + self.event_sender_id = sender_id + self.event_receiver_id = receiver_id + self.event_subject = "function" + + async def log_enter(self, function: str, file: str): + return await super().log_event( + sender_id=self.event_sender_id, + receiver_id=self.event_receiver_id, + subject=self.event_subject, + event="enter", + properties={ + "function": function, + "file": file, + }, + text="Enter:{} of {}".format(function, file) + ) + + async def log_exit(self, function: str, file: str, excution_time_in_ns: int): + return await super().log_event( + sender_id=self.event_sender_id, + receiver_id=self.event_receiver_id, + subject=self.event_subject, + event="exit", + properties={ + "function": function, + "file": file, + "excution_time_in_ns": excution_time_in_ns + }, + text="Exit:{} of {}".format(function, file) + ) + + async def log_exception(self, exception: Exception, function: str, file: str, excution_time_in_ns: int) -> None: + return await super().log_exception( + sender_id=self.event_sender_id, + receiver_id=self.event_receiver_id, + subject=self.event_subject, + exception=exception, + text="Exception:{} of {}".format(function, file), + properties={ + "function": function, + "file": file, + "excution_time_in_ns": excution_time_in_ns + }, + ) diff --git a/infra/log/log_utils.py b/infra/log/log_utils.py new file mode 100644 index 0000000..579dee8 --- /dev/null +++ b/infra/log/log_utils.py @@ -0,0 +1,25 @@ +import os +from .function_logger import FunctionLogger +import time +import functools + + +def log_entry_exit_async(func): + @functools.wraps(func) + async def wrapper(*args, **kwargs): + file_path = os.path.relpath(func.__code__.co_filename) + function_logger = FunctionLogger(sender_id="log_entry_exit_async", receiver_id="function_logger") + start_time = time.process_time_ns() + try: + await function_logger.log_enter(func.__name__, file_path) + result = await func(*args, **kwargs) + await function_logger.log_exit(func.__name__, file_path, time.process_time_ns() - start_time) + return result + except Exception as exception: + await function_logger.log_exception( + exception=exception, + function=func.__name__, + file=file_path, + excution_time_in_ns=time.process_time_ns() - start_time) + raise + return wrapper diff --git a/infra/log/module_logger.py b/infra/log/module_logger.py new file mode 100644 index 0000000..3426b0b --- /dev/null +++ b/infra/log/module_logger.py @@ -0,0 +1,46 @@ +from .application_logger import ApplicationLogger + + +class ModuleLogger(ApplicationLogger): + def __init__(self, sender_id: str) -> None: + super().__init__() + self.event_sender_id = sender_id + self.event_receiver_id = "ModuleLogger" + self.event_subject = "module" + + async def log_exception(self, exception: Exception, text: str = "Exception", properties: dict[str, any] = None) -> None: + return await super().log_exception( + sender_id=self.event_sender_id, + receiver_id=self.event_receiver_id, + subject=self.event_subject, + exception=exception, + text=text, + properties=properties, + ) + + async def log_info(self, info: str, properties: dict[str, any] = None) -> None: + return await super().log_info( + sender_id=self.event_sender_id, + receiver_id=self.event_receiver_id, + subject=self.event_subject, + text=info, + properties=properties, + ) + + async def log_warning(self, warning: str, properties: dict[str, any] = None) -> None: + return await super().log_warning( + sender_id=self.event_sender_id, + receiver_id=self.event_receiver_id, + subject=self.event_subject, + text=warning, + properties=properties, + ) + + async def log_error(self, error: str, properties: dict[str, any] = None) -> None: + return await super().log_error( + sender_id=self.event_sender_id, + receiver_id=self.event_receiver_id, + subject=self.event_subject, + text=error, + properties=properties, + ) diff --git a/infra/log/user_logger.py b/infra/log/user_logger.py new file mode 100644 index 0000000..d931975 --- /dev/null +++ b/infra/log/user_logger.py @@ -0,0 +1,14 @@ +from .base_logger import LoggerBase +from common.config.app_settings import app_settings + +import json + + +class UserLogger(LoggerBase): + def __init__(self, user_activities: dict[str, any] = {}) -> None: + extra_fileds = {} + if user_activities: + extra_fileds.update(user_activities) + super().__init__( + logger_name=app_settings.USER_ACTIVITY_LOG, extra_fileds=extra_fileds + ) diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index cde8834..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -pika==1.3.2 \ No newline at end of file diff --git a/sites/central_storage/.env b/sites/central_storage/.env new file mode 100644 index 0000000..d8a6620 --- /dev/null +++ b/sites/central_storage/.env @@ -0,0 +1 @@ +export AZURE_STORAGE_DOCUMENT_API_KEY=xbiFtFeQ6v5dozgVM99fZ9huUomL7QcLu6s0y8zYHtIXZ8XdneKDMcg4liQr/9oNlVoRFcZhWjLY+ASt9cjICQ== \ No newline at end of file