From c33700fa661f3da4978b226715e1d8926f7cc478 Mon Sep 17 00:00:00 2001 From: Mike Liao Date: Thu, 31 Oct 2024 06:42:21 -0700 Subject: [PATCH] fixed bugs and tested major functions --- .../common/config/log_settings.py | 2 +- apps/central_storage/.env | 3 +- .../common/config/log_settings.py | 2 +- .../central_storage/webapi/routes/__init__.py | 2 +- .../webapi/routes/upload_document.py | 41 ------------------- .../webapi/routes/upload_file.py | 2 - apps/content/backend/content/__init__.py | 4 +- .../backend/content/content_service.py | 11 +++-- .../content/content_sharepoint_manager.py | 31 ++++---------- .../backend/document/document_manager.py | 12 +++--- apps/content/common/config/app_settings.py | 5 ++- apps/content/common/config/log_settings.py | 2 +- apps/content/database/mongo/mongo_driver.py | 6 +-- apps/content/webapi/providers/database.py | 4 +- .../common/config/log_settings.py | 2 +- sites/content/deploy/alpha/.env | 1 + sites/content/deploy/common/.env | 1 - sites/content/deploy/dev/.env | 1 + sites/content/deploy/local/.env | 1 + sites/content/deploy/prod/.env | 1 + 20 files changed, 44 insertions(+), 90 deletions(-) delete mode 100644 apps/central_storage/webapi/routes/upload_document.py diff --git a/apps/authentication/common/config/log_settings.py b/apps/authentication/common/config/log_settings.py index 6d86462..48b97a4 100644 --- a/apps/authentication/common/config/log_settings.py +++ b/apps/authentication/common/config/log_settings.py @@ -2,7 +2,7 @@ from pydantic_settings import BaseSettings from .app_settings import app_settings -class LogSettings(): +class LogSettings(BaseSettings): LOG_LEVEL: str = "DEBUG" LOG_BASE_PATH: str = app_settings.LOG_BASE_PATH LOG_PATH: str = LOG_BASE_PATH + '/' + app_settings.BACKEND_LOG_FILE_NAME + '.log' diff --git a/apps/central_storage/.env b/apps/central_storage/.env index 8d3503c..bf6460b 100644 --- a/apps/central_storage/.env +++ b/apps/central_storage/.env @@ -10,7 +10,7 @@ export APPLICATION_ACTIVITY_LOG=$APP_NAME-activity export MONGODB_NAME=freeleaps2 export MONGODB_PORT=27017 GIT_REPO_ROOT=/mnt/freeleaps/freeleaps-service-hub -CODEBASE_ROOT=/mnt/freeleaps/freeleaps-service-hub/app/central_storage +CODEBASE_ROOT=/mnt/freeleaps/freeleaps-service-hub/apps/central_storage SITE_DEPLOY_FOLDER=/mnt/freeleaps/freeleaps-service-hub/sites/central_storage/deploy #!/bin/bash export VENV_DIR=venv_t @@ -21,4 +21,5 @@ export DOCKER_BACKEND_HOME=$DOCKER_APP_HOME/$APP_NAME export DOCKER_BACKEND_LOG_HOME=$DOCKER_BACKEND_HOME/log export MONGODB_URI=mongodb://localhost:27017/ export FREELEAPS_ENV=local +export LOG_BASE_PATH=${CODEBASE_ROOT}/log diff --git a/apps/central_storage/common/config/log_settings.py b/apps/central_storage/common/config/log_settings.py index 6d86462..48b97a4 100644 --- a/apps/central_storage/common/config/log_settings.py +++ b/apps/central_storage/common/config/log_settings.py @@ -2,7 +2,7 @@ from pydantic_settings import BaseSettings from .app_settings import app_settings -class LogSettings(): +class LogSettings(BaseSettings): LOG_LEVEL: str = "DEBUG" LOG_BASE_PATH: str = app_settings.LOG_BASE_PATH LOG_PATH: str = LOG_BASE_PATH + '/' + app_settings.BACKEND_LOG_FILE_NAME + '.log' diff --git a/apps/central_storage/webapi/routes/__init__.py b/apps/central_storage/webapi/routes/__init__.py index 9f744dd..66c5947 100644 --- a/apps/central_storage/webapi/routes/__init__.py +++ b/apps/central_storage/webapi/routes/__init__.py @@ -1,6 +1,6 @@ from fastapi import APIRouter from .retrieve_document_info import router as ri_router -from .upload_document import router as ud_router +from .upload_file import router as ud_router from .read_document_as_http_media import router as rd_router api_router = APIRouter() diff --git a/apps/central_storage/webapi/routes/upload_document.py b/apps/central_storage/webapi/routes/upload_document.py deleted file mode 100644 index a4d20f1..0000000 --- a/apps/central_storage/webapi/routes/upload_document.py +++ /dev/null @@ -1,41 +0,0 @@ -from fastapi import APIRouter, UploadFile, File, Form, HTTPException -from fastapi import APIRouter, Depends -from starlette.status import HTTP_401_UNAUTHORIZED -from fastapi.encoders import jsonable_encoder -from fastapi.responses import JSONResponse -from backend.application.document_hub import DocumentHub -from pydantic import BaseModel - -router = APIRouter() - - -class Item(BaseModel): - associated_with: str - name: str - blob: bytes - - -@router.post( - "/upload-document", - summary="upload a document with a given associated_with id, document name and document data.", - description="upload a document. If success, returning the document id", -) -async def upload_document(item: Item): - - document_hub = DocumentHub() - # File processing - try: - document_id = await document_hub.upload_document( - item.associated_with, item.name, item.blob - ) - - if document_id: - result = {"document_id": str(document_id)} - return JSONResponse(content=jsonable_encoder(result)) - else: - return JSONResponse( - status_code=500, content={"error": "Document upload failed"} - ) - - except Exception as e: - return JSONResponse(status_code=500, content={"error": "Internal server error"}) diff --git a/apps/central_storage/webapi/routes/upload_file.py b/apps/central_storage/webapi/routes/upload_file.py index 243297f..830272d 100644 --- a/apps/central_storage/webapi/routes/upload_file.py +++ b/apps/central_storage/webapi/routes/upload_file.py @@ -1,6 +1,5 @@ from fastapi import APIRouter, UploadFile, File, Form, HTTPException from fastapi import APIRouter, Depends -from infra.token.token_manager import TokenManager from starlette.status import HTTP_401_UNAUTHORIZED from fastapi.encoders import jsonable_encoder from fastapi.responses import JSONResponse @@ -8,7 +7,6 @@ from backend.application.document_hub import DocumentHub router = APIRouter() -token_manager = TokenManager() @router.post( diff --git a/apps/content/backend/content/__init__.py b/apps/content/backend/content/__init__.py index 7c1a88b..770644c 100644 --- a/apps/content/backend/content/__init__.py +++ b/apps/content/backend/content/__init__.py @@ -1,4 +1,4 @@ -from backend.content.models import ContentDirectory,ContentFolderDoc +from backend.content.models import ContentFolderDoc content_models = [] -content_models.extend([ContentDirectory, ContentFolderDoc]) +content_models.extend([ContentFolderDoc]) diff --git a/apps/content/backend/content/content_service.py b/apps/content/backend/content/content_service.py index b00bf8f..e67bc9c 100644 --- a/apps/content/backend/content/content_service.py +++ b/apps/content/backend/content/content_service.py @@ -20,12 +20,17 @@ class ContentService: ContentFolderDoc.folder_name == folder_name, ContentFolderDoc.region == region ) + if folder is None: - folder = await ContentSharePointManager().retrieve_directories_for_folder(folder_name=folder_name, region=region) + await ContentSharePointManager().retrieve_directories_for_folder(folder_name=folder_name, region=region) + folder = await ContentFolderDoc.find_one( + ContentFolderDoc.folder_name == folder_name, + ContentFolderDoc.region == region + ) + return folder.content_directories if folder else None async def retrieve_content_as_media_data(self, document_id: str) -> Optional[str]: document_manager = DocumentManager() - await document_manager.load_document(document_id) - return await document_manager.read_document_file_as_http_media_data() + return await document_manager.retrieve_document_as_http_media(document_id) diff --git a/apps/content/backend/content/content_sharepoint_manager.py b/apps/content/backend/content/content_sharepoint_manager.py index a4b3dc6..05183a1 100644 --- a/apps/content/backend/content/content_sharepoint_manager.py +++ b/apps/content/backend/content/content_sharepoint_manager.py @@ -64,18 +64,10 @@ class ContentSharePointManager: ): cover_file_content = self.sharepoint_client.get_file_content(sp_file['id']) cover_document_manager = DocumentManager() - await cover_document_manager.new_document( - file_name=sp_file['name'].lower(), - media_type=ContentMediaType.PNG, - data_format=ContentDataFormat.RAW, - created_by=self.__generate_created__by__(folder_name=folder_name) - ) - content_directory.cover_document_id = ( - cover_document_manager.get_document_id() - ) - - await cover_document_manager.save_document_file( - cover_file_content + file_name=sp_file['name'].lower() + created_by=self.__generate_created__by__(folder_name=folder_name) + content_directory.cover_document_id = await cover_document_manager.save_document_file( + created_by, file_name, cover_file_content ) elif ( @@ -109,17 +101,10 @@ class ContentSharePointManager: ): content_file_content = self.sharepoint_client.get_file_content(sp_file['id']) content_document_manager = DocumentManager() - await content_document_manager.new_document( - file_name=sp_file['name'], - media_type=ContentMediaType.PDF, - data_format=ContentDataFormat.RAW, - created_by=self.__generate_created__by__(folder_name=folder_name) - ) - content_directory.content_document_id = ( - content_document_manager.get_document_id() - ) - await content_document_manager.save_document_file( - content_file_content + file_name=sp_file['name'] + created_by=self.__generate_created__by__(folder_name=folder_name) + content_directory.content_document_id = await content_document_manager.save_document_file( + created_by, file_name, content_file_content ) folder.content_directories.append(content_directory) diff --git a/apps/content/backend/document/document_manager.py b/apps/content/backend/document/document_manager.py index c0e62f9..3a46641 100644 --- a/apps/content/backend/document/document_manager.py +++ b/apps/content/backend/document/document_manager.py @@ -3,7 +3,7 @@ import httpx class DocumentManager: def __init__(self): - self.storage_service_api_base = app_settings.CENTRAL_STORAGE_WEBAPI_URL_BASE + self.storage_service_api_base = app_settings.CENTRAL_STORAGE_WEBAPI_URL_BASE.rstrip('/') + '/' async def retrieve_document_info(self, document_id:str): api_url = self.storage_service_api_base + "retrieve_document_info/" + document_id @@ -18,13 +18,13 @@ class DocumentManager: return response.json()['media'] async def save_document_file(self,associated_with:str, name:str,blob:bytes)->str: - api_url = self.storage_service_api_base + "upload_document/" + api_url = self.storage_service_api_base + "upload-file" + files = {'file':(name,blob)} async with httpx.AsyncClient() as client: response = await client.post(api_url, data={ - 'associated_with':associated_with, - 'name':name, - 'blob':blob - } + 'associated_with':associated_with + }, + files=files ) return response.json()['document_id'] diff --git a/apps/content/common/config/app_settings.py b/apps/content/common/config/app_settings.py index b05d89d..8a84051 100644 --- a/apps/content/common/config/app_settings.py +++ b/apps/content/common/config/app_settings.py @@ -1,7 +1,7 @@ import os from pydantic_settings import BaseSettings -class AppSettings(): +class AppSettings(BaseSettings): NAME: str = "content" APP_NAME:str = NAME @@ -9,6 +9,9 @@ class AppSettings(): CENTRAL_STORAGE_WEBAPI_URL_BASE:str ="" + MONGODB_NAME:str ="" + MONGODB_URI:str ="" + LOG_BASE_PATH : str = "./log" BACKEND_LOG_FILE_NAME: str = APP_NAME APPLICATION_ACTIVITY_LOG: str = APP_NAME + "-application-activity" diff --git a/apps/content/common/config/log_settings.py b/apps/content/common/config/log_settings.py index 6d86462..48b97a4 100644 --- a/apps/content/common/config/log_settings.py +++ b/apps/content/common/config/log_settings.py @@ -2,7 +2,7 @@ from pydantic_settings import BaseSettings from .app_settings import app_settings -class LogSettings(): +class LogSettings(BaseSettings): LOG_LEVEL: str = "DEBUG" LOG_BASE_PATH: str = app_settings.LOG_BASE_PATH LOG_PATH: str = LOG_BASE_PATH + '/' + app_settings.BACKEND_LOG_FILE_NAME + '.log' diff --git a/apps/content/database/mongo/mongo_driver.py b/apps/content/database/mongo/mongo_driver.py index b981d56..92ad7c3 100755 --- a/apps/content/database/mongo/mongo_driver.py +++ b/apps/content/database/mongo/mongo_driver.py @@ -1,4 +1,4 @@ -from webapi.config.site_settings import site_settings +from common.config.app_settings import app_settings from beanie import init_beanie from motor.motor_asyncio import AsyncIOMotorClient from database.mongo.models import mongo_models @@ -8,7 +8,7 @@ class MongoDriver: pass async def initiate_database(self): - client = AsyncIOMotorClient(site_settings.MONGODB_URI, serverSelectionTimeoutMS=60000) + client = AsyncIOMotorClient(app_settings.MONGODB_URI, serverSelectionTimeoutMS=60000) await init_beanie( - database=client[site_settings.MONGODB_NAME], document_models=mongo_models + database=client[app_settings.MONGODB_NAME], document_models=mongo_models ) diff --git a/apps/content/webapi/providers/database.py b/apps/content/webapi/providers/database.py index 59ed3ab..d2e283c 100644 --- a/apps/content/webapi/providers/database.py +++ b/apps/content/webapi/providers/database.py @@ -1,7 +1,7 @@ from webapi.config.site_settings import site_settings from beanie import init_beanie from motor.motor_asyncio import AsyncIOMotorClient - +from database.mongo.mongo_driver import MongoDriver def register(app): app.debug = site_settings.DEBUG @@ -14,4 +14,4 @@ def register(app): async def initiate_database(): #init your database here - pass + await MongoDriver().initiate_database() diff --git a/apps/notification/common/config/log_settings.py b/apps/notification/common/config/log_settings.py index 6d86462..48b97a4 100644 --- a/apps/notification/common/config/log_settings.py +++ b/apps/notification/common/config/log_settings.py @@ -2,7 +2,7 @@ from pydantic_settings import BaseSettings from .app_settings import app_settings -class LogSettings(): +class LogSettings(BaseSettings): LOG_LEVEL: str = "DEBUG" LOG_BASE_PATH: str = app_settings.LOG_BASE_PATH LOG_PATH: str = LOG_BASE_PATH + '/' + app_settings.BACKEND_LOG_FILE_NAME + '.log' diff --git a/sites/content/deploy/alpha/.env b/sites/content/deploy/alpha/.env index 68a59e2..a7a6509 100755 --- a/sites/content/deploy/alpha/.env +++ b/sites/content/deploy/alpha/.env @@ -1,2 +1,3 @@ export MONGODB_URI='mongodb+srv://jetli:8IHKx6dZK8BfugGp@freeleaps2.hanbj.mongodb.net/' +export CENTRAL_STORAGE_WEBAPI_URL_BASE="http://if030-w2-if-vm.mathmast.com:8005/api" export FREELEAPS_ENV=alpha diff --git a/sites/content/deploy/common/.env b/sites/content/deploy/common/.env index b11fec5..6523d4a 100755 --- a/sites/content/deploy/common/.env +++ b/sites/content/deploy/common/.env @@ -4,7 +4,6 @@ export MONGODB_NAME=freeleaps2 export MONGODB_PORT=27017 export CONTAINER_APP_ROOT=/app export FREELEAPS_WWW_AS_AZURE_CLIENT_SECRET=3gK8Q~PJbyWmiNqaGgho2ZqCY~OXzABSyN8wWasK -export CENTRAL_STORAGE_WEBAPI_URL_BASE="http://if010-w2-if-vm.mathmast.com:8005/api" export LOG_BASE_PATH=$CONTAINER_APP_ROOT/log/$APP_NAME export BACKEND_LOG_FILE_NAME=$APP_NAME export APPLICATION_ACTIVITY_LOG=$APP_NAME-activity diff --git a/sites/content/deploy/dev/.env b/sites/content/deploy/dev/.env index 503eb4c..24f8fa8 100755 --- a/sites/content/deploy/dev/.env +++ b/sites/content/deploy/dev/.env @@ -1,2 +1,3 @@ export MONGODB_URI=mongodb://freeleaps2-mongodb:27017/ +export CENTRAL_STORAGE_WEBAPI_URL_BASE="http://central_storage:8005/api" export FREELEAPS_ENV=dev diff --git a/sites/content/deploy/local/.env b/sites/content/deploy/local/.env index aec25fc..da76755 100644 --- a/sites/content/deploy/local/.env +++ b/sites/content/deploy/local/.env @@ -1,4 +1,5 @@ export MONGODB_URI=mongodb://localhost:27017/ +export CENTRAL_STORAGE_WEBAPI_URL_BASE="http://localhost:8005/api" export FREELEAPS_ENV=local export LOG_BASE_PATH=${CODEBASE_ROOT}/log diff --git a/sites/content/deploy/prod/.env b/sites/content/deploy/prod/.env index 0167e99..82cfbfc 100755 --- a/sites/content/deploy/prod/.env +++ b/sites/content/deploy/prod/.env @@ -1,2 +1,3 @@ export MONGODB_URI='mongodb+srv://freeadmin:0eMV0bt8oyaknA0m@freeleaps2.zmsmpos.mongodb.net/?retryWrites=true&w=majority' +export CENTRAL_STORAGE_WEBAPI_URL_BASE="http://if010-w2-if-vm.mathmast.com:8005/api" export FREELEAPS_ENV=prod