initial implementation (ludwig-ai#2)

.gitignore

@@ -0,0 +1,16 @@
+# Compiled python modules.
+*.pyc
+
+# Setuptools distribution folder.
+/dist/
+
+# Python egg metadata, regenerated from source files by setuptools.
+/*.egg-info
+/*.egg
+.vscode
+.coverage
+.pytest_cache
+.pypirc
+.vim
+.idea
+.env

Dockerfile

@@ -0,0 +1,19 @@
+# Pull base image
+FROM python:3.8
+
+# Set environment varibles
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+WORKDIR /code/
+
+# Install dependencies
+COPY poetry.lock /
+COPY pyproject.toml .
+RUN pip install poetry && \
+    poetry config virtualenvs.create false && \
+    poetry install
+
+COPY . /code/
+
+EXPOSE 8000

LICENSE

@@ -1,4 +1,4 @@
-                                 Apache License
+ Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
 

README.md

@@ -0,0 +1,28 @@
+# Model_hub built on top of fastapi and mongodb   
+
+## Features 
+
+- Docker with [MongoDB](https://www.mongodb.com) and [FastAPI](http://fastapi.tiangolo.com)  
+- [Poetry](https://python-poetry.org) as dependency manager    
+- Works well **async** (all, with db)  
+- Supported snake_case -> cammelCase conversion 
+- Env file parsed by Pydantic    
+- **ObjectID** works well with **FastAPI** & **Pydantic** (I've created custom field. Compatible with FastAPI generic docs)    
+- Structure with **Dependency Injection** (database implementation)    
+
+Build on **Python: 3.8**.    
+
+
+## Installation and usage 
+
+- Create env from template: ```cp example.env .env``` (only once)    
+- Run docker stack ```sudo docker-compose up```    
+
+## TODO 
+
+> Example is completely and works very well. In the future probably I add more.
+
+- Scheme for MongoDB
+- More examples with custom response models 
+- [Maybe] File handling with external provider (Amazon S3, DO Spaces)    
+- [Maybe] Authorization by external provider (Auth0)    

docker-compose.yml

@@ -0,0 +1,20 @@
+version: "3"
+
+services:
+  web:
+    build: .
+    command: bash -c "uvicorn model_hub.main:model_hub --host 0.0.0.0 --port 8000 --reload"
+    volumes:
+      - .:/code
+    ports:
+      - 8000:8000
+    depends_on:
+      - mongo
+  mongo:
+    image: mongo
+    restart: always
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: mongo_user
+      MONGO_INITDB_ROOT_PASSWORD: mongo_password
+    ports:
+      - 27017:27017 # remove this line on prod

example.env

@@ -0,0 +1 @@
+DB_PATH=mongodb://mongo_user:mongo_password@mongo:27017

model_hub/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.1.0'

model_hub/config.py

@@ -0,0 +1,16 @@
+from functools import lru_cache
+
+from pydantic import BaseSettings
+
+
+class Config(BaseSettings):
+    app_name: str = "Model Hub MongoDB API"
+    db_path: str
+
+    class Config:
+        env_file = ".env"
+
+
+@lru_cache()
+def get_config():
+    return Config()

model_hub/db/__init__.py

@@ -0,0 +1,8 @@
+from model_hub.db.database_manager import DatabaseManager
+from model_hub.db.impl.mongo_manager import MongoManager
+
+db = MongoManager()
+
+
+async def get_database() -> DatabaseManager:
+    return db

model_hub/db/database_manager.py

@@ -0,0 +1,41 @@
+from abc import abstractmethod
+from typing import List
+from model_hub.db.models import Model
+
+
+class DatabaseManager(object):
+    @property
+    def client(self):
+        raise NotImplementedError
+
+    @property
+    def db(self):
+        raise NotImplementedError
+
+    @abstractmethod
+    async def connect_to_database(self, path: str):
+        pass
+
+    @abstractmethod
+    async def close_database_connection(self):
+        pass
+
+    @abstractmethod
+    async def get_models(self) -> List[Model]:
+        pass
+
+    @abstractmethod
+    async def get_model(self, model_url: str) -> Model:
+        pass
+
+    @abstractmethod
+    async def add_model(self, model: Model):
+        pass
+
+    @abstractmethod
+    async def update_model(self, model_url: str, model: Model):
+        pass
+
+    @abstractmethod
+    async def delete_model(self, model_url: str):
+        pass

model_hub/db/impl/mongo_manager.py

@@ -0,0 +1,49 @@
+import logging
+from typing import List
+
+from bson import ObjectId
+from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorDatabase
+
+from model_hub.db.database_manager import DatabaseManager
+from model_hub.db.models import Model
+
+
+class MongoManager(DatabaseManager):
+    client: AsyncIOMotorClient = None
+    db: AsyncIOMotorDatabase = None
+
+    async def connect_to_database(self, path: str):
+        logging.info("Connecting to MongoDB.")
+        self.client = AsyncIOMotorClient(
+            path,
+            maxPoolSize=10,
+            minPoolSize=10)
+        self.db = self.client.main_db
+        logging.info("Connected to MongoDB.")
+
+    async def close_database_connection(self):
+        logging.info("Closing connection with MongoDB.")
+        self.client.close()
+        logging.info("Closed connection with MongoDB.")
+
+    async def get_models(self) -> List[Model]:
+        models_list = []
+        models_q = self.db.models.find()
+        async for post in models_q:
+            models_list.append(Model(**post, id=post['_id']))
+        return models_list
+
+    async def get_model(self, model_url: str) -> Model:
+        model_q = await self.db.models.find_one({'_id': ObjectId(model_url)})
+        if model_q:
+            return Model(**model_q, id=model_q['_id'])
+
+    async def delete_model(self, model_url: str):
+        await self.db.models.delete_one({'_id': ObjectId(model_url)})
+
+    async def update_model(self, model_url: str, post: Model):
+        await self.db.models.update_one({'_id': ObjectId(model_url)},
+                                       {'$set': post.dict(exclude={'id'})})
+
+    async def add_model(self, model: Model):
+        await self.db.models.insert_one(model.dict(exclude={'id'}))

model_hub/db/models.py

@@ -0,0 +1,45 @@
+"""
+    I preferred using DB postfix for db models.
+    It will not be confused with response objects - if you will need anything other than a simple CRUD.
+"""
+from pydantic.main import BaseModel
+from typing import Optional
+from bson import ObjectId
+
+
+class OID(str):
+    @classmethod
+    def __get_validators__(cls):
+        yield cls.validate
+
+    @classmethod
+    def validate(cls, v):
+        if v == '':
+            raise TypeError('ObjectId is empty')
+        if ObjectId.is_valid(v) is False:
+            raise TypeError('ObjectId invalid')
+        return str(v)
+
+
+class BaseDBModel(BaseModel):
+    class Config:
+        orm_mode = True
+        allow_population_by_field_name = True
+
+        @classmethod
+        def alias_generator(cls, string: str) -> str:
+            """ Camel case generator """
+            temp = string.split('_')
+            return temp[0] + ''.join(ele.title() for ele in temp[1:])
+
+
+class Model(BaseDBModel):
+    id: Optional[OID]
+    model_url: str
+    name: str
+    description: str
+    version: str
+    ludwig_version: str
+    author: str
+    namespace: str
+

model_hub/main.py

@@ -0,0 +1,25 @@
+import uvicorn
+from fastapi import FastAPI
+
+from model_hub.config import get_config
+from model_hub.db import db
+from model_hub.model import models
+
+model_hub = FastAPI(title="Async FastAPI For ModelHub")
+
+model_hub.include_router(models.router, prefix='/api/models')
+
+
+@model_hub.on_event("startup")
+async def startup():
+    config = get_config()
+    await db.connect_to_database(path=config.db_path)
+
+
+@model_hub.on_event("shutdown")
+async def shutdown():
+    await db.close_database_connection()
+
+
+if __name__ == "__main__":
+    uvicorn.run(model_hub, host="0.0.0.0", port=8000)

model_hub/model/models.py

@@ -0,0 +1,36 @@
+from fastapi import APIRouter, Depends
+
+from model_hub.db.database_manager import DatabaseManager
+from model_hub.db import get_database
+from model_hub.db.models import Model
+
+router = APIRouter()
+
+
+@router.get('/')
+async def all_models(db: DatabaseManager = Depends(get_database)):
+    models = await db.get_models()
+    return models
+
+
+@router.get('/{model_url}')
+async def one_model(model_url: str, db: DatabaseManager = Depends(get_database)):
+    model = await db.get_model(model_url=model_url)
+    return model
+
+
+@router.put('/{model_url}')
+async def update_model(model_url: str, model: Model, db: DatabaseManager = Depends(get_database)):
+    post = await db.update_model(model=model, model_url=model_url)
+    return post
+
+
+@router.post('/', status_code=201)
+async def add_model(post_response: Model, db: DatabaseManager = Depends(get_database)):
+    post = await db.add_model(post_response)
+    return post
+
+
+@router.delete('/{model_url}')
+async def delete_model(model_url: str, db: DatabaseManager = Depends(get_database)):
+    await db.delete_model(model_url=model_url)