made model persistence compatible with postgres

alphamind/data/dbmodel/models.py

@@ -1656,6 +1656,20 @@ class FullFactorView(Base):
     l_srisk = Column(Float(53))
 
 
+class Models(Base):
+    __tablename__ = 'models'
+    __table_args__ = (
+        Index('model_pk', 'trade_date', 'portfolio_name', 'model_type', 'version', unique=True),
+    )
+
+    trade_date = Column(DateTime, primary_key=True, nullable=False)
+    model_desc = Column(JSON, nullable=False)
+    portfolio_name = Column(String(30), primary_key=True, nullable=False)
+    model_type = Column(String(30), primary_key=True, nullable=False)
+    version = Column(BigInteger, primary_key=True, nullable=False)
+    update_time = Column(DateTime, nullable=False)
+
+
 if __name__ == '__main__':
     from sqlalchemy import create_engine
 

alphamind/data/engines/sqlengine.py

@@ -27,10 +27,11 @@ from alphamind.data.dbmodel.models import RiskCovDay
 from alphamind.data.dbmodel.models import RiskCovShort
 from alphamind.data.dbmodel.models import RiskCovLong
 from alphamind.data.dbmodel.models import FullFactorView
+from alphamind.data.dbmodel.models import Models
 from alphamind.data.dbmodel.models import Universe as UniverseTable
 from alphamind.data.transformer import Transformer
+from alphamind.model.loader import load_model
 from PyFin.api import advanceDateByCalendar
-from PyFin.Analysis.SecurityValueHolders import SecurityLatestValueHolder
 
 risk_styles = ['BETA',
                'MOMENTUM',
@@ -522,6 +523,33 @@ class SqlEngine(object):
         total_data['factor'] = factor_data
         return total_data
 
+    def fetch_model(self,
+                    ref_date,
+                    portfolio=None,
+                    model_type=None,
+                    version=None) -> pd.DataFrame:
+
+        conditions = [Models.trade_date == ref_date]
+
+        if portfolio:
+            conditions.append(Models.portfolio_name == portfolio)
+
+        if model_type:
+            conditions.append(Models.model_type == model_type)
+
+        if version:
+            conditions.append(Models.version == version)
+
+        query = select([Models]).where(and_(*conditions))
+
+        model_df = pd.read_sql(query, self.engine)
+
+        for i, model_desc in enumerate(model_df.model_desc):
+            model_df.loc[i, 'model'] = load_model(model_desc)
+
+        del model_df['model_desc']
+        return model_df
+
 
 if __name__ == '__main__':
     import datetime as dt

alphamind/examples/model_zoo.py

+# -*- coding: utf-8 -*-
+"""
+Created on 2017-9-5
+
+@author: cheng.li
+"""
+
+import sqlalchemy as sa
+import arrow
+import numpy as np
+import pandas as pd
+from alphamind.api import *
+from alphamind.data.dbmodel.models import Models
+from alphamind.model.linearmodel import LinearRegression
+
+engine = SqlEngine('postgresql+psycopg2://postgres:A12345678!@10.63.6.220/alpha')
+
+x = np.random.randn(1000, 3)
+y = np.random.randn(1000)
+
+model = LinearRegression(['a', 'b', 'c'])
+model.fit(x, y)
+
+model_desc = model.save()
+
+df = pd.DataFrame()
+
+new_row = dict(trade_date='2017-09-05',
+               portfolio_name='test',
+               model_type='LinearRegression',
+               version=1,
+               model_desc=model_desc,
+               update_time=arrow.now().format())
+
+df = df.append([new_row])
+
+df.to_sql(Models.__table__.name, engine.engine,
+          if_exists='append',
+          index=False,
+          dtype={'model_desc': sa.types.JSON})
+
+model_in_db = engine.fetch_model('2017-09-05')
+
+print(model_in_db)
+

alphamind/model/linearmodel.py

@@ -5,6 +5,7 @@ Created on 2017-5-10
 @author: cheng.li
 """
 
+import base64
 import pickle
 import numpy as np
 import arrow
@@ -63,8 +64,9 @@ class LinearRegression(ModelBase):
 
     def save(self) -> dict:
         model_desc = super().save()
-        model_desc['internal_model'] = self.impl.__class__.__module__ + "." + self.impl.__class__.__name__,
-        model_desc['desc'] = pickle.dumps(self.impl)
+        model_desc['internal_model'] = self.impl.__class__.__module__ + "." + self.impl.__class__.__name__
+        encoded = base64.encodebytes(pickle.dumps(self.impl))
+        model_desc['desc'] = encoded.decode('ascii')
         model_desc['sklearn_version'] = sklearn_version
         model_desc['trained_time'] = self.trained_time
         return model_desc
@@ -83,7 +85,8 @@ class LinearRegression(ModelBase):
                                  'Loaded model may work incorrectly.'.format(
                 sklearn_version, model_desc['sklearn_version']))
 
-        obj_layout.impl = pickle.loads(model_desc['desc'])
+        encoded = model_desc['desc'].encode('ascii')
+        obj_layout.impl = pickle.loads(base64.decodebytes(encoded))
         return obj_layout
 
 

alphamind/model/loader.py

@@ -6,7 +6,18 @@ Created on 2017-9-5
 """
 
 from alphamind.model.modelbase import ModelBase
+from alphamind.model.linearmodel import ConstLinearModel
+from alphamind.model.linearmodel import LinearRegression
 
 
 def load_model(model_desc: dict) -> ModelBase:
-    pass
+
+    model_name = model_desc['model_name']
+    model_name_parts = set(model_name.split('.'))
+
+    if 'ConstLinearModel' in model_name_parts:
+        return ConstLinearModel.load(model_desc)
+    elif 'LinearRegression' in model_name_parts:
+        return LinearRegression.load(model_desc)
+    else:
+        raise ValueError('{0} is not currently supported in model loader.'.format(model_name))

alphamind/tests/model/test_loader.py

+# -*- coding: utf-8 -*-
+"""
+Created on 2017-9-5
+
+@author: cheng.li
+"""
+
+import unittest
+import numpy as np
+from alphamind.model.linearmodel import LinearRegression
+from alphamind.model.loader import load_model
+
+
+class TestLoader(unittest.TestCase):
+
+    def setUp(self):
+        self.n = 3
+        self.trained_x = np.random.randn(1000, self.n)
+        self.trained_y = np.random.randn(1000, 1)
+
+        self.predict_x = np.random.randn(100, self.n)
+
+    def test_load_model(self):
+        model = LinearRegression(['a', 'b', 'c'])
+        model.fit(self.trained_x, self.trained_y)
+
+        model_desc = model.save()
+        new_model = load_model(model_desc)
+
+        np.testing.assert_array_almost_equal(model.predict(self.predict_x),
+                                             new_model.predict(self.predict_x))
+
+        self.assertEqual(model.features, new_model.features)
+        self.assertEqual(model.trained_time, new_model.trained_time)

alphamind/tests/test_suite.py

@@ -27,6 +27,7 @@ from alphamind.tests.analysis.test_perfanalysis import TestPerformanceAnalysis
 from alphamind.tests.analysis.test_factoranalysis import TestFactorAnalysis
 from alphamind.tests.analysis.test_quantilieanalysis import TestQuantileAnalysis
 from alphamind.tests.model.test_linearmodel import TestLinearModel
+from alphamind.tests.model.test_loader import TestLoader
 
 
 if __name__ == '__main__':
@@ -45,6 +46,7 @@ if __name__ == '__main__':
                          TestPerformanceAnalysis,
                          TestFactorAnalysis,
                          TestQuantileAnalysis,
-                         TestLinearModel],
+                         TestLinearModel,
+                         TestLoader],
                         alpha_logger)
     runner.run()