Merge pull request #5 from alpha-miner/master

merge update

README.md

@@ -20,6 +20,16 @@
 
 **Alpha - Mind** 是基于 **Python** 开发的股票多因子研究框架。
 
+## 依赖
+
+该项目主要有两个主要的github外部依赖：
+
+* [portfolio - optimizer](https://github.com/alpha-miner/portfolio-optimizer)：该项目是相同作者编写的用于资产组合配置的优化器工具包；
+
+* [xgboost](https://github.com/dmlc/xgboost)： 该项目是alpha - mind中一些机器模型的基础库。
+
+这两个库都已经使用git子模块的方式打包到alpha-mind代码库中。
+
 ## 功能
 
 alpha - mind 提供了多因子研究中常用的工具链，包括：

alphamind/analysis/crosssetctions.py

@@ -33,7 +33,7 @@ def cs_impl(ref_date,
     total_risk_exp = total_data[constraint_risk]
 
     er = total_data[[factor_name]].values.astype(float)
-    er = factor_processing(er, [winsorize_normal, standardize], total_risk_exp.values, [winsorize_normal, standardize]).flatten()
+    er = factor_processing(er, [winsorize_normal, standardize], total_risk_exp.values, [standardize]).flatten()
     industry = total_data.industry_name.values
 
     codes = total_data.code.tolist()
@@ -46,7 +46,10 @@ def cs_impl(ref_date,
     total_risk_exp = target_pos[constraint_risk]
     activate_weight = target_pos['weight'].values
     excess_return = np.exp(target_pos[['dx']].values) - 1.
-    excess_return = factor_processing(excess_return, [winsorize_normal, standardize], total_risk_exp.values, [winsorize_normal, standardize]).flatten()
+    excess_return = factor_processing(excess_return,
+                                      [winsorize_normal, standardize],
+                                      total_risk_exp.values,
+                                      [winsorize_normal, standardize]).flatten()
     port_ret = np.log(activate_weight @ excess_return + 1.)
     ic = np.corrcoef(excess_return, activate_weight)[0, 1]
     x = sm.add_constant(activate_weight)

alphamind/cython/optimizers.pyx

@@ -7,13 +7,14 @@ Created on 2017-7-20
 """
 
 cimport numpy as cnp
+from libcpp.string cimport string
 from libcpp.vector cimport vector
 import numpy as np
 
 
 cdef extern from "lpoptimizer.hpp" namespace "pfopt":
     cdef cppclass LpOptimizer:
-        LpOptimizer(int, int, double*, double*, double*, double*) except +
+        LpOptimizer(int, int, double*, double*, double*, double*, string) except +
         vector[double] xValue()
         double feval()
         int status()
@@ -26,12 +27,15 @@ cdef class LPOptimizer:
     cdef int m
 
     def __cinit__(self,
-                 cnp.ndarray[double, ndim=2] cons_matrix,
-                 double[:] lbound,
-                 double[:] ubound,
-                 double[:] objective):
+                  cnp.ndarray[double, ndim=2] cons_matrix,
+                  double[:] lbound,
+                  double[:] ubound,
+                  double[:] objective,
+                  str method='simplex'):
         self.n = lbound.shape[0]
         self.m = cons_matrix.shape[0]
+        py_bytes = method.encode('ascii')
+        cdef string c_str = py_bytes
         cdef double[:] cons = cons_matrix.flatten(order='C');
 
         self.cobj = new LpOptimizer(self.n,
@@ -39,7 +43,8 @@ cdef class LPOptimizer:
                                     &cons[0],
                                     &lbound[0],
                                     &ubound[0],
-                                    &objective[0])
+                                    &objective[0],
+                                    c_str)
 
     def __dealloc__(self):
         del self.cobj
@@ -151,9 +156,25 @@ cdef extern from "mvoptimizer.hpp" namespace "pfopt":
         int status()
 
 
+cdef extern from "qpalglib.hpp" namespace "pfopt":
+    cdef cppclass QPAlglib:
+        QPAlglib(int,
+                 double*,
+                 double*,
+                 double*,
+                 double*,
+                 double) except +
+        vector[double] xValue()
+        int status()
+
+
 cdef class QPOptimizer:
 
     cdef MVOptimizer* cobj
+    cdef QPAlglib* cobj2
+    cdef cnp.ndarray er
+    cdef cnp.ndarray cov
+    cdef double risk_aversion
     cdef int n
     cdef int m
 
@@ -169,12 +190,15 @@ cdef class QPOptimizer:
 
         self.n = lbound.shape[0]
         self.m = 0
+        self.er = np.array(expected_return)
+        self.cov = np.array(cov_matrix)
+        self.risk_aversion = risk_aversion
         cdef double[:] cov = cov_matrix.flatten(order='C')
         cdef double[:] cons
 
         if cons_matrix is not None:
             self.m = cons_matrix.shape[0]
-            cons = cons_matrix.flatten(order='C');
+            cons = cons_matrix.flatten(order='C')
 
             self.cobj = new MVOptimizer(self.n,
                                         &expected_return[0],
@@ -187,25 +211,39 @@ cdef class QPOptimizer:
                                         &cubound[0],
                                         risk_aversion)
         else:
-            self.cobj = new MVOptimizer(self.n,
-                                        &expected_return[0],
-                                        &cov[0],
-                                        &lbound[0],
-                                        &ubound[0],
-                                        0,
-                                        NULL,
-                                        NULL,
-                                        NULL,
-                                        risk_aversion)
+            self.cobj2 = new QPAlglib(self.n,
+                                      &expected_return[0],
+                                      &cov[0],
+                                      &lbound[0],
+                                      &ubound[0],
+                                      risk_aversion)
 
     def __dealloc__(self):
-        del self.cobj
+        if self.cobj:
+            del self.cobj
+        else:
+            del self.cobj2
 
     def feval(self):
-        return self.cobj.feval()
+        if self.cobj:
+            return self.cobj.feval()
+        else:
+            x = np.array(self.cobj2.xValue())
+            return 0.5 * self.risk_aversion * x @ self.cov @ x - self.er @ x
 
     def x_value(self):
-        return np.array(self.cobj.xValue())
+        if self.cobj:
+            return np.array(self.cobj.xValue())
+        else:
+            return np.array(self.cobj2.xValue())
 
     def status(self):
-        return self.cobj.status()
\ No newline at end of file
+        if self.cobj:
+            return self.cobj.status()
+        else:
+            status = self.cobj2.status()
+
+            if 1 <= status <= 4:
+                return 0
+            else:
+                return status

alphamind/data/engines/universe.py

@@ -14,8 +14,8 @@ from sqlalchemy import select
 from sqlalchemy import join
 from sqlalchemy import outerjoin
 from alphamind.data.dbmodel.models import Universe as UniverseTable
-from alphamind.data.dbmodel.models import FullFactor
 from alphamind.data.engines.utilities import _map_factors
+from alphamind.data.dbmodel.models import Market
 from alphamind.data.engines.utilities import factor_tables
 from alphamind.data.transformer import Transformer
 from alphamind.utilities import encode
@@ -86,22 +86,22 @@ class Universe(object):
 
             dependency = transformer.dependency
             factor_cols = _map_factors(dependency, factor_tables)
-            big_table = FullFactor
+            big_table = Market
 
             for t in set(factor_cols.values()):
-                if t.__table__.name != FullFactor.__table__.name:
-                    big_table = outerjoin(big_table, t, and_(FullFactor.trade_date == t.trade_date,
-                                                             FullFactor.code == t.code,
-                                                             FullFactor.trade_date.in_(
-                                                                 dates) if dates else FullFactor.trade_date.between(
+                if t.__table__.name != Market.__table__.name:
+                    big_table = outerjoin(big_table, t, and_(Market.trade_date == t.trade_date,
+                                                             Market.code == t.code,
+                                                             Market.trade_date.in_(
+                                                                 dates) if dates else Market.trade_date.between(
                                                                  start_date, end_date)))
             big_table = join(big_table, UniverseTable,
-                             and_(FullFactor.trade_date == UniverseTable.trade_date,
-                                  FullFactor.code == UniverseTable.code,
+                             and_(Market.trade_date == UniverseTable.trade_date,
+                                  Market.code == UniverseTable.code,
                                   universe_cond))
 
             query = select(
-                [FullFactor.trade_date, FullFactor.code] + list(factor_cols.keys())) \
+                [Market.trade_date, Market.code] + list(factor_cols.keys())) \
                 .select_from(big_table).distinct()
 
             df = pd.read_sql(query, engine.engine).sort_values(['trade_date', 'code']).dropna()

alphamind/data/processing.py

@@ -9,6 +9,7 @@ from typing import Optional
 from typing import List
 import numpy as np
 from alphamind.data.neutralize import neutralize
+from alphamind.utilities import alpha_logger
 
 
 def factor_processing(raw_factors: np.ndarray,
@@ -29,6 +30,8 @@ def factor_processing(raw_factors: np.ndarray,
 
     if post_process:
         for p in post_process:
+            if p.__name__ == 'winsorize_normal':
+                alpha_logger.warning("winsorize_normal normally should not be done after neutralize")
             new_factors = p(new_factors, groups=groups)
 
     return new_factors

pfopt @ 1dcecd88

-Subproject commit d04d8ac4ca33c4482ac2baae9b537a0df0d1b036
+Subproject commit 1dcecd88728512ff50730f418d9d58195bf33851

alphamind/portfolio/linearbuilder.py

@@ -17,7 +17,8 @@ def linear_builder(er: np.ndarray,
                    risk_constraints: np.ndarray,
                    risk_target: Tuple[np.ndarray, np.ndarray],
                    turn_over_target: float = None,
-                   current_position: np.ndarray = None) -> Tuple[str, np.ndarray, np.ndarray]:
+                   current_position: np.ndarray = None,
+                   method: str='simplex') -> Tuple[str, np.ndarray, np.ndarray]:
     er = er.flatten()
     n, m = risk_constraints.shape
 
@@ -36,7 +37,7 @@ def linear_builder(er: np.ndarray,
 
     if not turn_over_target:
         cons_matrix = np.concatenate((risk_constraints.T, risk_lbound, risk_ubound), axis=1)
-        opt = LPOptimizer(cons_matrix, lbound, ubound, -er)
+        opt = LPOptimizer(cons_matrix, lbound, ubound, -er, method)
 
         status = opt.status()
 
@@ -77,7 +78,7 @@ def linear_builder(er: np.ndarray,
         risk_ubound = np.concatenate((risk_ubound, np.inf * np.ones((n, 1))), axis=0)
 
         cons_matrix = np.concatenate((risk_constraints, risk_lbound, risk_ubound), axis=1)
-        opt = LPOptimizer(cons_matrix, lbound, ubound, -er)
+        opt = LPOptimizer(cons_matrix, lbound, ubound, -er, method)
 
         status = opt.status()
 

alphamind/tests/portfolio/test_linearbuild.py

@@ -37,6 +37,25 @@ class TestLinearBuild(unittest.TestCase):
         expected_risk = np.zeros(self.risk_exp.shape[1])
         np.testing.assert_array_almost_equal(calc_risk, expected_risk)
 
+    def test_linear_build_with_interior(self):
+        bm = self.bm / self.bm.sum()
+        eplson = 1e-6
+
+        status, _, w = linear_builder(self.er,
+                                      0.,
+                                      0.01,
+                                      self.risk_exp,
+                                      (bm @ self.risk_exp, bm @ self.risk_exp),
+                                      method='interior')
+        self.assertEqual(status, 'optimal')
+        self.assertAlmostEqual(np.sum(w), 1.)
+        self.assertTrue(np.all(w <= 0.01 + eplson))
+        self.assertTrue(np.all(w >= -eplson))
+
+        calc_risk = (w - bm) @ self.risk_exp
+        expected_risk = np.zeros(self.risk_exp.shape[1])
+        np.testing.assert_array_almost_equal(calc_risk, expected_risk)
+
     def test_linear_build_with_inequality_constraints(self):
         bm = self.bm / self.bm.sum()
         eplson = 1e-6

build_linux_dependencies.sh

@@ -21,7 +21,10 @@ fi
 cd ../..
 
 cd alphamind/pfopt
+
+export BUILD_TEST=OFF
 ./build_linux.sh
+
 if [ $? -ne 0 ] ; then
     cd ../..
     exit 1

build_windows_dependencies.bat

@@ -19,6 +19,7 @@ cd ../..
 
 cd alphamind\pfopt
 
+set BUILD_TEST=OFF
 call build_windows.bat
 
 if %errorlevel% neq 0 exit /b 1

notebooks/Example 3 - Multi Weight Gap Comparison.ipynb

@@ -44,7 +44,7 @@
     "batch = 0\n",
     "horizon = map_freq(freq)\n",
     "universe = Universe(\"custom\", ['zz800'])\n",
-    "data_source = 'postgres+psycopg2://postgres:A12345678!@10.63.6.220/alpha'\n",
+    "data_source = 'postgres+psycopg2://postgres:we083826@localhost/alpha'\n",
     "benchmark_code = 905\n",
     "method = 'tv'\n",
     "target_vol = 0.05\n",
@@ -269,9 +269,9 @@
    "outputs": [],
    "source": [
     "def create_report(ret_df, windows):\n",
-    "    sharp_calc = MovingSharp(windows)\n",
-    "    drawdown_calc = MovingMaxDrawdown(windows)\n",
-    "    max_drawdown_calc = MovingMaxDrawdown(len(ret_df))\n",
+    "    sharp_calc = MovingSharp(windows, x='ret', y='riskFree')\n",
+    "    drawdown_calc = MovingMaxDrawdown(windows, x='ret')\n",
+    "    max_drawdown_calc = MovingMaxDrawdown(len(ret_df), x='ret')\n",
     "\n",
     "    res_df = pd.DataFrame(columns=['daily_return', 'cum_ret', 'sharp', 'drawdown', 'max_drawn', 'leverage'])\n",
     "    total_returns = 0.\n",
@@ -285,8 +285,8 @@
     "\n",
     "        res_df.loc[date, 'daily_return'] = ret\n",
     "        res_df.loc[date, 'cum_ret'] = total_returns\n",
-    "        res_df.loc[date, 'drawdown'] = drawdown_calc.result()[0]\n",
-    "        res_df.loc[date, 'max_drawn'] = max_drawdown_calc.result()[0]\n",
+    "        res_df.loc[date, 'drawdown'] = drawdown_calc.result()\n",
+    "        res_df.loc[date, 'max_drawn'] = max_drawdown_calc.result()\n",
     "        res_df.loc[date, 'leverage'] = ret_df.loc[date, 'leverage']\n",
     "\n",
     "        if i < 5:\n",
@@ -328,13 +328,6 @@
     "        alpha_logger.info(f\"target_vol: {target_vol:.4f} finished\")"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -349,18 +342,6 @@
    "language": "python",
    "name": "python3"
   },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.6.4"
-  },
   "varInspector": {
    "cols": {
     "lenName": 16,

setup.py

@@ -51,7 +51,16 @@ else:
                                 "alphamind/pfopt/include/pfopt",
                                 "alphamind/pfopt/include/eigen",
                                 "alphamind/pfopt/include/alglib"],
-                  libraries=['pfopt', 'alglib', 'libClp', 'libCoinUtils', 'libipopt', 'libcoinhsl', 'libcoinblas', 'libcoinlapack', 'libcoinmetis'],
+                  libraries=['pfopt',
+                             'alglib',
+                             'libClp',
+                             'libCoinUtils',
+                             'libipopt',
+                             'libcoinhsl',
+                             'libcoinblas',
+                             'libcoinlapack',
+                             'libcoinmetis',
+                             'libcoinmumps'],
                   library_dirs=['alphamind/pfopt/lib'],
                   extra_compile_args=['/MD']),
     ]

xgboost @ a187ed6c

-Subproject commit bf4367184164e593cd2856ef38f8dd4f8cc76999
+Subproject commit a187ed6c8f3aa40b47d5be80667cbbe6a6fd563d