added example for optimizer check

notebooks/optimizer check.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import sqlalchemy as sa\n",
+    "from cvxpy import *\n",
+    "from alphamind.data.neutralize import neutralize\n",
+    "\n",
+    "engine = sa.create_engine('mysql+pymysql://sa:We051253524522@rm-bp1psdz5615icqc0yo.mysql.rds.aliyuncs.com/uqer?charset=utf8')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "codes_list = [ \"1\",\"2\",\"4\",\"5\",\"6\",\"7\",\"8\",\"9\",\"10\",\"11\",\"12\",\"14\",\"16\",\"17\",\"18\",\"19\",\"20\",\"21\",\"22\",\"23\",\"25\",\"26\",\"27\",\"28\",\"29\",\"30\",\"31\",\"32\",\"33\",\"34\",\"35\",\"36\",\"37\",\"38\",\"39\",\"40\",\"42\",\"43\",\"45\",\"46\",\"48\",\"49\",\"50\",\"55\",\"56\",\"58\",\"59\",\"60\",\"61\",\"62\",\"63\",\"65\",\"66\",\"68\",\"69\",\"70\",\"78\",\"88\",\"89\",\"90\",\"96\",\"99\",\"100\",\"150\",\"151\",\"153\",\"155\",\"156\",\"157\",\"158\",\"159\",\"166\",\"301\",\"333\",\"338\",\"400\",\"401\",\"402\",\"403\",\"404\",\"407\",\"408\",\"409\",\"410\",\"411\",\"413\",\"415\",\"416\",\"417\",\"418\",\"419\",\"420\",\"421\",\"422\",\"423\",\"425\",\"426\",\"428\",\"429\",\"430\",\"488\",\"498\",\"501\",\"502\",\"503\",\"504\",\"505\",\"506\",\"507\",\"509\",\"510\",\"511\",\"513\",\"514\",\"516\",\"517\",\"518\",\"519\",\"520\",\"521\",\"523\",\"524\",\"525\",\"526\",\"528\",\"529\",\"530\",\"531\",\"532\",\"533\",\"534\",\"536\",\"537\",\"538\",\"539\",\"540\",\"541\",\"543\",\"544\",\"545\",\"546\",\"547\",\"548\",\"550\",\"551\",\"552\",\"553\",\"554\",\"555\",\"557\",\"558\",\"559\",\"560\",\"561\",\"563\",\"564\",\"565\",\"566\",\"567\",\"568\",\"570\",\"571\",\"572\",\"573\",\"576\",\"581\",\"582\",\"584\",\"585\",\"586\",\"587\",\"589\",\"590\",\"591\",\"592\",\"593\",\"595\",\"596\",\"597\",\"598\",\"599\",\"600\",\"601\",\"603\",\"605\",\"606\",\"607\",\"608\",\"609\",\"610\",\"611\",\"612\",\"613\",\"615\",\"616\",\"617\",\"619\",\"620\",\"622\",\"623\",\"625\",\"626\",\"627\",\"628\",\"629\",\"630\",\"631\",\"632\",\"633\",\"635\",\"636\",\"637\",\"638\",\"639\",\"650\",\"651\",\"652\",\"655\",\"656\",\"657\",\"659\",\"661\",\"662\",\"663\",\"665\",\"666\",\"667\",\"668\",\"669\",\"670\",\"671\",\"672\",\"673\",\"676\",\"677\",\"678\",\"679\",\"680\",\"681\",\"682\",\"683\",\"685\",\"686\",\"687\",\"688\",\"690\",\"691\",\"692\",\"693\",\"695\",\"697\",\"698\",\"700\",\"701\",\"702\",\"703\",\"705\",\"707\",\"708\",\"709\",\"710\",\"711\",\"712\",\"713\",\"715\",\"716\",\"717\",\"718\",\"719\",\"720\",\"721\",\"722\",\"723\",\"725\",\"726\",\"727\",\"728\",\"729\",\"731\",\"732\",\"733\",\"735\",\"736\",\"737\",\"738\",\"739\",\"750\",\"751\",\"752\",\"753\",\"755\",\"756\",\"757\",\"758\",\"759\",\"760\",\"761\",\"762\",\"766\",\"767\",\"768\",\"776\",\"777\",\"778\",\"779\",\"780\",\"782\",\"783\",\"785\",\"786\",\"788\",\"789\",\"790\",\"791\",\"792\",\"793\",\"795\",\"796\",\"797\",\"798\",\"799\",\"800\",\"801\",\"802\",\"803\",\"806\",\"807\",\"809\",\"810\",\"811\",\"812\",\"813\",\"815\",\"816\",\"818\",\"819\",\"820\",\"821\",\"822\",\"823\",\"825\",\"826\",\"828\",\"829\",\"830\",\"831\",\"833\",\"835\",\"836\",\"837\",\"838\",\"839\",\"848\",\"850\",\"851\",\"852\",\"856\",\"858\",\"859\",\"860\",\"861\",\"862\",\"863\",\"868\",\"869\",\"875\",\"876\",\"877\",\"878\",\"880\",\"881\",\"882\",\"883\",\"885\",\"886\",\"887\",\"888\",\"889\",\"890\",\"892\",\"893\",\"895\",\"897\",\"898\",\"899\",\"900\",\"901\",\"902\",\"903\",\"905\",\"906\",\"908\",\"909\",\"910\",\"911\",\"912\",\"913\",\"915\",\"916\",\"917\",\"918\",\"919\",\"920\",\"921\",\"922\",\"923\",\"925\",\"926\",\"927\",\"928\",\"929\",\"930\",\"931\",\"932\",\"933\",\"935\",\"936\",\"937\",\"938\",\"939\",\"948\",\"949\",\"950\",\"951\",\"952\",\"953\",\"955\",\"957\",\"958\",\"959\",\"960\",\"961\",\"962\",\"963\",\"965\",\"966\",\"967\",\"968\",\"969\",\"970\",\"971\",\"972\",\"973\",\"975\",\"976\",\"977\",\"978\",\"979\",\"980\",\"981\",\"982\",\"983\",\"985\",\"987\",\"988\",\"989\",\"990\",\"993\",\"995\",\"996\",\"997\",\"998\",\"999\",\"1696\",\"1896\",\"1979\",\"2001\",\"2002\",\"2003\",\"2004\",\"2005\",\"2006\",\"2007\",\"2008\",\"2009\",\"2010\",\"2011\",\"2012\",\"2013\",\"2014\",\"2015\",\"2016\",\"2017\",\"2018\",\"2019\",\"2020\",\"2021\",\"2022\",\"2023\",\"2024\",\"2025\",\"2026\",\"2027\",\"2028\",\"2029\",\"2030\",\"2031\",\"2032\",\"2033\",\"2034\",\"2035\",\"2036\",\"2037\",\"2038\",\"2039\",\"2040\",\"2041\",\"2042\",\"2043\",\"2044\",\"2045\",\"2046\",\"2047\",\"2048\",\"2049\",\"2050\",\"2051\",\"2052\",\"2053\",\"2054\",\"2055\",\"2056\",\"2057\",\"2058\",\"2059\",\"2060\",\"2061\",\"2062\",\"2063\",\"2064\",\"2065\",\"2066\",\"2067\",\"2068\",\"2069\",\"2070\",\"2071\",\"2072\",\"2073\",\"2074\",\"2075\",\"2076\",\"2077\",\"2078\",\"2079\",\"2080\",\"2081\",\"2082\",\"2083\",\"2084\",\"2085\",\"2086\",\"2087\",\"2088\",\"2089\",\"2090\",\"2091\",\"2092\",\"2093\",\"2094\",\"2095\",\"2096\",\"2097\",\"2098\",\"2099\",\"2100\",\"2101\",\"2102\",\"2103\",\"2104\",\"2105\",\"2106\",\"2107\",\"2108\",\"2109\",\"2110\",\"2111\",\"2112\",\"2113\",\"2114\",\"2115\" ]\n",
+    "factor_list = [\"DebtEquityRatio\",\"EBITToTOR\"]\n",
+    "ref_date = '2017-05-23'\n",
+    "weights = np.array([0.3,0.7])\n",
+    "risk_penlty = 0.5\n",
+    "n = len(codes_list)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Data Preparing\n",
+    "--------------------------"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "codes = ','.join(codes_list)\n",
+    "factors = ','.join(factor_list)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "factor_data = pd.read_sql(\"select Code, {1} from factors where Date = '{0}' and Code in ({2})\".format(ref_date, factors, codes), engine).sort_values('Code')\n",
+    "risk_exposure_data = pd.read_sql(\"select * from risk_exposure where Date = '{0}' and Code in ({1})\".format(ref_date, codes), engine).sort_values('Code')\n",
+    "risk_exposure_data.drop(['Date', 'exchangeCD', 'secShortName'], axis=1, inplace=True)\n",
+    "risk_cov_data = pd.read_sql(\"select * from risk_cov_day where Date = '{0}'\".format(ref_date), engine).sort_values('FactorID')\n",
+    "risk_cov_data.drop(['Date', 'FactorID'], axis=1, inplace=True)\n",
+    "risk_cov_data.set_index('Factor', inplace=True)\n",
+    "specific_risk = pd.read_sql(\"select Code, SRISK from specific_risk_day where Date = '{0}' and Code in ({1})\".format(ref_date, codes), engine).sort_values('Code')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "factor_values = factor_data[factor_list].values\n",
+    "risk_factor_columns = sorted(risk_cov_data.columns[:38])\n",
+    "risk_exposure_values = risk_exposure_data[risk_factor_columns].values"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "ne_factor_values = neutralize(risk_exposure_values, factor_values)\n",
+    "signal = ne_factor_values @ weights"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "risk_cov_values = risk_cov_data.loc[risk_factor_columns, risk_factor_columns].astype(float)\n",
+    "sec_cov_values = risk_exposure_values @ risk_cov_values @ risk_exposure_values.T + np.diag(specific_risk.SRISK.values)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stocks_cov_matrix = pd.DataFrame(-0.5 * sec_cov_values, columns=codes_list, index=codes_list)\n",
+    "stocks_cov_matrix.to_csv('d:/stocks_cov_matrix.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sec_cov_values"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Optimizing Weights\n",
+    "-------------------------------------"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "w = Variable(n)\n",
+    "\n",
+    "lbound = 0.\n",
+    "ubound = 0.01\n",
+    "\n",
+    "objective = Minimize(risk_penlty * quad_form(w, sec_cov_values)  - signal * w)\n",
+    "constraints = [w >= lbound,\n",
+    "               w <= ubound,\n",
+    "               sum_entries(w) == 1,]\n",
+    "\n",
+    "prob = Problem(objective, constraints)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "prob.solve(verbose=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "optimal_weights1 = pd.DataFrame(np.array(w.value).flatten(), columns=['weight'], index=codes_list)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "prob.solve(verbose=True, solver='CVXOPT')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prob.status, prob.value, np.array(w.value).flatten()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "optimal_weights2 = pd.DataFrame(np.array(w.value).flatten(), columns=['weight'], index=codes_list)\n",
+    "optimal_weights2.to_csv('d:/optimal_weights.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}