{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": true,
        "row": 0,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "%pylab inline\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('ggplot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import sqlalchemy\n",
    "import pandas as pd\n",
    "import alphamind.data.neutralize as ne\n",
    "import alphamind.data.winsorize as ws\n",
    "import alphamind.data.standardize as st\n",
    "import alphamind.portfolio.rankbuilder as rb\n",
    "import alphamind.portfolio.percentbuilder as pb\n",
    "import alphamind.portfolio.linearbuilder as lb\n",
    "import alphamind.analysis.riskanalysis as ra"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 4,
        "hidden": true,
        "row": 0,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "risk_factor_table = 'risk_factor_500'\n",
    "benchmark = 'zz500'\n",
    "factor = 'NetCash' # 'DROEAfterNonRecurring'\n",
    "index_compentns = '500Weight'\n",
    "\n",
    "conn = sqlalchemy.create_engine('mysql+pymysql://xx:xxxx@xxxx:xxxx/multifactor?charset=utf8')\n",
    "factor_df = pd.read_sql('select factor_data.{0},  factor_data.申万一级行业, trade_data.Return as dailyReturn, {1}.*, 1 as Market '\n",
    "                     'from factor_data, trade_data, {1} '\n",
    "                     'where factor_data.Date = {1}.Date and factor_data.Code = {1}.Code '\n",
    "                     'and factor_data.Date = trade_data.Date and factor_data.Code = trade_data.Code;'.format(factor,\n",
    "                                                                                                             risk_factor_table),\n",
    "                     conn)\n",
    "\n",
    "index_components_df = pd.read_sql('select Date, Code, {0} from index_components;'.format(index_compentns), conn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "del factor_df['Bank']\n",
    "del factor_df['NonBankFinancial']\n",
    "factor_df.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "raw_df = pd.merge(factor_df, index_components_df, on=['Date', 'Code'], how='left')\n",
    "raw_df.fillna(0, inplace=True)\n",
    "raw_df[index_compentns] = raw_df[index_compentns] / 100."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 8,
        "height": 10,
        "hidden": true,
        "row": 0,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "risk_facto_cols = raw_df.columns[5:-1]\n",
    "risk_facto_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "df = raw_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "df['d1ret'] = df.dailyReturn.groupby(df.Code).shift(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "benchmark_data = pd.read_sql('select {0}, Date from index_data'.format(benchmark), conn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "benchmark_data['ret'] = benchmark_data[benchmark] / benchmark_data[benchmark].shift(1) - 1.\n",
    "benchmark_data['d1ret_b'] = benchmark_data['ret'] .shift(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "df = pd.merge(df, benchmark_data[['Date', 'd1ret_b']], on='Date', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": true,
        "row": 0,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "df.dropna(inplace=True)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "# to transform industry codes to int variable\n",
    "old_ind_values = df['申万一级行业'].copy()\n",
    "\n",
    "ind_list = df['申万一级行业'].unique()\n",
    "ind_dict = {}\n",
    "for i, ind in enumerate(ind_list):\n",
    "    ind_dict[ind] = i\n",
    "\n",
    "df['申万一级行业'].replace(ind_dict, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": true,
        "row": 0,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "# Factor Date Preprocessing (Winsorize -> Standardize -> neutralize)\n",
    "-----------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "total_data = df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "total_data.loc[:, risk_facto_cols] = total_data[risk_facto_cols].groupby(total_data.Date).transform(lambda x: x / x.sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "y = total_data[factor].values\n",
    "y.shape = -1, 1\n",
    "groups = total_data.Date.values.astype(np.int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 4,
        "hidden": true,
        "row": 0,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "total_data['res'] = ne.neutralize(total_data[risk_facto_cols].values,\n",
    "                                  st.standardize(ws.winsorize_normal(y, groups=groups),\n",
    "                                                 groups=groups),\n",
    "                                  groups)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 9,
        "hidden": true,
        "row": 0,
        "width": 5
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "total_data[[factor, 'res', 'Date', 'Code']].tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": false,
        "row": 0,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "# Factor Performance (long_short)\n",
    "------------------------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 5,
        "height": 4,
        "hidden": true,
        "row": 0,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "total_data['pos_long_short'] = total_data.res.groupby(groups).apply(lambda x: x / np.abs(x).sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 9,
        "hidden": true,
        "row": 4,
        "width": 6
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "total_data[['pos_long_short', 'res', 'dailyReturn', 'd1ret', 'd1ret_b', 'Code', 'Date']].tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "to_look_into = total_data[risk_facto_cols].multiply(total_data.pos_long_short, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 6,
        "height": 18,
        "hidden": true,
        "row": 4,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "to_look_into.groupby(total_data.Date).sum().max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 18,
        "hidden": true,
        "row": 4,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "to_look_into.groupby(total_data.Date).sum().min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "ret_series = (total_data.pos_long_short * total_data.d1ret).groupby(total_data.Date).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 15,
        "hidden": false,
        "row": 4,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "ret_series.cumsum().plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 16,
        "hidden": false,
        "row": 19,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "ret_series.cumsum()[-60:].plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 5,
        "height": 5,
        "hidden": true,
        "row": 0,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "total_data.pos_long_short.groupby(total_data.Date).apply(lambda x: x.sum()).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": false,
        "row": 35,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "# Factor Performance (Long Only - Top 100 Equal Weighted)\n",
    "------------------------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": true,
        "row": 17,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "use_rank = 100\n",
    "total_data['pos_100'] = rb.rank_build(total_data.res.values, use_rank, groups) / use_rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 9,
        "hidden": true,
        "row": 17,
        "width": 6
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "total_data[['pos_100', 'res', 'dailyReturn', 'd1ret', 'd1ret_b', 'Code', 'Date']].tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "to_look_into = total_data[risk_facto_cols].multiply(total_data.pos_100 - total_data[index_compentns], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 18,
        "hidden": true,
        "row": 17,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "to_look_into.groupby(total_data.Date).sum().max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 18,
        "hidden": true,
        "row": 17,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "to_look_into.groupby(total_data.Date).sum().min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "ret_series = ((total_data.pos_100 - total_data[index_compentns]) * total_data.d1ret).groupby(total_data.Date).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 15,
        "hidden": false,
        "row": 39,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "ret_series.cumsum().plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 16,
        "hidden": false,
        "row": 54,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "ret_series.cumsum()[-30:].plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 5,
        "hidden": true,
        "row": 17,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "total_data.pos_100.groupby(total_data.Date).sum().head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 5,
        "hidden": false,
        "row": 70,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "# Factor Performance (Long Only - Top 100 Uniformly Distributed In Each Sector Equal Weighted)\n",
    "-------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": true,
        "row": 31,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "factor_data_values = total_data[['Date', 'res', '申万一级行业']]\n",
    "\n",
    "def get_percent_pos(x):\n",
    "    res_values = x.res.values\n",
    "    percent = 115. / len(res_values)\n",
    "    ind_values = x['申万一级行业'].values.astype(int)\n",
    "    final_choosed = pb.percent_build(res_values, percent, ind_values)\n",
    "    return pd.Series(final_choosed / final_choosed.sum())\n",
    "\n",
    "total_data['pos_100_uind'] = factor_data_values.groupby('Date').apply(get_percent_pos).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "to_look_into = total_data[risk_facto_cols].multiply(total_data.pos_100_uind - total_data[index_compentns], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 18,
        "hidden": true,
        "row": 31,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "to_look_into.groupby(total_data.Date).sum().max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 8,
        "height": 18,
        "hidden": true,
        "row": 31,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "to_look_into.groupby(total_data.Date).sum().min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 9,
        "hidden": true,
        "row": 39,
        "width": 6
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "total_data[['pos_100_uind', 'res', 'dailyReturn', 'd1ret', 'd1ret_b', 'Code', 'Date']].tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "ret_series = ((total_data.pos_100_uind - total_data[index_compentns]) * total_data.d1ret).groupby(total_data.Date).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 15,
        "hidden": false,
        "row": 75,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "ret_series.cumsum().plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 15,
        "hidden": false,
        "row": 90,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "ret_series.cumsum()[-60:].plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 5,
        "hidden": true,
        "row": 39,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "total_data.pos_100_uind.groupby(total_data.Date).sum().head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": false,
        "row": 105,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "# Factor Performance (Long Only - Match Benchark Sectors)\n",
    "-------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 12,
        "hidden": true,
        "row": 44,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "lbound_exposure = -0.00001 * np.ones(len(risk_facto_cols))\n",
    "ubound_exposure = 0.00001 * np.ones(len(risk_facto_cols))\n",
    "lbound_exposure[-2] = -0.01\n",
    "ubound_exposure[-2] = 0.01\n",
    "\n",
    "def get_benchmark_match_pos(x):\n",
    "    er = x.res.values\n",
    "    bm = x[index_compentns].values\n",
    "    lbound = 0.\n",
    "    ubound = 0.01 + bm\n",
    "    risk_exposure = x[risk_facto_cols].values\n",
    "    \n",
    "    status, value , ret = lb.linear_build(er,\n",
    "                                          lbound=lbound,\n",
    "                                          ubound=ubound,\n",
    "                                          risk_exposure=risk_exposure,\n",
    "                                          bm=bm,\n",
    "                                          risk_target=(lbound_exposure, ubound_exposure),\n",
    "                                          solver=None)\n",
    "    print(x.Date.unique()[0], ': ', status)\n",
    "    \n",
    "    if status != 'optimal':\n",
    "        return pd.Series(np.ones(len(er)) / len(er))\n",
    "    else:\n",
    "        return pd.Series(ret)\n",
    "\n",
    "res = total_data.groupby('Date').apply(get_benchmark_match_pos).values\n",
    "total_data['pos_bmat'] = res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "to_look_into = total_data[risk_facto_cols].multiply(total_data.pos_bmat - total_data[index_compentns], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 18,
        "hidden": true,
        "row": 44,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "to_look_into.groupby(total_data.Date).sum().max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 8,
        "height": 18,
        "hidden": true,
        "row": 44,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "to_look_into.groupby(total_data.Date).sum().min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 9,
        "hidden": true,
        "row": 44,
        "width": 6
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "total_data[['pos_bmat', 'res', 'dailyReturn', 'd1ret', 'd1ret_b', 'Code', 'Date']].tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "ret_series = ((total_data.pos_bmat - total_data[index_compentns]) * total_data.d1ret).groupby(total_data.Date).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 15,
        "hidden": false,
        "row": 109,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "ret_series.cumsum().plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 16,
        "hidden": false,
        "row": 124,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "ret_series.cumsum()[-40:].plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": false,
        "row": 140,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "# Postion Analysis and Comparison\n",
    "----------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "pos_table = total_data[['Date', 'Code', '申万一级行业', 'pos_long_short', 'pos_100', 'pos_100_uind', 'pos_bmat', index_compentns]].copy()\n",
    "pos_table.loc[:, '申万一级行业'] = old_ind_values.values\n",
    "\n",
    "for name in ['pos_100', 'pos_100_uind', 'pos_bmat']:\n",
    "    pos_table.loc[:, name] = pos_table[name] - pos_table[index_compentns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_pos_table = pos_table.groupby(['Date', '申万一级行业']).sum()\n",
    "aggregated_pos_table.reset_index(level=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 15,
        "hidden": false,
        "row": 144,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_pos_table.loc[aggregated_pos_table['申万一级行业'] == '申万家用电器',['pos_long_short', 'pos_100', 'pos_100_uind', 'pos_bmat']].plot(figsize=(16,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 6,
        "hidden": false,
        "row": 174,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_pos_table[['pos_long_short', 'pos_100', 'pos_100_uind', 'pos_bmat']].corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 5,
        "height": 9,
        "hidden": true,
        "row": 74,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "turn_over_table = {}\n",
    "\n",
    "for name in ['pos_long_short', 'pos_100', 'pos_100_uind', 'pos_bmat']:\n",
    "    pos_series = pos_table[['Date', 'Code', name]]\n",
    "    pivot_position = pos_series.pivot_table(name, index='Date', columns='Code').fillna(0.)\n",
    "    turn_over_series = pivot_position.diff().abs().sum(axis=1)\n",
    "    turn_over_table[name] = turn_over_series.values\n",
    "    \n",
    "turn_over_table = pd.DataFrame(turn_over_table, index=pos_table.Date.unique())\n",
    "turn_over_table.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 15,
        "hidden": false,
        "row": 159,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "(turn_over_table.cumsum() * 0.0015).plot(figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": false,
        "row": 180,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "# Risk Exposure (Long Short)\n",
    "---------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "net_weight = total_data.pos_long_short\n",
    "next_bar_return_series = total_data.d1ret\n",
    "risk_table = total_data[risk_facto_cols]\n",
    "net_weight.index = total_data.Date\n",
    "next_bar_return_series.index = total_data.Date\n",
    "risk_table.index = total_data.Date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 4,
        "hidden": true,
        "row": 74,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "explained_table, exposure_table = ra.risk_analysis(net_weight, next_bar_return_series, risk_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_bars = explained_table.groupby(level=0).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 23,
        "hidden": false,
        "row": 184,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "top_sources = aggregated_bars.sum().abs().sort_values(ascending=False).index[:10]\n",
    "aggregated_bars.sum().sort_values(ascending=False).plot(kind='bar', figsize=(16, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 8,
        "hidden": true,
        "row": 74,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].cumsum().plot(figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 15,
        "hidden": false,
        "row": 207,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].plot(figsize=(14, 7))\n",
    "plt.legend(loc='upper center', ncol=int(len(top_sources[1:]) // 3) + 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": false,
        "row": 222,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "# Risk Exposure (Long Only - Top 100)\n",
    "-------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "net_weight = total_data.pos_100 - total_data[index_compentns]\n",
    "net_weight.index = total_data.Date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": true,
        "row": 89,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "explained_table, exposure_table = ra.risk_analysis(net_weight, next_bar_return_series, risk_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_bars = explained_table.groupby(level=0).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 23,
        "hidden": false,
        "row": 226,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "top_sources = aggregated_bars.sum().abs().sort_values(ascending=False).index[:10]\n",
    "aggregated_bars.sum().sort_values(ascending=False).plot(kind='bar', figsize=(16, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 8,
        "height": 8,
        "hidden": true,
        "row": 89,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].cumsum().plot(figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 15,
        "hidden": false,
        "row": 249,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].plot(figsize=(14, 7))\n",
    "plt.legend(loc='upper center', ncol=int(len(top_sources[1:]) // 3) + 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": false,
        "row": 264,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "# Risk Exposure (Long Only - Top 100 Uniformly Distributed)\n",
    "-------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "net_weight = total_data.pos_100_uind - total_data[index_compentns]\n",
    "net_weight.index = total_data.Date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 8,
        "height": 4,
        "hidden": true,
        "row": 89,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "explained_table, exposure_table = ra.risk_analysis(net_weight, next_bar_return_series, risk_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_bars = explained_table.groupby(level=0).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 23,
        "hidden": false,
        "row": 268,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "top_sources = aggregated_bars.sum().abs().sort_values(ascending=False).index[:10]\n",
    "aggregated_bars.sum().sort_values(ascending=False).plot(kind='bar', figsize=(16, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 8,
        "hidden": true,
        "row": 104,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].cumsum().plot(figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 15,
        "hidden": false,
        "row": 291,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].plot(figsize=(14, 7))\n",
    "plt.legend(loc='upper center', ncol=int(len(top_sources[1:]) // 3) + 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 6,
        "hidden": false,
        "row": 306,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "# Risk Exposure (Long Only - Match Benchmark Sectors)\n",
    "--------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "net_weight = total_data.pos_bmat - total_data[index_compentns]\n",
    "net_weight.index = total_data.Date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 8,
        "height": 4,
        "hidden": true,
        "row": 112,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "explained_table, exposure_table = ra.risk_analysis(net_weight, next_bar_return_series, risk_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_bars = explained_table.groupby(level=0).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 23,
        "hidden": false,
        "row": 312,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "top_sources = aggregated_bars.sum().abs().sort_values(ascending=False).index[:10]\n",
    "aggregated_bars.sum().sort_values(ascending=False).plot(kind='bar', figsize=(16, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 8,
        "hidden": true,
        "row": 108,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].cumsum().plot(figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 15,
        "hidden": false,
        "row": 335,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].plot(figsize=(14, 7))\n",
    "plt.legend(loc='upper center', ncol=int(len(top_sources[1:]) // 3) + 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": false,
        "row": 350,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "# Risk Exposure for Historical Position\n",
    "-----------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "total_data2 = df.copy()\n",
    "unique_code = total_data2.Code.unique()\n",
    "unique_date = total_data2.Date.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "hist_data = pd.read_csv('portfolio.csv')\n",
    "hist_data.Date = pd.to_datetime(hist_data.Date.astype('str'), format='%Y%m%d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "def func(x):\n",
    "    x = x.set_index('Code')\n",
    "    return x.ix[unique_code]\n",
    "\n",
    "hist_data = hist_data.groupby('Date').apply(func)['Alpha_Trading']\n",
    "hist_data = hist_data.reset_index()\n",
    "hist_data['Alpha_Trading'] = hist_data['Alpha_Trading'].fillna(0.).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "hist_data = pd.merge(total_data2, hist_data, on=['Date', 'Code'], how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "hist_data = hist_data[hist_data.Date >= '2015-01-09'].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "hist_data['Alpha_Trading'] = hist_data['Alpha_Trading'].groupby(hist_data.Code).fillna(method='pad')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "hist_data.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "net_weight = hist_data.Alpha_Trading - hist_data[index_compentns]\n",
    "next_bar_return_series = hist_data.d1ret\n",
    "risk_table = hist_data[risk_facto_cols]\n",
    "\n",
    "net_weight.index = hist_data.Date\n",
    "risk_table.index = hist_data.Date\n",
    "next_bar_return_series.index = hist_data.Date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "ret_series = (next_bar_return_series * net_weight).groupby(level=0).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 9,
        "hidden": true,
        "row": 130,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "ret_series.cumsum().plot(figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 9,
        "hidden": true,
        "row": 130,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "ret_series.cumsum()[-60:].plot(figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 8,
        "height": 4,
        "hidden": true,
        "row": 130,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "explained_table, exposure_table = ra.risk_analysis(net_weight, next_bar_return_series, risk_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_bars = explained_table.groupby(level=0).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 23,
        "hidden": false,
        "row": 354,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "top_sources = aggregated_bars.sum().abs().sort_values(ascending=False).index[:10]\n",
    "aggregated_bars.sum().sort_values(ascending=False).plot(kind='bar', figsize=(16, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 8,
        "hidden": true,
        "row": 130,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].cumsum().plot(figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 16,
        "hidden": false,
        "row": 377,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].plot(figsize=(14, 7))\n",
    "plt.legend(loc='upper center', ncol=int(len(top_sources[1:]) // 3) + 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 8,
        "height": 4,
        "hidden": true,
        "row": 130,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "source": [
    "# Clean up\n",
    "--------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "extensions": {
   "jupyter_dashboards": {
    "activeView": "grid_default",
    "version": 1,
    "views": {
     "grid_default": {
      "cellMargin": 10,
      "defaultCellHeight": 20,
      "maxColumns": 12,
      "name": "grid",
      "type": "grid"
     },
     "report_default": {
      "name": "report",
      "type": "report"
     }
    }
   }
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
