{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%pylab inline\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('ggplot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sqlalchemy\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "factor_table = 'prod_500'\n",
    "risk_table = 'risk_factor_500'\n",
    "trade_table = 'trade_data'\n",
    "index_components = '500Weight'\n",
    "benchmark = 'zz500'\n",
    "\n",
    "prod_factors = ['VAL', 'BDTO', 'RVOL', 'CFinc1', 'CHV']\n",
    "\n",
    "factor_list = ','.join([factor_table + '.' + f for f in prod_factors])\n",
    "\n",
    "server = '10.63.6.176'\n",
    "user = 'sa'\n",
    "pwd = 'we083826'\n",
    "\n",
    "engine = sqlalchemy.create_engine('mysql+pymysql://{0}:{1}@{2}/multifactor?charset=utf8'.format(user, pwd, server))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sql_template1 = 'select {factor_list},{trade_table}.Return as dailyReturn, {risk_table}.* ' \\\n",
    "               'from {factor_table}, trade_data, {risk_table} ' \\\n",
    "               'where {factor_table}.Date = trade_data.Date and {factor_table}.Code = trade_data.Code ' \\\n",
    "               'and {factor_table}.Date = {risk_table}.Date and {factor_table}.Code = {risk_table}.Code;'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "sql1 = sql_template1.format(factor_list=factor_list,\n",
    "                            factor_table=factor_table,\n",
    "                            risk_table=risk_table,\n",
    "                            trade_table=trade_table)\n",
    "\n",
    "sql2 = 'select * from factor_data'\n",
    "\n",
    "# factor_df1 = pd.read_sql(sql1, engine)\n",
    "# common_factor_df = pd.read_sql(sql2, engine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "factor_df = pd.read_hdf('factor_data.hdf', 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "index_components_df = pd.read_sql('select Date, Code, {0} from index_components;'.format(index_components), engine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del factor_df['Bank']\n",
    "del factor_df['NonBankFinancial']\n",
    "factor_df.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "raw_df = pd.merge(factor_df, index_components_df, on=['Date', 'Code'], how='left')\n",
    "raw_df.fillna(0, inplace=True)\n",
    "raw_df[index_components] = raw_df[index_components] / 100."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "risk_factor_cols = raw_df.columns[8:35]\n",
    "ob_risk_factor_cols = risk_factor_cols.copy()\n",
    "risk_factor_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "common_factor = raw_df.columns[38:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = raw_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['d1ret'] = df.dailyReturn.groupby(df.Code).shift(-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "benchmark_data = pd.read_sql('select {0}, Date from index_data'.format(benchmark), engine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "benchmark_data['ret'] = benchmark_data[benchmark] / benchmark_data[benchmark].shift(1) - 1.\n",
    "benchmark_data['d1ret_b'] = benchmark_data['ret'] .shift(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.merge(df, benchmark_data[['Date', 'd1ret_b']], on='Date', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.dropna(inplace=True)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate Market Risk\n",
    "--------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "engine = sqlalchemy.create_engine('mysql+pymysql://sa:we083826@10.63.6.176/multifactor?charset=utf8')\n",
    "\n",
    "window = 250\n",
    "return_data = pd.read_sql('select Date, Code, Price, `Return` from trade_data', engine)\n",
    "return_pivot = return_data.pivot_table('Price', 'Date', 'Code')\n",
    "return_pivot.fillna(method='pad', inplace=True)\n",
    "return_pivot = return_pivot / return_pivot.shift(1) - 1.\n",
    "index_data = benchmark_data.set_index('Date')\n",
    "\n",
    "full_df = pd.merge(return_pivot, index_data, left_index=True, right_index=True)\n",
    "del full_df['zz500']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "corr_table = full_df.rolling(window=window).corr(full_df['ret'])\n",
    "std_table = full_df.rolling(window=window).std()\n",
    "\n",
    "beta_table = std_table.copy()\n",
    "for col in full_df.columns:\n",
    "    beta_table[col] = corr_table[col] * std_table[col] / std_table['ret']\n",
    "beta_table = beta_table.stack().reset_index()\n",
    "\n",
    "beta_table.columns = ['Date', 'Code', 'market']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "beta_table.to_hdf('factor_data.hdf', 'beta')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.merge(df, beta_table, on=['Date', 'Code'], how='inner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "analysis_factors = prod_factors + ['Size']\n",
    "analysis_factors_weights = np.array([3., 1., 1., 0.5, 0.5, 3.])\n",
    "\n",
    "ob_risk_factor_cols = ob_risk_factor_cols.append(pd.Index(['market']))\n",
    "risk_factor_cols = ob_risk_factor_cols.difference(analysis_factors)\n",
    "risk_factor_cols = risk_factor_cols.unique()\n",
    "ob_risk_factor_cols = ob_risk_factor_cols.unique()\n",
    "risk_factor_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ob_risk_factor_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# to transform industry codes to int variable\n",
    "old_ind_values = df['申万一级行业'].copy()\n",
    "\n",
    "ind_list = df['申万一级行业'].unique()\n",
    "ind_dict = {}\n",
    "for i, ind in enumerate(ind_list):\n",
    "    ind_dict[ind] = i\n",
    "\n",
    "df['申万一级行业'].replace(ind_dict, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Factor Date Preprocessing (Winsorize -> Standardize -> neutralize)\n",
    "---------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import alphamind.data.neutralize as ne\n",
    "import alphamind.data.winsorize as ws\n",
    "import alphamind.data.standardize as st"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "total_data = df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y = total_data[common_factor].values\n",
    "groups = total_data.Date.values.astype(np.int)\n",
    "\n",
    "total_data[common_factor] = st.standardize(ws.winsorize_normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y = total_data[analysis_factors].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "neutralized_values = ne.neutralize(total_data[risk_factor_cols].values,\n",
    "                                   st.standardize(ws.winsorize_normal(y, groups=groups),\n",
    "                                                  groups=groups),\n",
    "                                   groups)\n",
    "\n",
    "total_data['res'] = neutralized_values @ analysis_factors_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "total_data.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Factor Performance (long_short)\n",
    "------------------------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "total_data['pos_long_short'] = total_data.res.groupby(groups).apply(lambda x: x / np.abs(x).sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "total_data[['pos_long_short', 'res', 'dailyReturn', 'd1ret', 'd1ret_b', 'Code', 'Date']].tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "to_look_into = total_data[ob_risk_factor_cols].multiply(total_data.pos_long_short, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "to_look_into.groupby(total_data.Date).sum()['Size'].plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "to_look_into.groupby(total_data.Date).sum().min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_series = (total_data.pos_long_short * total_data.d1ret).groupby(total_data.Date).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_series.cumsum().plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_series[-60:].cumsum().plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "total_data.pos_long_short.groupby(total_data.Date).apply(lambda x: x.sum()).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Factor Performance (Long Only - Top 100 Equal Weighted)\n",
    "------------------------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import alphamind.portfolio.rankbuilder as rb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "use_rank = 100\n",
    "total_data['pos_100'] = rb.rank_build(total_data.res.values, use_rank, groups) / use_rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "total_data[['pos_100', 'res', 'dailyReturn', 'd1ret', 'd1ret_b', 'Code', 'Date']].tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "to_look_into = total_data[ob_risk_factor_cols].multiply(total_data.pos_100 - total_data[index_components], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "to_look_into.groupby(total_data.Date).sum()['Size'].plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "to_look_into.groupby(total_data.Date).sum().min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_series = ((total_data.pos_100 - total_data[index_components]) * total_data.d1ret).groupby(total_data.Date).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_series.cumsum().plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_series[-60:].cumsum().plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "total_data.pos_100.groupby(total_data.Date).sum().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_pos_100 = ret_series.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Factor Performance (Long Only - Top 100 Uniformly Distributed In Each Sector Equal Weighted)\n",
    "-------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import alphamind.portfolio.percentbuilder as pb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "factor_data_values = total_data[['Date', 'res', '申万一级行业']]\n",
    "\n",
    "def get_percent_pos(x):\n",
    "    res_values = x.res.values\n",
    "    percent = 115. / len(res_values)\n",
    "    ind_values = x['申万一级行业'].values.astype(int)\n",
    "    final_choosed = pb.percent_build(res_values, percent, ind_values)\n",
    "    return pd.Series(final_choosed / final_choosed.sum())\n",
    "\n",
    "total_data['pos_100_uind'] = factor_data_values.groupby('Date').apply(get_percent_pos).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(total_data['pos_100_uind'] > 0).groupby(total_data.Date).sum().plot(figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "to_look_into = total_data[ob_risk_factor_cols].multiply(total_data.pos_100_uind - total_data[index_components], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "to_look_into.groupby(total_data.Date).sum()['Size'].plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "to_look_into.groupby(total_data.Date).sum().min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "total_data[['pos_100_uind', 'res', 'dailyReturn', 'd1ret', 'd1ret_b', 'Code', 'Date']].tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_series = ((total_data.pos_100_uind - total_data[index_components]) * total_data.d1ret).groupby(total_data.Date).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_series.cumsum().plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_series[-60:].cumsum().plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "total_data.pos_100_uind.groupby(total_data.Date).sum().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_100_uind = ret_series.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Factor Performance (Long Only - Match Benchark Sectors)\n",
    "-------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import alphamind.portfolio.linearbuilder as lb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "lbound_exposure = -1e-3 * np.ones(len(risk_factor_cols))\n",
    "ubound_exposure = 1e-3 * np.ones(len(risk_factor_cols))\n",
    "\n",
    "def get_benchmark_match_pos(x):\n",
    "    er = x.res.values\n",
    "    bm = x[index_components].values\n",
    "    lbound = 0.\n",
    "    ubound = 0.01 #+ bm\n",
    "    risk_exposure = x[risk_factor_cols].values\n",
    "    \n",
    "    status, value , ret = lb.linear_build(er,\n",
    "                                          lbound=lbound,\n",
    "                                          ubound=ubound,\n",
    "                                          risk_exposure=risk_exposure,\n",
    "                                          bm=bm,\n",
    "                                          risk_target=(lbound_exposure, ubound_exposure),\n",
    "                                          solver=None)\n",
    "    print(x.Date.unique()[0], ': ', status)\n",
    "    \n",
    "    if status != 'optimal':\n",
    "        return pd.Series(np.ones(len(er)) / len(er))\n",
    "    else:\n",
    "        return pd.Series(ret)\n",
    "\n",
    "res = total_data.groupby('Date').apply(get_benchmark_match_pos).values\n",
    "total_data['pos_bmat'] = res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(total_data['pos_bmat'] > 1e-4).groupby(total_data.Date).sum().plot(ylim=(50, 250), figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "to_look_into = total_data[ob_risk_factor_cols].multiply(total_data.pos_bmat - total_data[index_components], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "to_look_into.groupby(total_data.Date).sum()['Size'].plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "to_look_into.groupby(total_data.Date).sum().min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "total_data[['pos_bmat', 'res', 'dailyReturn', 'd1ret', 'd1ret_b', 'Code', 'Date']].tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_series = ((total_data.pos_bmat - total_data[index_components]) * total_data.d1ret).groupby(total_data.Date).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_series.cumsum().plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_series[-60:].cumsum().plot(figsize=(14,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_bmat = ret_series.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance Analysis and Comparison\n",
    "------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_mat = pd.DataFrame({'pos_100': ret_pos_100,\n",
    "                        'pos_100_uind': ret_100_uind,\n",
    "                        'pos_bmat': ret_bmat})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sharp_mat = ret_mat.rolling(window=250).mean() / ret_mat.rolling(window=250).std() * np.sqrt(250)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sharp_mat.plot(figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Postion Analysis and Comparison\n",
    "----------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pos_table = total_data[['Date', 'Code', '申万一级行业', 'pos_long_short', 'pos_100', 'pos_100_uind', 'pos_bmat', index_components]].copy()\n",
    "pos_table.loc[:, '申万一级行业'] = old_ind_values.values\n",
    "\n",
    "for name in ['pos_100', 'pos_100_uind', 'pos_bmat']:\n",
    "    pos_table.loc[:, name] = pos_table[name] - pos_table[index_components]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aggregated_pos_table = pos_table.groupby(['Date', '申万一级行业']).sum()\n",
    "aggregated_pos_table.reset_index(level=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aggregated_pos_table.loc[aggregated_pos_table['申万一级行业'] == '申万化工',['pos_long_short', 'pos_100', 'pos_100_uind', 'pos_bmat']].plot(figsize=(16,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aggregated_pos_table[['pos_long_short', 'pos_100', 'pos_100_uind', 'pos_bmat']].corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "turn_over_table = {}\n",
    "\n",
    "for name in ['pos_long_short', 'pos_100', 'pos_100_uind', 'pos_bmat']:\n",
    "    pos_series = pos_table[['Date', 'Code', name]]\n",
    "    pivot_position = pos_series.pivot_table(name, index='Date', columns='Code').fillna(0.)\n",
    "    turn_over_series = pivot_position.diff().abs().sum(axis=1)\n",
    "    turn_over_table[name] = turn_over_series.values\n",
    "    \n",
    "turn_over_table = pd.DataFrame(turn_over_table, index=pos_table.Date.unique())\n",
    "turn_over_table.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "turn_over_table.plot(figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(turn_over_table * 0.0015).plot(figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(turn_over_table[-60:].cumsum() * 0.0015).plot(figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Risk Exposure (Long Short)\n",
    "---------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import alphamind.analysis.riskanalysis as ra"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "net_weight = total_data.pos_long_short\n",
    "next_bar_return_series = total_data.d1ret\n",
    "risk_table = total_data[ob_risk_factor_cols]\n",
    "net_weight.index = total_data.Date\n",
    "next_bar_return_series.index = total_data.Date\n",
    "risk_table.index = total_data.Date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "explained_table, exposure_table = ra.risk_analysis(net_weight, next_bar_return_series, risk_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aggregated_bars = explained_table.groupby(level=0).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "top_sources = aggregated_bars.sum().abs().sort_values(ascending=False).index[:10]\n",
    "aggregated_bars.sum().sort_values(ascending=False).plot(kind='bar', figsize=(16, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].cumsum().plot(figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].plot(figsize=(14, 7))\n",
    "plt.legend(loc='upper center', ncol=int(len(top_sources[1:]) // 3) + 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Risk Exposure (Long Only - Top 100)\n",
    "-------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "net_weight = total_data.pos_100 - total_data[index_components]\n",
    "net_weight.index = total_data.Date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "explained_table, exposure_table = ra.risk_analysis(net_weight, next_bar_return_series, risk_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aggregated_bars = explained_table.groupby(level=0).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "top_sources = aggregated_bars.sum().abs().sort_values(ascending=False).index[:10]\n",
    "aggregated_bars.sum().sort_values(ascending=False).plot(kind='bar', figsize=(16, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].cumsum().plot(figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].plot(figsize=(14, 7))\n",
    "plt.legend(loc='upper center', ncol=int(len(top_sources[1:]) // 3) + 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Risk Exposure (Long Only - Top 100 Uniformly Distributed)\n",
    "-------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "net_weight = total_data.pos_100_uind - total_data[index_components]\n",
    "net_weight.index = total_data.Date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "explained_table, exposure_table = ra.risk_analysis(net_weight, next_bar_return_series, risk_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aggregated_bars = explained_table.groupby(level=0).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "top_sources = aggregated_bars.sum().abs().sort_values(ascending=False).index[:10]\n",
    "aggregated_bars.sum().sort_values(ascending=False).plot(kind='bar', figsize=(16, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].cumsum().plot(figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].plot(figsize=(14, 7))\n",
    "plt.legend(loc='upper center', ncol=int(len(top_sources[1:]) // 3) + 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Risk Exposure (Long Only - Match Benchmark Sectors)\n",
    "--------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "net_weight = total_data.pos_bmat - total_data[index_components]\n",
    "net_weight.index = total_data.Date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "explained_table, exposure_table = ra.risk_analysis(net_weight, next_bar_return_series, risk_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aggregated_bars = explained_table.groupby(level=0).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "top_sources = aggregated_bars.sum().abs().sort_values(ascending=False).index[:10]\n",
    "aggregated_bars.sum().sort_values(ascending=False).plot(kind='bar', figsize=(16, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].cumsum().plot(figsize=(14, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aggregated_bars[top_sources.difference(['idiosyncratic'])].plot(figsize=(14, 7))\n",
    "plt.legend(loc='upper center', ncol=int(len(top_sources[1:]) // 3) + 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
