{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 请在环境变量中设置`DB_URI`指向数据库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "from alphamind.api import *\n",
    "from PyFin.api import *\n",
    "\n",
    "plt.style.use('ggplot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Back test parameter settings\n",
    "\"\"\"\n",
    "\n",
    "start_date = '2017-01-01'\n",
    "end_date = '2018-02-24'\n",
    "\n",
    "frequency = '10b'\n",
    "industry_lower = 1.0\n",
    "industry_upper = 1.0\n",
    "method = 'risk_neutral'\n",
    "neutralize_risk = industry_styles\n",
    "industry_name = 'sw'\n",
    "industry_level = 1\n",
    "benchmark_total_lower = 0.8\n",
    "benchmark_total_upper = 1.0\n",
    "horizon = map_freq(frequency)\n",
    "weight_gap = 0.01\n",
    "benchmark_code = 905\n",
    "universe_name = ['zz800']\n",
    "universe = Universe('custom', universe_name)\n",
    "ref_dates = makeSchedule(start_date, end_date, frequency, 'china.sse')\n",
    "\n",
    "executor = NaiveExecutor()\n",
    "data_source = os.environ['DB_URI']\n",
    "engine = SqlEngine(data_source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Constraints settings\n",
    "\"\"\"\n",
    "\n",
    "industry_names = industry_list(industry_name, industry_level)\n",
    "constraint_risk = ['SIZE', 'SIZENL', 'BETA'] + industry_names\n",
    "total_risk_names = constraint_risk + ['benchmark', 'total']\n",
    "\n",
    "b_type = []\n",
    "l_val = []\n",
    "u_val = []\n",
    "\n",
    "for name in total_risk_names:\n",
    "    if name == 'benchmark':\n",
    "        b_type.append(BoundaryType.RELATIVE)\n",
    "        l_val.append(benchmark_total_lower)\n",
    "        u_val.append(benchmark_total_upper)\n",
    "    elif name in {'SIZE', 'SIZENL', 'BETA'}:\n",
    "        b_type.append(BoundaryType.ABSOLUTE)\n",
    "        l_val.append(0.0)\n",
    "        u_val.append(0.0)\n",
    "    else:\n",
    "        b_type.append(BoundaryType.RELATIVE)\n",
    "        l_val.append(industry_lower)\n",
    "        u_val.append(industry_upper)\n",
    "\n",
    "bounds = create_box_bounds(total_risk_names, b_type, l_val, u_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def factor_analysis(engine, factor_name, universe, benchmark_code, positive):\n",
    "    \n",
    "    \"\"\"\n",
    "    Data phase\n",
    "    \"\"\"\n",
    "    index_return = engine.fetch_dx_return_index_range(benchmark_code, start_date, end_date, horizon=horizon,\n",
    "                                                  offset=1).set_index('trade_date')\n",
    "\n",
    "    codes_return = engine.fetch_dx_return_range(universe,\n",
    "                                                dates=ref_dates,\n",
    "                                                horizon=horizon,\n",
    "                                                offset=1)\n",
    "\n",
    "    return_groups = codes_return.groupby('trade_date')\n",
    "    \n",
    "    \"\"\"\n",
    "    Model phase: we need 1 constant linear model and one linear regression model\n",
    "    \"\"\"\n",
    "    industry_total = engine.fetch_industry_matrix_range(universe, dates=ref_dates, category=industry_name, level=industry_level)\n",
    "    industry_groups = industry_total.groupby('trade_date')\n",
    "    \n",
    "    alpha_name = [str(factor_name) + '_' + ('pos' if positive else 'neg')]\n",
    "    simple_expression = CSRes(LAST(factor_name), 'roe_q') if positive else -CSRes(LAST(factor_name), 'roe_q')\n",
    "\n",
    "    const_features = {alpha_name[0]: simple_expression}\n",
    "    const_weights = {alpha_name[0]: 1.}\n",
    "\n",
    "    const_model = ConstLinearModel(features=alpha_name,\n",
    "                                   weights=const_weights)\n",
    "\n",
    "    const_model_factor_data = engine.fetch_data_range(universe,\n",
    "                                                      const_features,\n",
    "                                                      dates=ref_dates,\n",
    "                                                      benchmark=benchmark_code)['factor'].dropna()\n",
    "\n",
    "    rets = []\n",
    "    turn_overs = []\n",
    "    leverags = []\n",
    "    ics = []\n",
    "    index_dates = []\n",
    "    factor_groups = const_model_factor_data.groupby('trade_date')\n",
    "\n",
    "    for i, value in enumerate(factor_groups):\n",
    "        date = value[0]\n",
    "        data = value[1]\n",
    "        index_dates.append(date)\n",
    "        \n",
    "        industry_matrix = industry_groups.get_group(date)\n",
    "        total_data = data.fillna(data[alpha_name].median())\n",
    "        total_data = pd.merge(total_data, industry_matrix, on=['code'])\n",
    "        alpha_logger.info('{0}: {1}'.format(date, len(total_data)))\n",
    "        risk_exp = total_data[neutralize_risk].values.astype(float)\n",
    "        benchmark_w = total_data.weight.values\n",
    "        is_in_benchmark = (benchmark_w > 0.).astype(float).reshape(-1, 1)\n",
    "\n",
    "        constraint_exp = total_data[constraint_risk].values\n",
    "        risk_exp_expand = np.concatenate((constraint_exp,\n",
    "                                          is_in_benchmark,\n",
    "                                          np.ones_like(is_in_benchmark)), axis=1).astype(float)\n",
    "        total_risk_exp = pd.DataFrame(risk_exp_expand, columns=total_risk_names)\n",
    "        constraints = LinearConstraints(bounds, total_risk_exp, benchmark_w)\n",
    "\n",
    "        lbound = np.maximum(0., benchmark_w - weight_gap)\n",
    "        ubound = weight_gap + benchmark_w\n",
    "\n",
    "        factor_values = factor_processing(total_data[alpha_name].values,\n",
    "                                          pre_process=[winsorize_normal, standardize],\n",
    "                                          risk_factors=risk_exp,\n",
    "                                          post_process=[winsorize_normal, standardize])\n",
    "\n",
    "        # const linear model\n",
    "        er = const_model.predict(pd.DataFrame(data={alpha_name[0]: factor_values.flatten()}))\n",
    "\n",
    "        alpha_logger.info('{0} full re-balance'.format(date))\n",
    "        target_pos, _ = er_portfolio_analysis(er,\n",
    "                                              total_data.industry_name.values,\n",
    "                                              None,\n",
    "                                              constraints,\n",
    "                                              False,\n",
    "                                              benchmark_w,\n",
    "                                              method=method,\n",
    "                                              lbound=lbound,\n",
    "                                              ubound=ubound)\n",
    "\n",
    "        target_pos['code'] = total_data['code'].values\n",
    "\n",
    "        turn_over, executed_pos = executor.execute(target_pos=target_pos)\n",
    "        dx_returns = return_groups.get_group(date)\n",
    "\n",
    "        result = pd.merge(executed_pos, total_data[['code', 'weight']], on=['code'], how='inner')\n",
    "        result = pd.merge(result, dx_returns, on=['code'])\n",
    "\n",
    "        leverage = result.weight_x.abs().sum()\n",
    "\n",
    "        excess_return = np.exp(result.dx.values) - 1. - index_return.loc[date, 'dx']\n",
    "        raw_weight = result.weight_x.values\n",
    "        activate_weight = raw_weight - result.weight_y.values\n",
    "        ret = raw_weight @ excess_return\n",
    "        risk_adjusted_ic = np.corrcoef(excess_return, activate_weight)[0, 1]\n",
    "        rets.append(np.log(1. + ret))\n",
    "        ics.append(risk_adjusted_ic)\n",
    "        executor.set_current(executed_pos)\n",
    "        turn_overs.append(turn_over)\n",
    "        leverags.append(leverage)\n",
    "\n",
    "        alpha_logger.info('{0} is finished'.format(date))\n",
    "\n",
    "    ret_df = pd.DataFrame({'returns': rets, 'turn_over': turn_overs, 'IC': ics, 'leverage': leverags}, index=index_dates)\n",
    "\n",
    "    ret_df.loc[advanceDateByCalendar('china.sse', ref_dates[-1], frequency)] = 0.\n",
    "    ret_df = ret_df.shift(1)\n",
    "    ret_df.iloc[0] = 0.\n",
    "    ret_df['tc_cost'] = ret_df.turn_over * 0.002\n",
    "\n",
    "    return alpha_name[0], ret_df\n",
    "\n",
    "def worker_func_positive(factor_name):\n",
    "    from alphamind.api import SqlEngine\n",
    "    engine = SqlEngine(data_source)\n",
    "    return factor_analysis(engine, factor_name, universe, benchmark_code, positive=True)\n",
    "\n",
    "\n",
    "def worker_func_negative(factor_name):\n",
    "    from alphamind.api import SqlEngine\n",
    "    engine = SqlEngine(data_source)\n",
    "    return factor_analysis(engine, factor_name, universe, benchmark_code, positive=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "factors = ['EPS', 'ROE']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "res1 = [worker_func_positive(factor) for factor in factors]\n",
    "res2 = [worker_func_negative(factor) for factor in factors]\n",
    "\n",
    "factor_df = pd.DataFrame()\n",
    "ic_df = pd.DataFrame()\n",
    "\n",
    "for f_name, res in res1:\n",
    "    factor_df[f_name] = res['returns']\n",
    "    ic_df[f_name] = res['IC']\n",
    "\n",
    "for f_name, res in res2:\n",
    "    factor_df[f_name] = res['returns']\n",
    "    ic_df[f_name] = res['IC']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "factor_res = factor_df.agg(['mean', 'std']).T\n",
    "factor_res['t.'] = factor_res['mean'] / factor_res['std'] * np.sqrt(len(factor_df))\n",
    "\n",
    "ic_res = ic_df.agg(['mean', 'std']).T\n",
    "ic_res['t.'] = ic_res['mean'] / ic_res['std'] * np.sqrt(len(ic_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with pd.ExcelWriter(f'{universe_name[0]}_{benchmark_code}.xlsx', engine='xlsxwriter') as writer:\n",
    "    factor_df.to_excel(writer, sheet_name='ret')\n",
    "    ic_df.to_excel(writer, sheet_name='ic')\n",
    "    factor_res.to_excel(writer, sheet_name='ret_stat')\n",
    "    ic_res.to_excel(writer, sheet_name='ic_stat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
