{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "from alphamind.api import *\n",
    "from PyFin.api import *\n",
    "from PyFin.Math.Accumulators.StatefulAccumulators import MovingAverage\n",
    "from PyFin.Math.Accumulators.StatefulAccumulators import MovingSharp\n",
    "from PyFin.Math.Accumulators.StatefulAccumulators import MovingMaxDrawdown\n",
    "\n",
    "plt.style.use('ggplot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Back test parameter settings\n",
    "\"\"\"\n",
    "\n",
    "start_date = '2016-01-01'\n",
    "end_date = '2018-02-24'\n",
    "\n",
    "freq = '10b'\n",
    "industry_lower = 1.0\n",
    "industry_upper = 1.0\n",
    "neutralized_risk = industry_styles\n",
    "industry_name = 'sw'\n",
    "industry_level = 1\n",
    "turn_over_target_base = 0.4\n",
    "benchmark_total_lower = 0.8\n",
    "benchmark_total_upper = 1.0\n",
    "batch = 0\n",
    "horizon = map_freq(freq)\n",
    "universe = Universe(\"custom\", ['zz800'])\n",
    "data_source = 'postgres+psycopg2://postgres:we083826@localhost/alpha'\n",
    "benchmark_code = 905\n",
    "method = 'tv'\n",
    "target_vol = 0.05\n",
    "risk_model = 'short'\n",
    "\n",
    "executor = NaiveExecutor()\n",
    "ref_dates = makeSchedule(start_date, end_date, freq, 'china.sse')\n",
    "engine = SqlEngine(data_source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Factor Model\n",
    "\"\"\"\n",
    "\n",
    "alpha_factors = {\n",
    "    'f01': LAST('CFO2EV'),\n",
    "    'f02': LAST('EPS'),\n",
    "    }\n",
    "\n",
    "\n",
    "weights = dict(f01=1.,\n",
    "               f02=1.)\n",
    "\n",
    "alpha_model = ConstLinearModel(features=alpha_factors, weights=weights)\n",
    "\n",
    "def predict_worker(params):\n",
    "    data_meta = DataMeta(freq=freq,\n",
    "                         universe=universe,\n",
    "                         batch=batch,\n",
    "                         neutralized_risk=neutralized_risk,\n",
    "                         risk_model='short',\n",
    "                         pre_process=[winsorize_normal, standardize],\n",
    "                         post_process=[winsorize_normal, standardize],\n",
    "                         warm_start=0,\n",
    "                         data_source=data_source)\n",
    "    ref_date, model = params\n",
    "    er = predict_by_model(ref_date, model, data_meta)\n",
    "    return er"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "\"\"\"\n",
    "Predicting Phase\n",
    "\"\"\"\n",
    "\n",
    "predicts = [predict_worker((d.strftime('%Y-%m-%d'), alpha_model)) for d in ref_dates]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Shared Data\n",
    "\"\"\"\n",
    "\n",
    "industry_names = industry_list(industry_name, industry_level)\n",
    "constraint_risk = ['SIZE', 'SIZENL', 'BETA'] + industry_names\n",
    "total_risk_names = constraint_risk + ['benchmark', 'total']\n",
    "all_styles = risk_styles + industry_styles + macro_styles\n",
    "\n",
    "b_type = []\n",
    "l_val = []\n",
    "u_val = []\n",
    "\n",
    "for name in total_risk_names:\n",
    "    if name == 'benchmark':\n",
    "        b_type.append(BoundaryType.RELATIVE)\n",
    "        l_val.append(benchmark_total_lower)\n",
    "        u_val.append(benchmark_total_upper)\n",
    "    elif name in {'SIZE', 'SIZENL', 'BETA', 'total'}:\n",
    "        b_type.append(BoundaryType.ABSOLUTE)\n",
    "        l_val.append(0.0)\n",
    "        u_val.append(0.0)\n",
    "    else:\n",
    "        b_type.append(BoundaryType.RELATIVE)\n",
    "        l_val.append(industry_lower)\n",
    "        u_val.append(industry_upper)\n",
    "        \n",
    "bounds = create_box_bounds(total_risk_names, b_type, l_val, u_val)\n",
    "industry_total = engine.fetch_industry_matrix_range(universe, dates=ref_dates, category=industry_name, level=industry_level)\n",
    "benchmark_total = engine.fetch_benchmark_range(dates=ref_dates, benchmark=benchmark_code)\n",
    "risk_cov_total, risk_exposure_total = engine.fetch_risk_model_range(universe, dates=ref_dates, risk_model=risk_model)\n",
    "index_return = engine.fetch_dx_return_index_range(benchmark_code, start_date, end_date, horizon=horizon, offset=1).set_index('trade_date')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rebalance\n",
    "\n",
    "def create_scenario(weight_gap, target_vol, method):\n",
    "\n",
    "    previous_pos = pd.DataFrame()\n",
    "    rets = []\n",
    "    turn_overs = []\n",
    "    leverags = []\n",
    "\n",
    "    for i, ref_date in enumerate(ref_dates):\n",
    "        ref_date = ref_date.strftime('%Y-%m-%d')\n",
    "        industry_matrix = industry_total[industry_total.trade_date == ref_date]\n",
    "        benchmark_w = benchmark_total[benchmark_total.trade_date == ref_date]\n",
    "        risk_exposure = risk_exposure_total[risk_exposure_total.trade_date == ref_date]\n",
    "        risk_cov = risk_cov_total[risk_cov_total.trade_date == ref_date]\n",
    "\n",
    "        res = pd.merge(industry_matrix, benchmark_w, on=['code'], how='left').fillna(0.)\n",
    "        res = pd.merge(res, risk_exposure, on=['code'])\n",
    "        res = res.dropna()\n",
    "        codes = res.code.values.tolist()\n",
    "        \n",
    "        risk_exposure = res[all_styles].values\n",
    "        risk_cov = risk_cov[all_styles].values\n",
    "        special_risk = res.srisk.values\n",
    "        sec_cov = risk_exposure @ risk_cov @ risk_exposure.T / 10000 + np.diag(special_risk ** 2) / 10000\n",
    "\n",
    "        benchmark_w = res.weight.values\n",
    "        is_in_benchmark = (benchmark_w > 0.).astype(float).reshape((-1, 1))\n",
    "\n",
    "        total_risk_exp = np.concatenate([res[constraint_risk].values.astype(float),\n",
    "                                         is_in_benchmark,\n",
    "                                         np.ones_like(is_in_benchmark)],\n",
    "                                        axis=1)\n",
    "        total_risk_exp = pd.DataFrame(total_risk_exp, columns=total_risk_names)\n",
    "        constraints = LinearConstraints(bounds, total_risk_exp, benchmark_w)\n",
    "\n",
    "        lbound = np.maximum(0., benchmark_w - weight_gap)  # np.zeros(len(total_data))\n",
    "        ubound = weight_gap + benchmark_w\n",
    "\n",
    "        if previous_pos.empty:\n",
    "            current_position = None\n",
    "            turn_over_target = None\n",
    "        else:\n",
    "            previous_pos.set_index('code', inplace=True)\n",
    "            remained_pos = previous_pos.loc[codes]\n",
    "\n",
    "            remained_pos.fillna(0., inplace=True)\n",
    "            turn_over_target = turn_over_target_base\n",
    "            current_position = remained_pos.weight.values\n",
    "\n",
    "        er = predicts[i].loc[codes].values\n",
    "\n",
    "        try:\n",
    "            target_pos, _ = er_portfolio_analysis(er,\n",
    "                                                  industry_matrix.industry_name.values,\n",
    "                                                  None,\n",
    "                                                  constraints,\n",
    "                                                  False,\n",
    "                                                  benchmark_w,\n",
    "                                                  method=method,\n",
    "                                                  turn_over_target=turn_over_target,\n",
    "                                                  current_position=current_position,\n",
    "                                                  lbound=lbound,\n",
    "                                                  ubound=ubound,\n",
    "                                                  target_vol=target_vol,\n",
    "                                                  cov=sec_cov)\n",
    "        except ValueError:\n",
    "            target_pos, _ = er_portfolio_analysis(er,\n",
    "                                                  industry_matrix.industry_name.values,\n",
    "                                                  None,\n",
    "                                                  constraints,\n",
    "                                                  False,\n",
    "                                                  benchmark_w,\n",
    "                                                  method=method,\n",
    "                                                  lbound=lbound,\n",
    "                                                  ubound=ubound,\n",
    "                                                  target_vol=target_vol,\n",
    "                                                  cov=sec_cov)\n",
    "\n",
    "        target_pos['code'] = codes\n",
    "        turn_over, executed_pos = executor.execute(target_pos=target_pos)\n",
    "\n",
    "        executed_codes = executed_pos.code.tolist()\n",
    "        dx_returns = engine.fetch_dx_return(ref_date, executed_codes, horizon=horizon, offset=1)\n",
    "        result = pd.merge(executed_pos, dx_returns, on=['code'])\n",
    "\n",
    "        leverage = result.weight.abs().sum()\n",
    "\n",
    "        ret = result.weight.values @ (np.exp(result.dx.values) - 1.)\n",
    "        rets.append(np.log(1. + ret))\n",
    "        executor.set_current(executed_pos)\n",
    "        turn_overs.append(turn_over)\n",
    "        leverags.append(leverage)\n",
    "\n",
    "        previous_pos = executed_pos\n",
    "    \n",
    "    ret_df = pd.DataFrame({'returns': rets, 'turn_over': turn_overs, 'leverage': leverags}, index=ref_dates)\n",
    "\n",
    "    # index return\n",
    "    \n",
    "    ret_df['index'] = index_return['dx']\n",
    "\n",
    "    ret_df.loc[advanceDateByCalendar('china.sse', ref_dates[-1], freq)] = 0.\n",
    "    ret_df = ret_df.shift(1)\n",
    "    ret_df.iloc[0] = 0.\n",
    "    ret_df['tc_cost'] = ret_df.turn_over * 0.002\n",
    "    ret_df['ret_after_tc'] = ret_df['returns'] - ret_df['tc_cost'] - ret_df['index'] * ret_df['leverage']\n",
    "    return ret_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_report(ret_df, windows):\n",
    "    sharp_calc = MovingSharp(windows, x='ret', y='riskFree')\n",
    "    drawdown_calc = MovingMaxDrawdown(windows, x='ret')\n",
    "    max_drawdown_calc = MovingMaxDrawdown(len(ret_df), x='ret')\n",
    "\n",
    "    res_df = pd.DataFrame(columns=['daily_return', 'cum_ret', 'sharp', 'drawdown', 'max_drawn', 'leverage'])\n",
    "    total_returns = 0.\n",
    "\n",
    "    for i, ret in enumerate(ret_df['ret_after_tc']):\n",
    "        date = ret_df.index[i]\n",
    "        total_returns += ret\n",
    "        sharp_calc.push({'ret': ret, 'riskFree': 0.})\n",
    "        drawdown_calc.push({'ret': ret})\n",
    "        max_drawdown_calc.push({'ret': ret})\n",
    "\n",
    "        res_df.loc[date, 'daily_return'] = ret\n",
    "        res_df.loc[date, 'cum_ret'] = total_returns\n",
    "        res_df.loc[date, 'drawdown'] = drawdown_calc.result()\n",
    "        res_df.loc[date, 'max_drawn'] = max_drawdown_calc.result()\n",
    "        res_df.loc[date, 'leverage'] = ret_df.loc[date, 'leverage']\n",
    "\n",
    "        if i < 5:\n",
    "            res_df.loc[date, 'sharp'] = 0.\n",
    "        else:\n",
    "            res_df.loc[date, 'sharp'] = sharp_calc.result() * np.sqrt(windows)\n",
    "    return res_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_gaps = [0.005, 0.010, 0.015, 0.020]\n",
    "\n",
    "with pd.ExcelWriter(f'{universe.base_universe[0]}_{benchmark_code}_gap.xlsx', engine='xlsxwriter') as writer:\n",
    "    for i, weight_gap in enumerate(weight_gaps):\n",
    "        ret_df = create_scenario(weight_gap, target_vol=0.01, method='risk_neutral')\n",
    "        res_df = create_report(ret_df, 25)\n",
    "        res_df.to_excel(writer, sheet_name=f'{i}')\n",
    "        alpha_logger.info(f\"weight_gap: {weight_gap} finished\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_vols = [0.015, 0.030, 0.045, 0.060]\n",
    "\n",
    "with pd.ExcelWriter(f'{universe.base_universe[0]}_{benchmark_code}_tv.xlsx', engine='xlsxwriter') as writer:\n",
    "    for i, target_vol in enumerate(target_vols):\n",
    "        ret_df = create_scenario(weight_gap=0.02, target_vol=target_vol, method='tv')\n",
    "        res_df = create_report(ret_df, 25)\n",
    "        res_df.to_excel(writer, sheet_name=f'{i}')\n",
    "        alpha_logger.info(f\"target_vol: {target_vol:.4f} finished\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
