{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import xgboost as xgb\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from alphamind.api import *\n",
    "from PyFin.api import *\n",
    "\n",
    "engine = SqlEngine()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start_date = '2012-01-01'\n",
    "end_date = '2018-01-05'\n",
    "\n",
    "features = ['roe_q',\n",
    "            'ep_q',\n",
    "            'DivP',\n",
    "            'cfinc1_q',\n",
    "            'EBIT',\n",
    "            'EARNYILD',\n",
    "            'EPIBS']\n",
    "\n",
    "freq = '5b'\n",
    "batch = 32\n",
    "universe = Universe('custom', ['zz500', 'hs300'])\n",
    "benchmark = 905\n",
    "neutralized_risk = ['SIZE'] + industry_styles\n",
    "horizon = map_freq(freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-01-10 14:56:47,595 - ALPHA_MIND - INFO - Starting data package fetching ...\n",
      "2018-01-10 14:56:54,781 - ALPHA_MIND - INFO - factor data loading finished\n",
      "2018-01-10 14:57:03,949 - ALPHA_MIND - INFO - return data loading finished\n",
      "2018-01-10 14:57:05,113 - ALPHA_MIND - INFO - industry data loading finished\n",
      "2018-01-10 14:57:05,828 - ALPHA_MIND - INFO - benchmark data loading finished\n",
      "2018-01-10 14:57:15,662 - ALPHA_MIND - INFO - risk data loading finished\n",
      "2018-01-10 14:57:17,773 - ALPHA_MIND - INFO - data merging finished\n",
      "2018-01-10 14:57:19,490 - ALPHA_MIND - INFO - Loading data is finished\n",
      "2018-01-10 14:57:35,324 - ALPHA_MIND - INFO - Data processing is finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 47.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "factor_data = fetch_data_package(engine,\n",
    "                                 features,\n",
    "                                 start_date,\n",
    "                                 end_date,\n",
    "                                 '5b',\n",
    "                                 universe,\n",
    "                                 benchmark,\n",
    "                                 batch=batch,\n",
    "                                 warm_start=batch,\n",
    "                                 neutralized_risk=neutralized_risk, \n",
    "                                 pre_process=[winsorize_normal, standardize],\n",
    "                                 post_process=[winsorize_normal, standardize])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = factor_data['x_names']\n",
    "\n",
    "train_x = factor_data['train']['x']\n",
    "train_y = factor_data['train']['y']\n",
    "train_risk = factor_data['train']['risk']\n",
    "ref_dates = sorted(train_x.keys())\n",
    "\n",
    "predict_x = factor_data['predict']['x']\n",
    "predict_y = factor_data['predict']['y']\n",
    "predict_risk = factor_data['predict']['risk']\n",
    "settlement = factor_data['settlement']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear Regression\n",
    "------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2012-02-15 00:00:00\n",
      "2012-06-06 00:00:00\n",
      "2012-09-20 00:00:00\n",
      "2013-01-15 00:00:00\n",
      "2013-05-14 00:00:00\n",
      "2013-08-30 00:00:00\n",
      "2013-12-24 00:00:00\n",
      "2014-04-17 00:00:00\n",
      "2014-08-05 00:00:00\n",
      "2014-11-26 00:00:00\n",
      "2015-03-20 00:00:00\n",
      "2015-07-08 00:00:00\n",
      "2015-10-30 00:00:00\n",
      "2016-02-22 00:00:00\n",
      "2016-06-08 00:00:00\n",
      "2016-09-27 00:00:00\n",
      "2017-01-18 00:00:00\n",
      "2017-05-15 00:00:00\n",
      "2017-08-30 00:00:00\n",
      "2017-12-20 00:00:00\n",
      "Wall time: 1.26 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_dates = list(train_x.keys())\n",
    "train_scores = []\n",
    "predict_scores = []\n",
    "\n",
    "for i, date in enumerate(train_dates):\n",
    "    if i % 15 == 0:\n",
    "        print(date)\n",
    "    x = train_x[date]\n",
    "    y = train_y[date]\n",
    "    \n",
    "    model = LinearRegression(fit_intercept=False, features=features)\n",
    "    model.fit(x, y)\n",
    "    train_scores.append(model.score(x, y))\n",
    "    \n",
    "    p_x = predict_x[date]\n",
    "    p_y = predict_y[date]\n",
    "    predict_scores.append(model.score(p_x, p_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0107609007052\n",
      "-0.480548329833\n"
     ]
    }
   ],
   "source": [
    "print(np.mean(train_scores))\n",
    "print(np.mean(predict_scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lasso Regression\n",
    "---------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2012-02-15 00:00:00\n",
      "2012-06-06 00:00:00\n",
      "2012-09-20 00:00:00\n",
      "2013-01-15 00:00:00\n",
      "2013-05-14 00:00:00\n",
      "2013-08-30 00:00:00\n",
      "2013-12-24 00:00:00\n",
      "2014-04-17 00:00:00\n",
      "2014-08-05 00:00:00\n",
      "2014-11-26 00:00:00\n",
      "2015-03-20 00:00:00\n",
      "2015-07-08 00:00:00\n",
      "2015-10-30 00:00:00\n",
      "2016-02-22 00:00:00\n",
      "2016-06-08 00:00:00\n",
      "2016-09-27 00:00:00\n",
      "2017-01-18 00:00:00\n",
      "2017-05-15 00:00:00\n",
      "2017-08-30 00:00:00\n",
      "2017-12-20 00:00:00\n",
      "Wall time: 1.58 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_dates = list(train_x.keys())\n",
    "train_scores = []\n",
    "predict_scores = []\n",
    "\n",
    "for i, date in enumerate(train_dates):\n",
    "    if i % 15 == 0:\n",
    "        print(date)\n",
    "    x = train_x[date]\n",
    "    y = train_y[date]\n",
    "    \n",
    "    model = LassoRegression(alpha=0.01, fit_intercept=False, features=features)\n",
    "    model.fit(x, y)\n",
    "    train_scores.append(model.score(x, y))\n",
    "    \n",
    "    p_x = predict_x[date]\n",
    "    p_y = predict_y[date]\n",
    "    predict_scores.append(model.score(p_x, p_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.00875291615929\n",
      "-0.475440026\n"
     ]
    }
   ],
   "source": [
    "print(np.mean(train_scores))\n",
    "print(np.mean(predict_scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear Regression with More Features\n",
    "----------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cross_product(x, y):\n",
    "    n, m = x.shape\n",
    "    res = []\n",
    "    \n",
    "    for j in range(m):\n",
    "        res.append(x[:, [j]] * y)\n",
    "        \n",
    "    return np.concatenate(res, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2017-01-03 00:00:00\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'cross_product' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'cross_product' is not defined"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_dates = list(train_x.keys())\n",
    "train_scores = []\n",
    "predict_scores = []\n",
    "\n",
    "for i, date in enumerate(train_dates[:1]):\n",
    "    if i % 15 == 0:\n",
    "        print(date)\n",
    "    x = train_x[date]\n",
    "    y = train_y[date]\n",
    "    risk = train_risk[date][:, 1:]\n",
    "    new_x = cross_product(x, risk)\n",
    "    \n",
    "    model = LinearRegression(fit_intercept=False, features=features)\n",
    "    model.fit(new_x, y)\n",
    "    train_scores.append(model.score(new_x, y))\n",
    "    \n",
    "    p_x = predict_x[date]\n",
    "    p_y = predict_y[date]\n",
    "    p_risk = predict_risk[date][:, 1:]\n",
    "    new_p_x = cross_product(p_x, p_risk)\n",
    "    predict_scores.append(model.score(new_p_x, p_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0291928676769\n",
      "-0.24146254373\n"
     ]
    }
   ],
   "source": [
    "print(np.mean(train_scores))\n",
    "print(np.mean(predict_scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lasso Regression with More Features\n",
    "----------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2017-01-03 00:00:00\n",
      "2017-04-27 00:00:00\n",
      "2017-08-15 00:00:00\n",
      "2017-12-05 00:00:00\n",
      "Wall time: 4.78 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_dates = list(train_x.keys())\n",
    "train_scores = []\n",
    "predict_scores = []\n",
    "\n",
    "for i, date in enumerate(train_dates):\n",
    "    if i % 15 == 0:\n",
    "        print(date)\n",
    "    x = train_x[date]\n",
    "    y = train_y[date]\n",
    "    risk = train_risk[date][:, 1:]\n",
    "    new_x = cross_product(x, risk)\n",
    "    \n",
    "    model = LassoRegression(alpha=0.01, fit_intercept=False, features=features)\n",
    "    model.fit(new_x, y)\n",
    "    train_scores.append(model.score(new_x, y))\n",
    "    \n",
    "    p_x = predict_x[date]\n",
    "    p_y = predict_y[date]\n",
    "    p_risk = predict_risk[date][:, 1:]\n",
    "    new_p_x = cross_product(p_x, p_risk)\n",
    "    predict_scores.append(model.score(new_p_x, p_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.000355789142204\n",
      "-0.200552889618\n"
     ]
    }
   ],
   "source": [
    "print(np.mean(train_scores))\n",
    "print(np.mean(predict_scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest Regressor\n",
    "---------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2017-01-03 00:00:00\n",
      "2017-04-27 00:00:00\n",
      "2017-08-15 00:00:00\n",
      "2017-12-05 00:00:00\n",
      "Wall time: 1min 18s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_dates = list(train_x.keys())\n",
    "train_scores = []\n",
    "predict_scores = []\n",
    "\n",
    "for i, date in enumerate(train_dates):\n",
    "    if i % 15 == 0:\n",
    "        print(date)\n",
    "    x = train_x[date]\n",
    "    y = train_y[date]\n",
    "    \n",
    "    model = RandomForestRegressor(n_estimators=500, max_features='sqrt', max_depth=3, n_jobs=-1)\n",
    "    model.fit(x, y)\n",
    "    train_scores.append(model.score(x, y))\n",
    "    \n",
    "    p_x = predict_x[date]\n",
    "    p_y = predict_y[date]\n",
    "    predict_scores.append(model.score(p_x, p_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0137863030105\n",
      "-0.197952235791\n"
     ]
    }
   ],
   "source": [
    "print(np.mean(train_scores))\n",
    "print(np.mean(predict_scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost Regressor\n",
    "------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2017-01-03 00:00:00\n",
      "2017-04-27 00:00:00\n",
      "2017-08-15 00:00:00\n",
      "2017-12-05 00:00:00\n",
      "Wall time: 1min 32s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_dates = list(train_x.keys())\n",
    "train_scores = []\n",
    "predict_scores = []\n",
    "\n",
    "for i, date in enumerate(train_dates):\n",
    "    if i % 15 == 0:\n",
    "        print(date)\n",
    "    x = train_x[date]\n",
    "    y = train_y[date]\n",
    "    model = XGBRegressor(n_estimators=500,\n",
    "                         learning_rate=0.02,\n",
    "                         max_depth=3,\n",
    "                         n_jobs=-1,\n",
    "                         subsample=0.25,\n",
    "                         colsample_bytree=0.5)\n",
    "    model.fit(x, y)\n",
    "    train_scores.append(model.score(x, y))\n",
    "    \n",
    "    p_x = predict_x[date]\n",
    "    p_y = predict_y[date]\n",
    "    predict_scores.append(model.score(p_x, p_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0575499865219\n",
      "-0.209037365429\n"
     ]
    }
   ],
   "source": [
    "print(np.mean(train_scores))\n",
    "print(np.mean(predict_scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Native XGBoost Regressor\n",
    "---------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2012-02-15 00:00:00\n",
      "2012-06-06 00:00:00\n",
      "2012-09-20 00:00:00\n",
      "2013-01-15 00:00:00\n",
      "2013-05-14 00:00:00\n",
      "2013-08-30 00:00:00\n",
      "2013-12-24 00:00:00\n",
      "2014-04-17 00:00:00\n",
      "2014-08-05 00:00:00\n",
      "2014-11-26 00:00:00\n",
      "2015-03-20 00:00:00\n",
      "2015-07-08 00:00:00\n",
      "2015-10-30 00:00:00\n",
      "2016-02-22 00:00:00\n",
      "2016-06-08 00:00:00\n",
      "2016-09-27 00:00:00\n",
      "2017-01-18 00:00:00\n",
      "2017-05-15 00:00:00\n",
      "2017-08-30 00:00:00\n",
      "2017-12-20 00:00:00\n",
      "Wall time: 6min 57s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_dates = list(train_x.keys())\n",
    "train_scores = []\n",
    "predict_scores = []\n",
    "\n",
    "for i, date in enumerate(train_dates):\n",
    "    if i % 15 == 0:\n",
    "        print(date)\n",
    "    x = train_x[date]\n",
    "    y = train_y[date]\n",
    "    \n",
    "    x_train, x_eval, y_train, y_eval = train_test_split(x, y, test_size=0.33, random_state=42)\n",
    "    \n",
    "    dtrain = xgb.DMatrix(x_train, y_train)\n",
    "    deval = xgb.DMatrix(x_eval, y_eval)\n",
    "    param = {'silent': 1,\n",
    "             'objective': 'reg:linear',\n",
    "             'max_depth': 3,\n",
    "             'eta': 0.005,\n",
    "             'boost': 'gbtree',\n",
    "             'tree_method': 'hist',\n",
    "             'subsample': 0.1,\n",
    "             'colsample_bytree': 0.25}\n",
    "    num_round = 2000\n",
    "    model = xgb.train(param, dtrain, num_round, evals=[(deval, 'eval')], early_stopping_rounds=50, verbose_eval=False)\n",
    "    \n",
    "    y_train_predict = model.predict(dtrain)\n",
    "    train_scores.append(r2_score(y_train, y_train_predict, multioutput='uniform_average'))\n",
    "    \n",
    "    p_x = predict_x[date]\n",
    "    p_y = predict_y[date]\n",
    "    dtest = xgb.DMatrix(p_x, p_y)\n",
    "    \n",
    "    y_test_predict = model.predict(dtest)\n",
    "    predict_scores.append(r2_score(p_y, y_test_predict, multioutput='uniform_average'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0158347715471\n",
      "-0.477095380466\n"
     ]
    }
   ],
   "source": [
    "print(np.mean(train_scores))\n",
    "print(np.mean(predict_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
