Commit 5fa76dd3 authored by Dr.李's avatar Dr.李

update store

parent 3f27c3e7
......@@ -6,6 +6,7 @@ Created on 2017-6-26
"""
from typing import Iterable
from typing import Union
import sqlalchemy as sa
import numpy as np
import pandas as pd
......@@ -64,44 +65,67 @@ industry_styles = [
]
def fetch_codes(codes: Union[str, Iterable[int]], start_date, end_date, engine):
code_table = None
code_str = None
if isinstance(codes, str):
# universe
sql = "select Date, Code from universe where Date >= '{0}' and Date <= '{1}' and universe = '{2}'" \
.format(start_date, end_date, codes)
code_table = pd.read_sql(sql, engine)
elif hasattr(codes, '__iter__'):
code_str = ','.join(str(c) for c in codes)
return code_table, code_str
def industry_mapping(industry_arr, industry_dummies):
return [industry_arr[row][0] for row in industry_dummies]
def fetch_data(factors: Iterable[str],
start_date: str,
end_date: str,
codes: Iterable[int] = None,
codes: Union[str, Iterable[int]] = None,
benchmark: int = None,
risk_model: str = 'day') -> dict:
engine = sa.create_engine('mysql+mysqldb://{user}:{password}@{host}/{db}?charset={charset}'
.format(**db_settings['uqer']))
factor_str = ','.join('factors.' + f for f in factors)
if codes:
code_str = ','.join(str(c) for c in codes)
else:
code_str = None
code_table, code_str = fetch_codes(codes, start_date, end_date, engine)
total_risk_factors = risk_styles + industry_styles
risk_str = ','.join('risk_exposure.' + f for f in total_risk_factors)
if code_str:
sql = "select factors.Date, factors.Code, {0}, {3}" \
" from factors INNER JOIN" \
" risk_exposure on factors.Date = risk_exposure.Date and factors.Code = risk_exposure.Code" \
sql = "select factors.Date, factors.Code, {0}, {3}, market.chgPct, market.isOpen" \
" from (factors INNER JOIN" \
" risk_exposure on factors.Date = risk_exposure.Date and factors.Code = risk_exposure.Code)" \
" INNER JOIN market on factors.Date = market.Date and factors.Code = market.Code" \
" where factors.Date >= '{1}' and factors.Date <= '{2}' and factors.Code in ({4})".format(factor_str,
start_date,
end_date,
risk_str,
code_str)
else:
sql = "select factors.Date, factors.Code, {0}, {3}" \
" from factors INNER JOIN" \
" risk_exposure on factors.Date = risk_exposure.Date and factors.Code = risk_exposure.Code" \
sql = "select factors.Date, factors.Code, {0}, {3}, market.chgPct, market.isOpen" \
" from (factors INNER JOIN" \
" risk_exposure on factors.Date = risk_exposure.Date and factors.Code = risk_exposure.Code)" \
" INNER JOIN market on factors.Date = market.Date and factors.Code = market.Code" \
" where factors.Date >= '{1}' and factors.Date <= '{2}'".format(factor_str,
start_date,
end_date,
risk_str)
start_date,
end_date,
risk_str)
factor_data = pd.read_sql(sql, engine)
if code_table is not None:
factor_data = pd.merge(factor_data, code_table, on=['Date', 'Code'])
risk_cov_table = 'risk_cov_' + risk_model
risk_str = ','.join(risk_cov_table + '.' + f for f in total_risk_factors)
......@@ -126,7 +150,7 @@ def fetch_data(factors: Iterable[str],
industry_arr = np.array(industry_styles)
industry_dummies = factor_data[industry_styles].values.astype(bool)
factor_data['industry'] = [industry_arr[row][0] for row in industry_dummies]
factor_data['industry'] = industry_mapping(industry_arr, industry_dummies)
return total_data
......@@ -135,6 +159,6 @@ if __name__ == '__main__':
import datetime as dt
start = dt.datetime.now()
res = fetch_data(['EPS'], '2017-01-03', '2017-06-05', benchmark=905)
res = fetch_data(['EPS'], '2017-01-03', '2017-06-05', benchmark=905, codes='zz500')
print(res)
print(dt.datetime.now() - start)
......@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"metadata": {
"collapsed": true
},
......@@ -20,7 +20,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 2,
"metadata": {
"collapsed": true
},
......@@ -43,7 +43,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 3,
"metadata": {
"collapsed": true
},
......@@ -55,9 +55,17 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wall time: 948 ms\n"
]
}
],
"source": [
"%%time\n",
"\n",
......@@ -79,7 +87,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 5,
"metadata": {
"collapsed": true
},
......@@ -90,9 +98,17 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 6,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wall time: 2 ms\n"
]
}
],
"source": [
"%%time\n",
"factor_values = factor_data[factor_list].values\n",
......@@ -102,9 +118,17 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 7,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wall time: 453 ms\n"
]
}
],
"source": [
"%%time\n",
"ne_factor_values = neutralize(risk_exposure_values, factor_values)\n",
......@@ -113,18 +137,37 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"(3236, 38)"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"risk_exposure_values.shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 9,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wall time: 110 ms\n"
]
}
],
"source": [
"%%time\n",
"risk_cov_values = risk_cov_data.loc[risk_factor_columns, risk_factor_columns].astype(float)\n",
......@@ -133,8 +176,10 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"execution_count": 10,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"n = 500\n",
......@@ -153,9 +198,17 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 11,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wall time: 68.6 ms\n"
]
}
],
"source": [
"%%time\n",
"w = Variable(n)\n",
......@@ -173,9 +226,57 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 12,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"ECOS 2.0.4 - (C) embotech GmbH, Zurich Switzerland, 2012-15. Web: www.embotech.com/ECOS\n",
"\n",
"It pcost dcost gap pres dres k/t mu step sigma IR | BT\n",
" 0 -1.877e-01 -1.780e+03 +3e+05 9e-01 7e-05 1e+00 3e+02 --- --- 1 1 - | - - \n",
" 1 -1.602e+03 -1.632e+03 +5e+03 2e-01 1e-06 4e-02 5e+00 0.9827 1e-04 2 3 3 | 0 0\n",
" 2 -2.076e+02 -2.079e+02 +3e+02 3e-03 1e-08 3e-03 3e-01 0.9890 3e-04 3 6 6 | 0 0\n",
" 3 -1.051e+02 -1.052e+02 +2e+02 1e-03 6e-09 1e-02 2e-01 0.5385 7e-02 5 7 8 | 0 0\n",
" 4 -8.891e+01 -8.897e+01 +1e+02 1e-03 5e-09 6e-02 1e-01 0.6989 6e-01 4 8 7 | 0 0\n",
" 5 -4.536e+01 -4.538e+01 +5e+01 5e-04 2e-09 3e-02 5e-02 0.6049 1e-01 5 8 8 | 0 0\n",
" 6 -2.019e+01 -2.020e+01 +2e+01 1e-04 7e-10 1e-02 2e-02 0.8600 2e-01 3 8 8 | 0 0\n",
" 7 -2.056e+01 -2.056e+01 +2e+01 1e-04 7e-10 1e-02 2e-02 0.1776 8e-01 3 7 7 | 0 0\n",
" 8 -1.649e+01 -1.649e+01 +1e+01 1e-04 5e-10 1e-02 1e-02 0.9885 7e-01 4 8 8 | 0 0\n",
" 9 -1.367e+01 -1.367e+01 +7e+00 7e-05 3e-10 6e-03 7e-03 0.9890 6e-01 3 7 7 | 0 0\n",
"10 -1.061e+01 -1.061e+01 +3e+00 3e-05 1e-10 3e-03 3e-03 0.7242 2e-01 3 7 7 | 0 0\n",
"11 -1.013e+01 -1.013e+01 +2e+00 3e-05 1e-10 2e-03 2e-03 0.8257 8e-01 3 7 6 | 0 0\n",
"12 -9.405e+00 -9.405e+00 +1e+00 2e-05 5e-11 1e-03 1e-03 0.7934 4e-01 3 7 6 | 0 0\n",
"13 -8.799e+00 -8.799e+00 +3e-01 1e-05 1e-11 3e-04 3e-04 0.8278 1e-01 3 7 6 | 0 0\n",
"14 -8.665e+00 -8.665e+00 +1e-01 8e-06 4e-12 9e-05 1e-04 0.9890 3e-01 3 6 6 | 0 0\n",
"15 -8.620e+00 -8.620e+00 +2e-02 2e-06 7e-13 1e-05 2e-05 0.9483 1e-01 3 5 5 | 0 0\n",
"16 -8.612e+00 -8.612e+00 +9e-04 8e-07 4e-14 8e-07 9e-07 0.9663 2e-02 3 4 4 | 0 0\n",
"17 -8.611e+00 -8.611e+00 +4e-05 2e-07 2e-15 4e-08 4e-08 0.9642 1e-02 3 4 4 | 0 0\n",
"18 -8.611e+00 -8.611e+00 +8e-06 5e-08 4e-16 7e-09 8e-09 0.8511 3e-02 2 3 3 | 0 0\n",
"19 -8.611e+00 -8.611e+00 +2e-06 3e-08 6e-17 2e-09 2e-09 0.9047 2e-01 3 4 4 | 0 0\n",
"20 -8.611e+00 -8.611e+00 +3e-07 6e-09 4e-18 3e-10 3e-10 0.9043 9e-02 2 2 2 | 0 0\n",
"21 -8.611e+00 -8.611e+00 +2e-08 9e-10 8e-19 2e-11 2e-11 0.9360 9e-03 3 3 3 | 0 0\n",
"\n",
"OPTIMAL (within feastol=8.6e-10, reltol=2.8e-09, abstol=2.4e-08).\n",
"Runtime: 1.956442 seconds.\n",
"\n",
"Wall time: 2.2 s\n"
]
},
{
"data": {
"text/plain": [
"-8.611391732276179"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%time\n",
"prob.solve(verbose=True)"
......@@ -183,18 +284,67 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 13,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"('optimal', -8.611391732276179)"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"prob.status, prob.value"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 14,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" pcost dcost gap pres dres k/t\n",
" 0: -2.3819e-01 -1.8511e+03 1e+05 2e+01 2e-03 1e+00\n",
" 1: -1.2862e+03 -1.4195e+03 7e+03 1e+00 2e-04 4e-02\n",
" 2: -5.5826e+01 -7.2123e+01 3e+02 2e-01 2e-05 4e-01\n",
" 3: -2.7307e+01 -3.2510e+01 9e+01 5e-02 6e-06 1e-01\n",
" 4: -1.7701e+01 -1.9666e+01 3e+01 2e-02 2e-06 4e-02\n",
" 5: -1.9140e+01 -2.0915e+01 3e+01 2e-02 2e-06 3e-02\n",
" 6: -1.5705e+01 -1.6831e+01 2e+01 1e-02 1e-06 2e-02\n",
" 7: -1.2820e+01 -1.3398e+01 8e+00 6e-03 7e-07 1e-02\n",
" 8: -1.1756e+01 -1.2160e+01 6e+00 4e-03 5e-07 7e-03\n",
" 9: -9.8873e+00 -1.0053e+01 2e+00 2e-03 2e-07 3e-03\n",
"10: -9.7630e+00 -9.9076e+00 2e+00 2e-03 2e-07 2e-03\n",
"11: -8.9206e+00 -8.9597e+00 5e-01 4e-04 5e-08 6e-04\n",
"12: -8.7573e+00 -8.7765e+00 2e-01 2e-04 2e-08 3e-04\n",
"13: -8.6970e+00 -8.7087e+00 1e-01 1e-04 1e-08 2e-04\n",
"14: -8.6316e+00 -8.6344e+00 4e-02 3e-05 3e-09 4e-05\n",
"15: -8.6130e+00 -8.6133e+00 3e-03 3e-06 3e-10 3e-06\n",
"16: -8.6114e+00 -8.6114e+00 8e-05 7e-08 9e-12 9e-08\n",
"17: -8.6114e+00 -8.6114e+00 8e-06 6e-09 3e-11 8e-09\n",
"Optimal solution found.\n",
"Wall time: 2.28 s\n"
]
},
{
"data": {
"text/plain": [
"-8.611395800866157"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%time\n",
"prob.solve(verbose=True, solver='CVXOPT')"
......@@ -202,18 +352,54 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 15,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"('optimal', -8.611395800866157)"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"prob.status, prob.value"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 16,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" pcost dcost gap pres dres\n",
" 0: -3.5750e+02 -4.1776e+02 2e+04 1e+02 5e-14\n",
" 1: -3.1011e+02 -2.5923e+02 1e+04 8e+01 4e-14\n",
" 2: -6.7480e+01 -1.4607e+02 2e+03 1e+01 4e-14\n",
" 3: -1.5291e+01 -1.3022e+02 4e+02 1e+00 1e-14\n",
" 4: -8.9120e+00 -5.0845e+01 6e+01 1e-01 2e-15\n",
" 5: -8.1091e+00 -1.9295e+01 1e+01 2e-02 2e-15\n",
" 6: -8.1870e+00 -1.1578e+01 4e+00 5e-03 2e-15\n",
" 7: -8.2484e+00 -9.7661e+00 2e+00 9e-04 1e-15\n",
" 8: -8.4016e+00 -9.1567e+00 8e-01 7e-16 2e-15\n",
" 9: -8.5308e+00 -8.7593e+00 2e-01 4e-16 1e-15\n",
"10: -8.5717e+00 -8.6888e+00 1e-01 1e-16 2e-15\n",
"11: -8.6012e+00 -8.6299e+00 3e-02 6e-16 2e-15\n",
"12: -8.6086e+00 -8.6152e+00 7e-03 2e-15 2e-15\n",
"13: -8.6114e+00 -8.6114e+00 9e-05 2e-15 2e-15\n",
"14: -8.6114e+00 -8.6114e+00 1e-06 7e-16 2e-15\n",
"Optimal solution found.\n",
"Wall time: 898 ms\n"
]
}
],
"source": [
"%%time\n",
"P = matrix(sec_cov_values)\n",
......@@ -249,7 +435,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 17,
"metadata": {
"collapsed": true
},
......@@ -260,7 +446,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 18,
"metadata": {
"collapsed": true
},
......@@ -274,7 +460,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 19,
"metadata": {
"collapsed": true
},
......@@ -297,7 +483,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 20,
"metadata": {
"collapsed": true
},
......@@ -330,9 +516,17 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 21,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.735152\n"
]
}
],
"source": [
"s = time_function(cvxopt, n)\n",
"print(s)"
......@@ -340,9 +534,26 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 22,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Scale for 200\n",
"Scale for 400\n",
"Scale for 600\n",
"Scale for 800\n",
"Scale for 1000\n",
"Scale for 1200\n",
"Scale for 1400\n",
"Scale for 1600\n",
"Scale for 1800\n",
"Scale for 2000\n"
]
}
],
"source": [
"n_steps = list(range(200, 2001, 200))\n",
"cvxpy_times = [None] * len(n_steps)\n",
......@@ -358,8 +569,10 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"execution_count": 23,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"df = pd.DataFrame({'cvxpy': cvxpy_times,\n",
......@@ -370,8 +583,10 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"execution_count": 24,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"ipopt_times = [63.4241, 157.568, 208.774, 436.105, 522.81, 744.118, 1133.87, 1685.59, 1409.36, 3364.33]"
......@@ -379,7 +594,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 25,
"metadata": {
"collapsed": true
},
......@@ -388,6 +603,132 @@
"df['ipopt'] = ipopt_times"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style>\n",
" .dataframe thead tr:only-child th {\n",
" text-align: right;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>cvxopt</th>\n",
" <th>cvxpy</th>\n",
" <th>ipopt</th>\n",
" </tr>\n",
" <tr>\n",
" <th>Problem Scale (n)</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>200</th>\n",
" <td>50.551</td>\n",
" <td>271.014</td>\n",
" <td>63.4241</td>\n",
" </tr>\n",
" <tr>\n",
" <th>400</th>\n",
" <td>398.852</td>\n",
" <td>1213.275</td>\n",
" <td>157.5680</td>\n",
" </tr>\n",
" <tr>\n",
" <th>600</th>\n",
" <td>1377.988</td>\n",
" <td>3749.239</td>\n",
" <td>208.7740</td>\n",
" </tr>\n",
" <tr>\n",
" <th>800</th>\n",
" <td>3437.146</td>\n",
" <td>9118.570</td>\n",
" <td>436.1050</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1000</th>\n",
" <td>6459.141</td>\n",
" <td>17635.912</td>\n",
" <td>522.8100</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1200</th>\n",
" <td>12870.037</td>\n",
" <td>30207.671</td>\n",
" <td>744.1180</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1400</th>\n",
" <td>21711.707</td>\n",
" <td>48569.790</td>\n",
" <td>1133.8700</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1600</th>\n",
" <td>26667.136</td>\n",
" <td>66676.783</td>\n",
" <td>1685.5900</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1800</th>\n",
" <td>38267.565</td>\n",
" <td>108751.792</td>\n",
" <td>1409.3600</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2000</th>\n",
" <td>58030.362</td>\n",
" <td>114277.842</td>\n",
" <td>3364.3300</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" cvxopt cvxpy ipopt\n",
"Problem Scale (n) \n",
"200 50.551 271.014 63.4241\n",
"400 398.852 1213.275 157.5680\n",
"600 1377.988 3749.239 208.7740\n",
"800 3437.146 9118.570 436.1050\n",
"1000 6459.141 17635.912 522.8100\n",
"1200 12870.037 30207.671 744.1180\n",
"1400 21711.707 48569.790 1133.8700\n",
"1600 26667.136 66676.783 1685.5900\n",
"1800 38267.565 108751.792 1409.3600\n",
"2000 58030.362 114277.842 3364.3300"
]
},
"execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df"
]
},
{
"cell_type": "code",
"execution_count": null,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment