Merge pull request #12 from alpha-miner/master

merge update

alphamind/model/data_preparing.py

@@ -108,7 +108,7 @@ def prepare_data(engine: SqlEngine,
     df.dropna(inplace=True)
 
     return dates, df[['trade_date', 'code', 'dx']], df[
-        ['trade_date', 'code', 'weight', 'isOpen', 'industry_code', 'industry'] + transformer.names]
+        ['trade_date', 'code', 'weight', 'industry_code', 'industry'] + transformer.names]
 
 
 def batch_processing(names,
@@ -231,7 +231,6 @@ def fetch_data_package(engine: SqlEngine,
     target_df['weight'] = train_x['weight']
     target_df['industry'] = train_x['industry']
     target_df['industry_code'] = train_x['industry_code']
-    target_df['isOpen'] = train_x['isOpen']
 
     if neutralized_risk:
         for i, name in enumerate(neutralized_risk):
@@ -314,7 +313,7 @@ def fetch_train_phase(engine,
     df = pd.merge(factor_df, target_df, on=['trade_date', 'code']).dropna()
 
     target_df, factor_df = df[['trade_date', 'code', 'dx']], df[
-        ['trade_date', 'code', 'isOpen'] + transformer.names]
+        ['trade_date', 'code'] + transformer.names]
 
     target_df, dates, date_label, risk_exp, x_values, y_values, _, _, codes = \
         _merge_df(engine, transformer.names, factor_df, target_df, universe, dates, risk_model, neutralized_risk)

alphamind/strategy/strategy.py

@@ -166,7 +166,7 @@ class Strategy(object):
 
         if self.dask_client is None:
             models = {}
-            for ref_date, _ in self.total_data_groups:
+            for ref_date, _ in total_data_groups:
                 models[ref_date] = train_model(ref_date.strftime('%Y-%m-%d'), self.alpha_model, self.data_meta)
         else:
             def worker(parameters):
@@ -334,8 +334,8 @@ if __name__ == '__main__':
                          universe=universe,
                          batch=1,
                          neutralized_risk=neutralized_risk,
-                         pre_process=None,  # [winsorize_normal, standardize],
-                         post_process=None,  # [standardize],
+                         pre_process=None,
+                         post_process=None,
                          warm_start=1)
 
     industries = industry_list('sw_adj', 1)

notebooks/Example 10 - Quadratic Optimizer Comparison with CVXOPT.ipynb

@@ -40,7 +40,7 @@
     "ref_date = '2018-02-08'\n",
     "\n",
     "engine = SqlEngine(os.environ['DB_URI'])\n",
-    "universe = Universe('custom', ['ashare_ex'])\n",
+    "universe = Universe('ashare_ex')\n",
     "codes = engine.fetch_codes(ref_date, universe)\n",
     "\n",
     "risk_cov, risk_exposure = engine.fetch_risk_model(ref_date, codes)\n",

notebooks/Example 11 - Long Short Strategy Model.ipynb

@@ -38,7 +38,7 @@
     "freq = '5b'\n",
     "horizon = map_freq(freq)\n",
     "neutralized_risk = risk_styles + industry_styles\n",
-    "universe = Universe(\"custom\", ['zz500'])\n",
+    "universe = Universe('zz500')\n",
     "data_source = os.environ['DB_URI']\n",
     "offset = 1\n",
     "method = 'ls'\n",
@@ -185,7 +185,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.3"
+   "version": "3.6.5"
   },
   "varInspector": {
    "cols": {

notebooks/Example 13 - Evaluation within Industry Groups.ipynb

@@ -30,7 +30,7 @@
     "from alphamind.api import *\n",
     "\n",
     "factor = 'CFO2EV'\n",
-    "universe = Universe('custom', ['zz800'])\n",
+    "universe = Universe('zz800')\n",
     "start_date = '2010-01-01'\n",
     "end_date = '2018-04-26'\n",
     "freq = '20b'\n",
@@ -185,7 +185,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sample_factor3 = sample_factor3[['code', 'isOpen', 'f3', 'industry']]\n",
+    "sample_factor3 = sample_factor3[['code', 'f3', 'industry']]\n",
     "sample_factor3.sort_values('f3', ascending=False).head(15)"
    ]
   },
@@ -306,7 +306,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.3"
+   "version": "3.6.5"
   },
   "varInspector": {
    "cols": {

notebooks/Example 2 - Strategy Analysis.ipynb

@@ -116,12 +116,7 @@
    "outputs": [],
    "source": [
     "# Running settings\n",
-    "running_setting = RunningSetting(universe,\n",
-    "                                 start_date,\n",
-    "                                 end_date,\n",
-    "                                 freq,\n",
-    "                                 benchmark=benchmark_code,\n",
-    "                                 weights_bandwidth=weights_bandwidth,\n",
+    "running_setting = RunningSetting(weights_bandwidth=weights_bandwidth,\n",
     "                                 rebalance_method=method,\n",
     "                                 bounds=bounds,\n",
     "                                 turn_over_target=turn_over_target)"
@@ -134,8 +129,16 @@
    "outputs": [],
    "source": [
     "# Strategy\n",
-    "strategy = Strategy(alpha_model, data_meta, running_setting)\n",
-    "ret_df, positions = strategy.run()"
+    "strategy = Strategy(alpha_model,\n",
+    "                    data_meta,\n",
+    "                    universe=universe,\n",
+    "                    start_date=start_date,\n",
+    "                    end_date=end_date,\n",
+    "                    freq=freq,\n",
+    "                    benchmark=benchmark_code)\n",
+    "\n",
+    "strategy.prepare_backtest_data()\n",
+    "ret_df, positions = strategy.run(running_setting=running_setting)"
    ]
   },
   {

notebooks/Example 3 - Multi Weight Gap Comparison.ipynb

@@ -42,25 +42,24 @@
     "benchmark_code = 905\n",
     "universe = Universe('zz800') + Universe('cyb')\n",
     "\n",
-    "def create_scenario(weights_bandwidth=0.02, target_vol=0.01, method='risk_neutral'):\n",
-    "    start_date = '2017-01-01'\n",
-    "    end_date = '2018-04-27'\n",
-    "    freq = '10b'\n",
-    "    neutralized_risk = None\n",
+    "start_date = '2017-01-01'\n",
+    "end_date = '2018-04-27'\n",
+    "freq = '10b'\n",
+    "neutralized_risk = None\n",
     "\n",
-    "    alpha_factors = {\n",
+    "alpha_factors = {\n",
     "    'f01': CSQuantiles(LAST('EPS'), groups='sw1'),\n",
     "    'f02': CSQuantiles(LAST('ROE'), groups='sw1'),\n",
-    "    }\n",
+    "}\n",
     "\n",
-    "    weights = dict(\n",
+    "weights = dict(\n",
     "               f01=1.,\n",
     "               f02=1.\n",
     "               )\n",
     "\n",
-    "    alpha_model = ConstLinearModel(features=alpha_factors, weights=weights)\n",
+    "alpha_model = ConstLinearModel(features=alpha_factors, weights=weights)\n",
     "\n",
-    "    data_meta = DataMeta(freq=freq,\n",
+    "data_meta = DataMeta(freq=freq,\n",
     "                     universe=universe,\n",
     "                     batch=1,\n",
     "                     neutralized_risk=None,\n",
@@ -68,6 +67,17 @@
     "                     post_process=None,\n",
     "                     data_source=os.environ['DB_URI'])\n",
     "\n",
+    "strategy = Strategy(alpha_model,\n",
+    "                    data_meta,\n",
+    "                    universe=universe,\n",
+    "                    start_date=start_date,\n",
+    "                    end_date=end_date,\n",
+    "                    freq=freq,\n",
+    "                    benchmark=benchmark_code)\n",
+    "\n",
+    "strategy.prepare_backtest_data()\n",
+    "\n",
+    "def create_scenario(weights_bandwidth=0.02, target_vol=0.01, method='risk_neutral'):\n",
     "    industry_names = industry_list('sw', 1)\n",
     "    constraint_risk = ['SIZE', 'SIZENL', 'BETA']\n",
     "    total_risk_names = constraint_risk + ['benchmark', 'total']\n",
@@ -93,19 +103,13 @@
     "            u_val.append(0.0)\n",
     "\n",
     "    bounds = create_box_bounds(total_risk_names, b_type, l_val, u_val)\n",
-    "    running_setting = RunningSetting(universe,\n",
-    "                                     start_date,\n",
-    "                                     end_date,\n",
-    "                                     freq,\n",
-    "                                     benchmark=benchmark_code,\n",
-    "                                     weights_bandwidth=weights_bandwidth,\n",
+    "    running_setting = RunningSetting(weights_bandwidth=weights_bandwidth,\n",
     "                                     rebalance_method=method,\n",
     "                                     bounds=bounds,\n",
     "                                     target_vol=target_vol,\n",
     "                                     turn_over_target=0.4)\n",
     "\n",
-    "    strategy = Strategy(alpha_model, data_meta, running_setting)\n",
-    "    ret_df, positions = strategy.run()\n",
+    "    ret_df, positions = strategy.run(running_setting)\n",
     "    return ret_df"
    ]
   },
@@ -150,6 +154,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "%%time\n",
     "weight_gaps = [0.005, 0.010, 0.015, 0.020]\n",
     "\n",
     "with pd.ExcelWriter(f'zz800_cyb_{benchmark_code}_gap.xlsx', engine='xlsxwriter') as writer:\n",
@@ -166,6 +171,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "%%time\n",
     "target_vols = [0.015, 0.030, 0.045, 0.060]\n",
     "\n",
     "with pd.ExcelWriter(f'zz800_cyb_{benchmark_code}_tv.xlsx', engine='xlsxwriter') as writer:\n",
@@ -190,6 +196,18 @@
    "language": "python",
    "name": "python3"
   },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.5"
+  },
   "varInspector": {
    "cols": {
     "lenName": 16,

notebooks/Example 6 - Target Volatility Builder.ipynb

@@ -224,12 +224,7 @@
     "\"\"\"\n",
     "Running Settings\n",
     "\"\"\"\n",
-    "running_setting = RunningSetting(universe,\n",
-    "                                 start_date,\n",
-    "                                 end_date,\n",
-    "                                 freq,\n",
-    "                                 benchmark=benchmark_code,\n",
-    "                                 weights_bandwidth=weights_bandwidth,\n",
+    "running_setting = RunningSetting(weights_bandwidth=weights_bandwidth,\n",
     "                                 rebalance_method='tv',\n",
     "                                 bounds=bounds,\n",
     "                                 target_vol=target_vol)"
@@ -244,8 +239,15 @@
     "\"\"\"\n",
     "Strategy run\n",
     "\"\"\"\n",
-    "strategy = Strategy(alpha_model, data_meta, running_setting)\n",
-    "ret_df, positions = strategy.run()"
+    "strategy = Strategy(alpha_model,\n",
+    "                    data_meta,\n",
+    "                    universe=universe,\n",
+    "                    start_date=start_date,\n",
+    "                    end_date=end_date,\n",
+    "                    freq=freq,\n",
+    "                    benchmark=benchmark_code)\n",
+    "strategy.prepare_backtest_data()\n",
+    "ret_df, positions = strategy.run(running_setting)"
    ]
   },
   {

notebooks/Example 7 - Portfolio Optimizer Performance.ipynb

@@ -63,7 +63,7 @@
     "data_source = os.environ['DB_URI']\n",
     "engine = SqlEngine(data_source)\n",
     "\n",
-    "universes = [Universe('custom', [u_name]) for u_name in u_names]\n",
+    "universes = [Universe(u_name) for u_name in u_names]\n",
     "codes_set = [engine.fetch_codes(ref_date, universe=universe) for universe in universes]\n",
     "data_set = [engine.fetch_data(ref_date, factor, codes, benchmark=b_code, risk_model=risk_model) for codes, b_code in zip(codes_set, b_codes)]"
    ]

notebooks/Example 8 - Neutralize Algorithm Comparison.ipynb

@@ -46,7 +46,7 @@
     "factor = 'EPS'\n",
     "\n",
     "engine = SqlEngine(os.environ['DB_URI'])\n",
-    "universe = Universe('custom', ['zz800'])"
+    "universe = Universe('zz800')"
    ]
   },
   {