replace column names

alphamind/analysis/calculators.py

@@ -10,13 +10,13 @@ import pandas as pd
 
 def calculate_turn_over(pos_table: pd.DataFrame) -> pd.DataFrame:
     turn_over_table = {}
-    total_factors = pos_table.columns.difference(['Code'])
+    total_factors = pos_table.columns.difference(['code'])
     pos_table.reset_index()
 
     for name in total_factors:
-        pivot_position = pos_table.pivot(values=name, columns='Code').fillna(0.)
+        pivot_position = pos_table.pivot(values=name, columns='code').fillna(0.)
         turn_over_series = pivot_position.diff().abs().sum(axis=1)
         turn_over_table[name] = turn_over_series.values
 
-    turn_over_table = pd.DataFrame(turn_over_table, index=pos_table.Date.unique())
+    turn_over_table = pd.DataFrame(turn_over_table, index=pos_table.trade_date.unique())
     return turn_over_table[total_factors]

alphamind/data/engines/sqlengine.py

@@ -108,7 +108,7 @@ def _map_risk_model_table(risk_model: str) -> tuple:
 
 def _map_factors(factors: Iterable[str]) -> dict:
     factor_cols = {}
-    excluded = {'Date', 'Code', 'isOpen', }
+    excluded = {'trade_date', 'code', 'isOpen', }
     for f in factors:
         for t in factor_tables:
             if f not in excluded and f in t.__table__.columns:
@@ -140,7 +140,7 @@ class SqlEngine(object):
     def fetch_strategy(self, ref_date: str, strategy: str) -> pd.DataFrame():
         query = select([Strategy.strategyName, Strategy.factor, Strategy.weight]).where(
             and_(
-                Strategy.Date == ref_date,
+                Strategy.trade_date == ref_date,
                 Strategy.strategyName == strategy
             )
         )
@@ -179,12 +179,12 @@ class SqlEngine(object):
         else:
             end_date = expiry_date
 
-        query = select([DailyReturn.Code, func.sum(self.ln_func(1. + DailyReturn.d1)).label('dx')]).where(
+        query = select([DailyReturn.code, func.sum(self.ln_func(1. + DailyReturn.d1)).label('dx')]).where(
             and_(
-                DailyReturn.Date.between(start_date, end_date),
-                DailyReturn.Code.in_(codes)
+                DailyReturn.trade_date.between(start_date, end_date),
+                DailyReturn.code.in_(codes)
             )
-        ).group_by(DailyReturn.Code)
+        ).group_by(DailyReturn.code)
 
         return pd.read_sql(query, self.session.bind)
 
@@ -202,21 +202,21 @@ class SqlEngine(object):
         end_date = advanceDateByCalendar('china.sse', end_date, str(horizon) + 'b').strftime('%Y-%m-%d')
 
         q2 = universe.query_range(start_date, end_date).alias('temp_universe')
-        big_table = join(DailyReturn, q2, and_(DailyReturn.Date == q2.c.Date, DailyReturn.Code == q2.c.Code))
+        big_table = join(DailyReturn, q2, and_(DailyReturn.trade_date == q2.c.trade_date, DailyReturn.code == q2.c.code))
 
         stats = func.sum(self.ln_func(1. + DailyReturn.d1)).over(
-            partition_by=DailyReturn.Code,
-            order_by=DailyReturn.Date,
+            partition_by=DailyReturn.code,
+            order_by=DailyReturn.trade_date,
             rows=(0, horizon)).label('dx')
 
-        query = select([DailyReturn.Date, DailyReturn.Code, stats]) \
+        query = select([DailyReturn.trade_date, DailyReturn.code, stats]) \
             .select_from(big_table) \
-            .where(DailyReturn.Date.between(start_date, end_date))
+            .where(DailyReturn.trade_date.between(start_date, end_date))
 
         df = pd.read_sql(query, self.session.bind)
 
         if dates:
-            df = df[df.Date.isin(dates)]
+            df = df[df.trade_date.isin(dates)]
 
         return df
 
@@ -240,17 +240,17 @@ class SqlEngine(object):
 
         big_table = Market
         for t in set(factor_cols.values()):
-            big_table = outerjoin(big_table, t, and_(Market.Date == t.Date, Market.Code == t.Code))
+            big_table = outerjoin(big_table, t, and_(Market.trade_date == t.trade_date, Market.code == t.code))
 
-        query = select([Market.Date, Market.Code, Market.isOpen] + list(factor_cols.keys())) \
+        query = select([Market.trade_date, Market.code, Market.isOpen] + list(factor_cols.keys())) \
             .select_from(big_table) \
-            .where(and_(Market.Date.between(start_date, end_date), Market.Code.in_(codes)))
+            .where(and_(Market.trade_date.between(start_date, end_date), Market.code.in_(codes)))
 
-        df = pd.read_sql(query, self.engine).sort_values(['Date', 'Code']).set_index('Date')
-        res = transformer.transform('Code', df)
+        df = pd.read_sql(query, self.engine).sort_values(['trade_date', 'code']).set_index('trade_date')
+        res = transformer.transform('code', df)
 
         for col in res.columns:
-            if col not in set(['Code', 'isOpen']) and col not in df.columns:
+            if col not in set(['code', 'isOpen']) and col not in df.columns:
                 df[col] = res[col].values
 
         df = df.loc[ref_date]
@@ -296,18 +296,18 @@ class SqlEngine(object):
 
         q2 = universe.query_range(real_start_date, real_end_date, real_dates).alias('temp_universe')
 
-        big_table = join(Market, q2, and_(Market.Date == q2.c.Date, Market.Code == q2.c.Code))
+        big_table = join(Market, q2, and_(Market.trade_date == q2.c.trade_date, Market.code == q2.c.code))
         for t in set(factor_cols.values()):
-            big_table = outerjoin(big_table, t, and_(Market.Date == t.Date, Market.Code == t.Code))
+            big_table = outerjoin(big_table, t, and_(Market.trade_date == t.trade_date, Market.code == t.code))
 
-        query = select([Market.Date, Market.Code, Market.isOpen] + list(factor_cols.keys())) \
+        query = select([Market.trade_date, Market.code, Market.isOpen] + list(factor_cols.keys())) \
             .select_from(big_table)
 
-        df = pd.read_sql(query, self.engine).sort_values(['Date', 'Code']).set_index('Date')
-        res = transformer.transform('Code', df)
+        df = pd.read_sql(query, self.engine).sort_values(['trade_date', 'code']).set_index('trade_date')
+        res = transformer.transform('code', df)
 
         for col in res.columns:
-            if col not in set(['Code', 'isOpen']) and col not in df.columns:
+            if col not in set(['code', 'isOpen']) and col not in df.columns:
                 df[col] = res[col].values
         if dates:
             df = df[df.index.isin(dates)]
@@ -318,9 +318,9 @@ class SqlEngine(object):
     def fetch_benchmark(self,
                         ref_date: str,
                         benchmark: int) -> pd.DataFrame:
-        query = select([IndexComponent.Code, (IndexComponent.weight / 100.).label('weight')]).where(
+        query = select([IndexComponent.code, (IndexComponent.weight / 100.).label('weight')]).where(
             and_(
-                IndexComponent.Date == ref_date,
+                IndexComponent.trade_date == ref_date,
                 IndexComponent.indexCode == benchmark
             )
         )
@@ -333,10 +333,10 @@ class SqlEngine(object):
                               end_date: str = None,
                               dates: Iterable[str] = None) -> pd.DataFrame:
 
-        cond = IndexComponent.Date.in_(dates) if dates else IndexComponent.Date.between(start_date, end_date)
+        cond = IndexComponent.trade_date.in_(dates) if dates else IndexComponent.trade_date.between(start_date, end_date)
 
         query = select(
-            [IndexComponent.Date, IndexComponent.Code, (IndexComponent.weight / 100.).label('weight')]).where(
+            [IndexComponent.trade_date, IndexComponent.code, (IndexComponent.weight / 100.).label('weight')]).where(
             and_(
                 cond,
                 IndexComponent.indexCode == benchmark
@@ -355,18 +355,18 @@ class SqlEngine(object):
         query = select([risk_cov_table.FactorID,
                         risk_cov_table.Factor]
                        + cov_risk_cols).where(
-            risk_cov_table.Date == ref_date
+            risk_cov_table.trade_date == ref_date
         )
         risk_cov = pd.read_sql(query, self.engine).sort_values('FactorID')
 
         risk_exposure_cols = [RiskExposure.__table__.columns[f] for f in total_risk_factors if f not in set(excluded)]
         big_table = outerjoin(special_risk_table, RiskExposure,
-                              and_(special_risk_table.Date == RiskExposure.Date,
-                                   special_risk_table.Code == RiskExposure.Code))
+                              and_(special_risk_table.trade_date == RiskExposure.trade_date,
+                                   special_risk_table.code == RiskExposure.code))
         query = select(
-            [RiskExposure.Code, special_risk_table.SRISK] + risk_exposure_cols) \
+            [RiskExposure.code, special_risk_table.SRISK] + risk_exposure_cols) \
             .select_from(big_table) \
-            .where(and_(RiskExposure.Date == ref_date, RiskExposure.Code.in_(codes)))
+            .where(and_(RiskExposure.trade_date == ref_date, RiskExposure.code.in_(codes)))
 
         risk_exp = pd.read_sql(query, self.engine)
 
@@ -384,30 +384,30 @@ class SqlEngine(object):
 
         cov_risk_cols = [risk_cov_table.__table__.columns[f] for f in total_risk_factors]
 
-        cond = risk_cov_table.Date.in_(dates) if dates else risk_cov_table.Date.between(start_date, end_date)
-        query = select([risk_cov_table.Date,
+        cond = risk_cov_table.trade_date.in_(dates) if dates else risk_cov_table.trade_date.between(start_date, end_date)
+        query = select([risk_cov_table.trade_date,
                         risk_cov_table.FactorID,
                         risk_cov_table.Factor]
                        + cov_risk_cols).where(
             cond
         )
 
-        risk_cov = pd.read_sql(query, self.engine).sort_values(['Date', 'FactorID'])
+        risk_cov = pd.read_sql(query, self.engine).sort_values(['trade_date', 'FactorID'])
 
         if not excluded:
             excluded = []
 
         risk_exposure_cols = [RiskExposure.__table__.columns[f] for f in total_risk_factors if f not in set(excluded)]
         big_table = outerjoin(special_risk_table, RiskExposure,
-                              and_(special_risk_table.Date == RiskExposure.Date,
-                                   special_risk_table.Code == RiskExposure.Code))
+                              and_(special_risk_table.trade_date == RiskExposure.trade_date,
+                                   special_risk_table.code == RiskExposure.code))
 
         q2 = universe.query_range(start_date, end_date, dates).alias('temp_universe')
         big_table = join(big_table, q2,
-                         and_(special_risk_table.Date == q2.c.Date, special_risk_table.Code == q2.c.Code))
+                         and_(special_risk_table.trade_date == q2.c.trade_date, special_risk_table.code == q2.c.code))
 
         query = select(
-            [RiskExposure.Date, RiskExposure.Code, special_risk_table.SRISK] + risk_exposure_cols) \
+            [RiskExposure.trade_date, RiskExposure.code, special_risk_table.SRISK] + risk_exposure_cols) \
             .select_from(big_table)
 
         risk_exp = pd.read_sql(query, self.engine)
@@ -428,13 +428,13 @@ class SqlEngine(object):
         if benchmark:
             benchmark_data = self.fetch_benchmark(ref_date, benchmark)
             total_data['benchmark'] = benchmark_data
-            factor_data = pd.merge(factor_data, benchmark_data, how='left', on=['Code'])
+            factor_data = pd.merge(factor_data, benchmark_data, how='left', on=['code'])
             factor_data['weight'] = factor_data['weight'].fillna(0.)
 
         if risk_model:
             excluded = list(set(total_risk_factors).intersection(transformer.dependency))
             risk_cov, risk_exp = self.fetch_risk_model(ref_date, codes, risk_model, excluded)
-            factor_data = pd.merge(factor_data, risk_exp, how='left', on=['Code'])
+            factor_data = pd.merge(factor_data, risk_exp, how='left', on=['code'])
             total_data['risk_cov'] = risk_cov
 
         total_data['factor'] = factor_data
@@ -458,13 +458,13 @@ class SqlEngine(object):
         if benchmark:
             benchmark_data = self.fetch_benchmark_range(benchmark, start_date, end_date, dates)
             total_data['benchmark'] = benchmark_data
-            factor_data = pd.merge(factor_data, benchmark_data, how='left', on=['Date', 'Code'])
+            factor_data = pd.merge(factor_data, benchmark_data, how='left', on=['trade_date', 'code'])
             factor_data['weight'] = factor_data['weight'].fillna(0.)
 
         if risk_model:
             excluded = list(set(total_risk_factors).intersection(transformer.dependency))
             risk_cov, risk_exp = self.fetch_risk_model_range(universe, start_date, end_date, dates, risk_model, excluded)
-            factor_data = pd.merge(factor_data, risk_exp, how='left', on=['Date', 'Code'])
+            factor_data = pd.merge(factor_data, risk_exp, how='left', on=['trade_date', 'code'])
             total_data['risk_cov'] = risk_cov
 
         total_data['factor'] = factor_data

alphamind/data/engines/universe.py

@@ -41,7 +41,7 @@ class Universe(object):
             all_and_conditions.append(univ_out)
 
         if self.exclude_codes:
-            codes_out = UniverseTable.Code.notin_(self.exclude_codes)
+            codes_out = UniverseTable.code.notin_(self.exclude_codes)
             all_and_conditions.append(codes_out)
 
         all_or_conditions = []
@@ -50,18 +50,18 @@ class Universe(object):
             all_or_conditions.append(univ_in)
 
         if self.include_codes:
-            codes_in = UniverseTable.Code.in_(self.include_codes)
+            codes_in = UniverseTable.code.in_(self.include_codes)
             all_or_conditions.append(codes_in)
 
         return all_and_conditions, all_or_conditions
 
     def query(self, ref_date):
-        query = select([UniverseTable.Date, UniverseTable.Code]).distinct()
+        query = select([UniverseTable.trade_date, UniverseTable.code]).distinct()
         all_and_conditions, all_or_conditions = self._create_condition()
 
         query = query.where(
             and_(
-                UniverseTable.Date == ref_date,
+                UniverseTable.trade_date == ref_date,
                 or_(
                     and_(*all_and_conditions),
                     *all_or_conditions
@@ -72,10 +72,10 @@ class Universe(object):
         return query
 
     def query_range(self, start_date=None, end_date=None, dates=None):
-        query = select([UniverseTable.Date, UniverseTable.Code]).distinct()
+        query = select([UniverseTable.trade_date, UniverseTable.code]).distinct()
         all_and_conditions, all_or_conditions = self._create_condition()
 
-        dates_cond = UniverseTable.Date.in_(dates) if dates else UniverseTable.Date.between(start_date, end_date)
+        dates_cond = UniverseTable.trade_date.in_(dates) if dates else UniverseTable.trade_date.between(start_date, end_date)
 
         query = query.where(
             and_(

alphamind/examples/model_training.py

@@ -32,13 +32,13 @@ training     - every 4 week
 engine = SqlEngine('postgresql+psycopg2://postgres:A12345678!@10.63.6.220/alpha')
 universe = Universe('zz500', ['zz500'])
 neutralize_risk = industry_styles
-alpha_factors = risk_styles
+alpha_factors = ['BDTO', 'RVOL', 'CHV', 'VAL', 'CFinc1'] # risk_styles
 benchmark = 905
 n_bins = 5
 frequency = '1w'
 batch = 4
 start_date = '2012-01-01'
-end_date = '2017-08-01'
+end_date = '2017-08-31'
 
 '''
 fetch data from target data base and do the corresponding data processing
@@ -74,7 +74,7 @@ for train_date in dates:
 
     model.fit(x, y)
     model_df.loc[train_date] = copy.deepcopy(model)
-    print('Date: {0} training finished'.format(train_date))
+    print('trade_date: {0} training finished'.format(train_date))
 
 '''
 predicting phase: using trained model on the re-balance dates
@@ -89,8 +89,8 @@ final_res = np.zeros((len(dates), n_bins))
 for i, predict_date in enumerate(dates):
     model = model_df[predict_date]
     x = predict_x[predict_date]
-    benchmark_w = settlement[settlement.Date == predict_date]['weight'].values
-    realized_r = settlement[settlement.Date == predict_date]['dx'].values
+    benchmark_w = settlement[settlement.trade_date == predict_date]['weight'].values
+    realized_r = settlement[settlement.trade_date == predict_date]['dx'].values
 
     predict_y = model.predict(x)
 
@@ -100,12 +100,14 @@ for i, predict_date in enumerate(dates):
                                benchmark=benchmark_w)
 
     final_res[i] = res / benchmark_w.sum()
-    print('Date: {0} predicting finished'.format(train_date))
+    print('trade_date: {0} predicting finished'.format(train_date))
 
 last_date = advanceDateByCalendar('china.sse', dates[-1], frequency)
 
 df = pd.DataFrame(final_res, index=dates[1:] + [last_date])
 df.sort_index(inplace=True)
-df = df.cumsum().plot()
-plt.title('Prod factors model training with Linear Regression from 2012 - 2017')
+df.cumsum().plot()
+plt.title('Risk style factors model training with Linear Regression from 2012 - 2017')
 plt.show()
+df = df.cumsum()
+df.to_csv('d:/20120101_20170823_bt.csv')

alphamind/model/data_preparing.py

@@ -50,15 +50,15 @@ def prepare_data(engine: SqlEngine,
     factor_df = engine.fetch_factor_range(universe,
                                           factors=transformer,
                                           dates=dates,
-                                          warm_start=warm_start).sort_values(['Date', 'Code'])
+                                          warm_start=warm_start).sort_values(['trade_date', 'code'])
     return_df = engine.fetch_dx_return_range(universe, dates=dates, horizon=horizon)
     benchmark_df = engine.fetch_benchmark_range(benchmark, dates=dates)
 
-    df = pd.merge(factor_df, return_df, on=['Date', 'Code']).dropna()
-    df = pd.merge(df, benchmark_df, on=['Date', 'Code'], how='left')
+    df = pd.merge(factor_df, return_df, on=['trade_date', 'code']).dropna()
+    df = pd.merge(df, benchmark_df, on=['trade_date', 'code'], how='left')
     df['weight'] = df['weight'].fillna(0.)
 
-    return df[['Date', 'Code', 'dx']], df[['Date', 'Code', 'weight'] + transformer.names]
+    return df[['trade_date', 'code', 'dx']], df[['trade_date', 'code', 'weight'] + transformer.names]
 
 
 def batch_processing(x_values,
@@ -137,10 +137,10 @@ def fetch_data_package(engine: SqlEngine,
     if neutralized_risk:
         risk_df = engine.fetch_risk_model_range(universe, dates=dates, risk_model=risk_model)[1]
         used_neutralized_risk = list(set(neutralized_risk).difference(transformer.names))
-        risk_df = risk_df[['Date', 'Code'] + used_neutralized_risk].dropna()
+        risk_df = risk_df[['trade_date', 'code'] + used_neutralized_risk].dropna()
 
-        train_x = pd.merge(factor_df, risk_df, on=['Date', 'Code'])
-        return_df = pd.merge(return_df, risk_df, on=['Date', 'Code'])[['Date', 'Code', 'dx']]
+        train_x = pd.merge(factor_df, risk_df, on=['trade_date', 'code'])
+        return_df = pd.merge(return_df, risk_df, on=['trade_date', 'code'])[['trade_date', 'code', 'dx']]
         train_y = return_df.copy()
 
         risk_exp = train_x[neutralized_risk].values.astype(float)
@@ -153,7 +153,7 @@ def fetch_data_package(engine: SqlEngine,
         x_values = train_x[transformer.names].values.astype(float)
         y_values = train_y[['dx']].values
 
-    date_label = pd.DatetimeIndex(factor_df.Date).to_pydatetime()
+    date_label = pd.DatetimeIndex(factor_df.trade_date).to_pydatetime()
     dates = np.unique(date_label)
 
     return_df['weight'] = train_x['weight']