Skip to content

A
alpha-mind
Commit bbb01231 authored by Dr.李's avatar Dr.李
Browse files
Options

using numba instead of cython to simplify codes

parent f786b960
Hide whitespace changes
Inline Side-by-side
Showing with 40 additions and 251 deletions
alphamind/aggregate.pyx
View file @ bbb01231
......@@ -7,20 +7,16 @@ Created on 2017-4-26
"""
import numpy as np
from numpy import zeros
from numpy import max as nmax
cimport numpy as np
cimport cython
from libc.math cimport sqrt
from libc.math cimport fabs
from libc.stdlib cimport malloc
from libc.stdlib cimport free
from libcpp.vector cimport vector as cpp_vector
from libcpp.unordered_map cimport unordered_map as cpp_map
from cython.operator cimport dereference as deref
np.import_array()
cdef extern from "numpy/arrayobject.h":
void PyArray_ENABLEFLAGS(np.ndarray arr, int flags)
ctypedef long long int64_t
......@@ -52,8 +48,9 @@ cpdef groupby(long[:] groups):
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.initializedcheck(False)
cdef long* group_mapping(long* groups, size_t length, size_t* max_g) nogil:
cdef long *res_ptr = <long*>malloc(length*sizeof(long))
cpdef np.ndarray[int, ndim=1] group_mapping(long[:] groups):
cdef size_t length = groups.shape[0]
cdef np.ndarray[int, ndim=1] res= zeros(length, dtype=int)
cdef cpp_map[long, long] current_hold
cdef long curr_tag
cdef long running_tag = -1
......@@ -65,218 +62,9 @@ cdef long* group_mapping(long* groups, size_t length, size_t* max_g) nogil:
it = current_hold.find(curr_tag)
if it == current_hold.end():
running_tag += 1
res_ptr[i] = running_tag
res[i] = running_tag
current_hold[curr_tag] = running_tag
else:
res_ptr[i] = deref(it).second
max_g[0] = running_tag
return res_ptr
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
@cython.initializedcheck(False)
cdef double* agg_sum(long* groups, size_t max_g, double* x, size_t length, size_t width) nogil:
cdef double* res_ptr = <double*>malloc((max_g+1)*width*sizeof(double))
cdef size_t i
cdef size_t j
cdef size_t loop_idx1
cdef size_t loop_idx2
cdef long curr
for i in range((max_g+1)*width):
res_ptr[i] = 0.
for i in range(length):
loop_idx1 = i*width
loop_idx2 = groups[i]*width
for j in range(width):
res_ptr[loop_idx2 + j] += x[loop_idx1 + j]
return res_ptr
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
@cython.initializedcheck(False)
cdef double* agg_abssum(long* groups, size_t max_g, double* x, size_t length, size_t width) nogil:
cdef double* res_ptr = <double*>malloc((max_g+1)*width*sizeof(double))
cdef size_t i
cdef size_t j
cdef size_t loop_idx1
cdef size_t loop_idx2
cdef long curr
for i in range((max_g+1)*width):
res_ptr[i] = 0.
for i in range(length):
loop_idx1 = i*width
loop_idx2 = groups[i]*width
for j in range(width):
res_ptr[loop_idx2 + j] += fabs(x[loop_idx1 + j])
return res_ptr
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
@cython.initializedcheck(False)
cdef double* agg_mean(long* groups, size_t max_g, double* x, size_t length, size_t width) nogil:
cdef double* res_ptr = <double*>malloc((max_g+1)*width*sizeof(double))
cdef long* bin_count_ptr = <long*>malloc((max_g+1)*sizeof(long))
cdef size_t i
cdef size_t j
cdef size_t loop_idx1
cdef size_t loop_idx2
cdef long curr
try:
for i in range((max_g+1)*width):
res_ptr[i] = 0.
for i in range(max_g+1):
bin_count_ptr[i] = 0
for i in range(length):
loop_idx1 = i*width
loop_idx2 = groups[i]*width
for j in range(width):
res_ptr[loop_idx2 + j] += x[loop_idx1 + j]
bin_count_ptr[groups[i]] += 1
for i in range(max_g+1):
curr = bin_count_ptr[i]
loop_idx1 = i*width
for j in range(width):
res_ptr[loop_idx1 + j] /= curr
finally:
free(bin_count_ptr)
return res_ptr
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.cdivision(True)
@cython.initializedcheck(False)
cdef double* agg_std(long* groups, size_t max_g, double* x, size_t length, size_t width, long ddof=1) nogil:
cdef double* running_sum_square_ptr = <double*>malloc((max_g+1)*width*sizeof(double))
cdef double* running_sum_ptr = <double*>malloc((max_g+1)*width*sizeof(double))
cdef long* bin_count_ptr = <long*>malloc((max_g+1)*sizeof(long))
cdef size_t i
cdef size_t j
cdef size_t loop_idx1
cdef size_t loop_idx2
cdef long curr
cdef double raw_value
try:
for i in range((max_g+1)*width):
running_sum_square_ptr[i] = 0.
running_sum_ptr[i] = 0.
for i in range(max_g+1):
bin_count_ptr[i] = 0
for i in range(length):
loop_idx1 = i * width
loop_idx2 = groups[i] * width
for j in range(width):
raw_value = x[loop_idx1 + j]
running_sum_ptr[loop_idx2 + j] += raw_value
running_sum_square_ptr[loop_idx2 + j] += raw_value * raw_value
bin_count_ptr[groups[i]] += 1
for i in range(max_g+1):
curr = bin_count_ptr[i]
loop_idx1 = i * width
for j in range(width):
loop_idx2 = loop_idx1 + j
running_sum_square_ptr[loop_idx2] = sqrt((running_sum_square_ptr[loop_idx2] - running_sum_ptr[loop_idx2] * running_sum_ptr[loop_idx2] / curr) / (curr - ddof))
finally:
free(running_sum_ptr)
free(bin_count_ptr)
return running_sum_square_ptr
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.initializedcheck(False)
cpdef np.ndarray[double, ndim=2] transform(long[:] groups, double[:, :] x, str func):
cdef size_t length = x.shape[0]
cdef size_t width = x.shape[1]
cdef size_t* max_g = <size_t*>malloc(1*sizeof(size_t))
cdef long* mapped_groups = group_mapping(&groups[0], length, max_g)
cdef double* res_data_ptr = <double*>malloc(length*width*sizeof(double))
cdef double* value_data_ptr
cdef np.ndarray[double, ndim=2] res
cdef size_t i
cdef size_t j
cdef size_t loop_idx1
cdef size_t loop_idx2
x = np.ascontiguousarray(x)
try:
if func == 'mean':
value_data_ptr = agg_mean(mapped_groups, max_g[0], &x[0, 0], length, width)
elif func == 'std':
value_data_ptr = agg_std(mapped_groups, max_g[0], &x[0, 0], length, width, ddof=1)
elif func == 'sum':
value_data_ptr = agg_sum(mapped_groups, max_g[0], &x[0, 0], length, width)
elif func =='abssum':
value_data_ptr = agg_abssum(mapped_groups, max_g[0], &x[0, 0], length, width)
else:
raise ValueError('({0}) is not recognized as valid functor'.format(func))
res[i] = deref(it).second
with nogil:
for i in range(length):
loop_idx1 = i*width
loop_idx2 = mapped_groups[i] * width
for j in range(width):
res_data_ptr[loop_idx1 + j] = value_data_ptr[loop_idx2 + j]
finally:
free(value_data_ptr)
free(mapped_groups)
free(max_g)
res = np.PyArray_SimpleNewFromData(2, [length, width], np.NPY_FLOAT64, res_data_ptr)
PyArray_ENABLEFLAGS(res, np.NPY_OWNDATA)
return res
@cython.boundscheck(False)
@cython.wraparound(False)
@cython.initializedcheck(False)
cpdef np.ndarray[double, ndim=2] aggregate(long[:] groups, double[:, :] x, str func):
cdef size_t length = x.shape[0]
cdef size_t width = x.shape[1]
cdef size_t* max_g = <size_t*>malloc(1*sizeof(size_t))
cdef long* mapped_groups = group_mapping(&groups[0], length, max_g)
cdef double* value_data_ptr
cdef np.ndarray[double, ndim=2] res
x = np.ascontiguousarray(x)
try:
if func == 'mean':
value_data_ptr = agg_mean(mapped_groups, max_g[0], &x[0, 0], length, width)
elif func == 'std':
value_data_ptr = agg_std(mapped_groups, max_g[0], &x[0, 0], length, width, ddof=1)
elif func == 'sum':
value_data_ptr = agg_sum(mapped_groups, max_g[0], &x[0, 0], length, width)
elif func =='abssum':
value_data_ptr = agg_abssum(mapped_groups, max_g[0], &x[0, 0], length, width)
else:
raise ValueError('({0}) is not recognized as valid functor'.format(func))
res = np.PyArray_SimpleNewFromData(2, [max_g[0]+1, width], np.NPY_FLOAT64, value_data_ptr)
PyArray_ENABLEFLAGS(res, np.NPY_OWNDATA)
finally:
free(mapped_groups)
free(max_g)
return res
\ No newline at end of file
alphamind/benchmarks/benchmarks.py
View file @ bbb01231
......@@ -19,33 +19,33 @@ from alphamind.benchmarks.settlement.simplesettle import benchmark_simple_settle
if __name__ == '__main__':
# benchmark_neutralize(3000, 10, 1000)
# benchmark_neutralize_with_groups(3000, 10, 1000, 30)
# benchmark_neutralize(30, 3, 50000)
# benchmark_neutralize_with_groups(30, 3, 50000, 3)
# benchmark_neutralize(50000, 50, 20)
# benchmark_neutralize_with_groups(50000, 50, 20, 50)
# benchmark_standardize(3000, 10, 1000)
# benchmark_standardize_with_group(3000, 10, 1000, 30)
# benchmark_standardize(30, 10, 50000)
# benchmark_standardize_with_group(30, 10, 5000, 5)
# benchmark_standardize(50000, 50, 20)
# benchmark_standardize_with_group(50000, 50, 20, 50)
# benchmark_winsorize_normal(3000, 10, 1000)
# benchmark_winsorize_normal_with_group(3000, 10, 1000, 30)
# benchmark_winsorize_normal(30, 10, 50000)
# benchmark_winsorize_normal_with_group(30, 10, 5000, 5)
# benchmark_winsorize_normal(50000, 50, 20)
# benchmark_winsorize_normal_with_group(50000, 50, 20, 50)
benchmark_neutralize(3000, 10, 1000)
benchmark_neutralize_with_groups(3000, 10, 1000, 30)
benchmark_neutralize(30, 3, 50000)
benchmark_neutralize_with_groups(30, 3, 50000, 3)
benchmark_neutralize(50000, 50, 20)
benchmark_neutralize_with_groups(50000, 50, 20, 50)
benchmark_standardize(3000, 10, 1000)
benchmark_standardize_with_group(3000, 10, 1000, 30)
benchmark_standardize(30, 10, 50000)
benchmark_standardize_with_group(30, 10, 5000, 5)
benchmark_standardize(50000, 50, 20)
benchmark_standardize_with_group(50000, 50, 20, 50)
benchmark_winsorize_normal(3000, 10, 1000)
benchmark_winsorize_normal_with_group(3000, 10, 1000, 30)
benchmark_winsorize_normal(30, 10, 50000)
benchmark_winsorize_normal_with_group(30, 10, 5000, 5)
benchmark_winsorize_normal(50000, 50, 20)
benchmark_winsorize_normal_with_group(50000, 50, 20, 50)
benchmark_build_rank(3000, 1000, 300)
benchmark_build_rank_with_group(3000, 1000, 10, 30)
benchmark_build_rank(30, 50000, 3)
benchmark_build_rank_with_group(30, 50000, 1, 3)
benchmark_build_rank(50000, 20, 3000)
benchmark_build_rank_with_group(50000, 20, 10, 300)
# benchmark_simple_settle(3000, 10, 1000)
# benchmark_simple_settle_with_group(3000, 10, 1000, 30)
# benchmark_simple_settle(30, 10, 50000)
# benchmark_simple_settle_with_group(30, 10, 50000, 5)
# benchmark_simple_settle(50000, 50, 20)
# benchmark_simple_settle_with_group(50000, 50, 20, 50)
benchmark_simple_settle(3000, 10, 1000)
benchmark_simple_settle_with_group(3000, 10, 1000, 30)
benchmark_simple_settle(30, 10, 50000)
benchmark_simple_settle_with_group(30, 10, 50000, 5)
benchmark_simple_settle(50000, 50, 20)
benchmark_simple_settle_with_group(50000, 50, 20, 50)
alphamind/data/standardize.py
View file @ bbb01231
......@@ -6,15 +6,14 @@ Created on 2017-4-25
"""
import numpy as np
from numpy import ascontiguousarray
from alphamind.aggregate import transform
from alphamind.aggregate import group_mapping
from alphamind.impl import transform
def standardize(x: np.ndarray, groups: np.ndarray=None) -> np.ndarray:
x = ascontiguousarray(x)
if groups is not None:
groups = group_mapping(groups)
mean_values = transform(groups, x, 'mean')
std_values = transform(groups, x, 'std')
......
alphamind/data/winsorize.py
View file @ bbb01231
......@@ -6,14 +6,14 @@ Created on 2017-4-25
"""
import numpy as np
from numpy import ascontiguousarray
from alphamind.aggregate import transform
from alphamind.aggregate import group_mapping
from alphamind.impl import transform
def winsorize_normal(x: np.ndarray, num_stds: int=3, groups: np.ndarray=None) -> np.ndarray:
x = ascontiguousarray(x)
if groups is not None:
groups = group_mapping(groups)
mean_values = transform(groups, x, 'mean')
std_values = transform(groups, x, 'std')
else:
......
alphamind/settlement/simplesettle.py
View file @ bbb01231
......@@ -6,7 +6,8 @@ Created on 2017-4-28
"""
import numpy as np
from alphamind.aggregate import aggregate
from alphamind.aggregate import group_mapping
from alphamind.impl import aggregate
def simple_settle(weights: np.ndarray, ret_series: np.ndarray, groups: np.ndarray=None) -> np.ndarray:
......@@ -16,6 +17,7 @@ def simple_settle(weights: np.ndarray, ret_series: np.ndarray, groups: np.ndarra
ret_mat = (ret_series * weights.T).T
if groups is not None:
groups = group_mapping(groups)
return aggregate(groups, ret_mat, 'sum')
else:
return ret_mat.sum(axis=0)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment