using numba instead of cython to simplify codes

alphamind/aggregate.pyx

@@ -7,20 +7,16 @@ Created on 2017-4-26
 """
 
 import numpy as np
+from numpy import zeros
+from numpy import max as nmax
 cimport numpy as np
 cimport cython
 from libc.math cimport sqrt
 from libc.math cimport fabs
-from libc.stdlib cimport malloc
-from libc.stdlib cimport free
 from libcpp.vector cimport vector as cpp_vector
 from libcpp.unordered_map cimport unordered_map as cpp_map
 from cython.operator cimport dereference as deref
 
-np.import_array()
-
-cdef extern from "numpy/arrayobject.h":
-    void PyArray_ENABLEFLAGS(np.ndarray arr, int flags)
 
 ctypedef long long int64_t
 
@@ -52,8 +48,9 @@ cpdef groupby(long[:] groups):
 @cython.boundscheck(False)
 @cython.wraparound(False)
 @cython.initializedcheck(False)
-cdef long* group_mapping(long* groups, size_t length, size_t* max_g) nogil:
-    cdef long *res_ptr = <long*>malloc(length*sizeof(long))
+cpdef np.ndarray[int, ndim=1] group_mapping(long[:] groups):
+    cdef size_t length = groups.shape[0]
+    cdef np.ndarray[int, ndim=1] res= zeros(length, dtype=int)
     cdef cpp_map[long, long] current_hold
     cdef long curr_tag
     cdef long running_tag = -1
@@ -65,218 +62,9 @@ cdef long* group_mapping(long* groups, size_t length, size_t* max_g) nogil:
         it = current_hold.find(curr_tag)
         if it == current_hold.end():
             running_tag += 1
-            res_ptr[i] = running_tag
+            res[i] = running_tag
             current_hold[curr_tag] = running_tag
         else:
-            res_ptr[i] = deref(it).second
-
-    max_g[0] = running_tag
-    return res_ptr
-
-
-@cython.boundscheck(False)
-@cython.wraparound(False)
-@cython.cdivision(True)
-@cython.initializedcheck(False)
-cdef double* agg_sum(long* groups, size_t max_g, double* x, size_t length, size_t width) nogil:
-    cdef double* res_ptr = <double*>malloc((max_g+1)*width*sizeof(double))
-    cdef size_t i
-    cdef size_t j
-    cdef size_t loop_idx1
-    cdef size_t loop_idx2
-    cdef long curr
-
-    for i in range((max_g+1)*width):
-        res_ptr[i] = 0.
-
-    for i in range(length):
-        loop_idx1 = i*width
-        loop_idx2 = groups[i]*width
-        for j in range(width):
-            res_ptr[loop_idx2 + j] += x[loop_idx1 + j]
-    return res_ptr
-
-
-@cython.boundscheck(False)
-@cython.wraparound(False)
-@cython.cdivision(True)
-@cython.initializedcheck(False)
-cdef double* agg_abssum(long* groups, size_t max_g, double* x, size_t length, size_t width) nogil:
-    cdef double* res_ptr = <double*>malloc((max_g+1)*width*sizeof(double))
-    cdef size_t i
-    cdef size_t j
-    cdef size_t loop_idx1
-    cdef size_t loop_idx2
-    cdef long curr
-
-    for i in range((max_g+1)*width):
-        res_ptr[i] = 0.
-
-    for i in range(length):
-        loop_idx1 = i*width
-        loop_idx2 = groups[i]*width
-        for j in range(width):
-            res_ptr[loop_idx2 + j] += fabs(x[loop_idx1 + j])
-    return res_ptr
-
-
-@cython.boundscheck(False)
-@cython.wraparound(False)
-@cython.cdivision(True)
-@cython.initializedcheck(False)
-cdef double* agg_mean(long* groups, size_t max_g, double* x, size_t length, size_t width) nogil:
-    cdef double* res_ptr = <double*>malloc((max_g+1)*width*sizeof(double))
-    cdef long* bin_count_ptr = <long*>malloc((max_g+1)*sizeof(long))
-    cdef size_t i
-    cdef size_t j
-    cdef size_t loop_idx1
-    cdef size_t loop_idx2
-    cdef long curr
-
-    try:
-        for i in range((max_g+1)*width):
-            res_ptr[i] = 0.
-
-        for i in range(max_g+1):
-            bin_count_ptr[i] = 0
-
-        for i in range(length):
-            loop_idx1 = i*width
-            loop_idx2 = groups[i]*width
-            for j in range(width):
-                res_ptr[loop_idx2 + j] += x[loop_idx1 + j]
-            bin_count_ptr[groups[i]] += 1
-
-        for i in range(max_g+1):
-            curr = bin_count_ptr[i]
-            loop_idx1 = i*width
-            for j in range(width):
-                res_ptr[loop_idx1 + j] /= curr
-    finally:
-        free(bin_count_ptr)
-    return res_ptr
-
-
-@cython.boundscheck(False)
-@cython.wraparound(False)
-@cython.cdivision(True)
-@cython.initializedcheck(False)
-cdef double* agg_std(long* groups, size_t max_g, double* x, size_t length, size_t width, long ddof=1) nogil:
-    cdef double* running_sum_square_ptr = <double*>malloc((max_g+1)*width*sizeof(double))
-    cdef double* running_sum_ptr = <double*>malloc((max_g+1)*width*sizeof(double))
-    cdef long* bin_count_ptr = <long*>malloc((max_g+1)*sizeof(long))
-    cdef size_t i
-    cdef size_t j
-    cdef size_t loop_idx1
-    cdef size_t loop_idx2
-    cdef long curr
-    cdef double raw_value
-
-    try:
-        for i in range((max_g+1)*width):
-            running_sum_square_ptr[i] = 0.
-            running_sum_ptr[i] = 0.
-
-        for i in range(max_g+1):
-            bin_count_ptr[i] = 0
-
-
-        for i in range(length):
-            loop_idx1 = i * width
-            loop_idx2 = groups[i] * width
-
-            for j in range(width):
-                raw_value = x[loop_idx1 + j]
-                running_sum_ptr[loop_idx2 + j] += raw_value
-                running_sum_square_ptr[loop_idx2 + j] += raw_value * raw_value
-            bin_count_ptr[groups[i]] += 1
-
-        for i in range(max_g+1):
-            curr = bin_count_ptr[i]
-            loop_idx1 = i * width
-            for j in range(width):
-                loop_idx2 = loop_idx1 + j
-                running_sum_square_ptr[loop_idx2] = sqrt((running_sum_square_ptr[loop_idx2] - running_sum_ptr[loop_idx2] * running_sum_ptr[loop_idx2] / curr) / (curr - ddof))
-    finally:
-        free(running_sum_ptr)
-        free(bin_count_ptr)
-    return running_sum_square_ptr
-
-
-@cython.boundscheck(False)
-@cython.wraparound(False)
-@cython.initializedcheck(False)
-cpdef np.ndarray[double, ndim=2] transform(long[:] groups, double[:, :] x, str func):
-    cdef size_t length = x.shape[0]
-    cdef size_t width = x.shape[1]
-    cdef size_t* max_g = <size_t*>malloc(1*sizeof(size_t))
-    cdef long* mapped_groups = group_mapping(&groups[0], length, max_g)
-    cdef double* res_data_ptr = <double*>malloc(length*width*sizeof(double))
-    cdef double* value_data_ptr
-    cdef np.ndarray[double, ndim=2] res
-    cdef size_t i
-    cdef size_t j
-    cdef size_t loop_idx1
-    cdef size_t loop_idx2
-
-    x = np.ascontiguousarray(x)
-
-    try:
-        if func == 'mean':
-            value_data_ptr = agg_mean(mapped_groups, max_g[0], &x[0, 0], length, width)
-        elif func == 'std':
-            value_data_ptr = agg_std(mapped_groups, max_g[0], &x[0, 0], length, width, ddof=1)
-        elif func == 'sum':
-            value_data_ptr = agg_sum(mapped_groups, max_g[0], &x[0, 0], length, width)
-        elif func =='abssum':
-            value_data_ptr = agg_abssum(mapped_groups, max_g[0], &x[0, 0], length, width)
-        else:
-            raise ValueError('({0}) is not recognized as valid functor'.format(func))
+            res[i] = deref(it).second
 
-        with nogil:
-            for i in range(length):
-                loop_idx1 = i*width
-                loop_idx2 = mapped_groups[i] * width
-                for j in range(width):
-                    res_data_ptr[loop_idx1 + j] = value_data_ptr[loop_idx2 + j]
-    finally:
-        free(value_data_ptr)
-        free(mapped_groups)
-        free(max_g)
-    res = np.PyArray_SimpleNewFromData(2, [length, width], np.NPY_FLOAT64, res_data_ptr)
-    PyArray_ENABLEFLAGS(res, np.NPY_OWNDATA)
     return res
-
-
-@cython.boundscheck(False)
-@cython.wraparound(False)
-@cython.initializedcheck(False)
-cpdef np.ndarray[double, ndim=2] aggregate(long[:] groups, double[:, :] x, str func):
-    cdef size_t length = x.shape[0]
-    cdef size_t width = x.shape[1]
-    cdef size_t* max_g = <size_t*>malloc(1*sizeof(size_t))
-    cdef long* mapped_groups = group_mapping(&groups[0], length, max_g)
-    cdef double* value_data_ptr
-    cdef np.ndarray[double, ndim=2] res
-
-    x = np.ascontiguousarray(x)
-
-    try:
-        if func == 'mean':
-            value_data_ptr = agg_mean(mapped_groups, max_g[0], &x[0, 0], length, width)
-        elif func == 'std':
-            value_data_ptr = agg_std(mapped_groups, max_g[0], &x[0, 0], length, width, ddof=1)
-        elif func == 'sum':
-            value_data_ptr = agg_sum(mapped_groups, max_g[0], &x[0, 0], length, width)
-        elif func =='abssum':
-            value_data_ptr = agg_abssum(mapped_groups, max_g[0], &x[0, 0], length, width)
-        else:
-            raise ValueError('({0}) is not recognized as valid functor'.format(func))
-
-        res = np.PyArray_SimpleNewFromData(2, [max_g[0]+1, width], np.NPY_FLOAT64, value_data_ptr)
-        PyArray_ENABLEFLAGS(res, np.NPY_OWNDATA)
-    finally:
-        free(mapped_groups)
-        free(max_g)
-
-    return res
\ No newline at end of file

alphamind/benchmarks/benchmarks.py

@@ -19,33 +19,33 @@ from alphamind.benchmarks.settlement.simplesettle import benchmark_simple_settle
 
 if __name__ == '__main__':
 
-    # benchmark_neutralize(3000, 10, 1000)
-    # benchmark_neutralize_with_groups(3000, 10, 1000, 30)
-    # benchmark_neutralize(30, 3, 50000)
-    # benchmark_neutralize_with_groups(30, 3, 50000, 3)
-    # benchmark_neutralize(50000, 50, 20)
-    # benchmark_neutralize_with_groups(50000, 50, 20, 50)
-    # benchmark_standardize(3000, 10, 1000)
-    # benchmark_standardize_with_group(3000, 10, 1000, 30)
-    # benchmark_standardize(30, 10, 50000)
-    # benchmark_standardize_with_group(30, 10, 5000, 5)
-    # benchmark_standardize(50000, 50, 20)
-    # benchmark_standardize_with_group(50000, 50, 20, 50)
-    # benchmark_winsorize_normal(3000, 10, 1000)
-    # benchmark_winsorize_normal_with_group(3000, 10, 1000, 30)
-    # benchmark_winsorize_normal(30, 10, 50000)
-    # benchmark_winsorize_normal_with_group(30, 10, 5000, 5)
-    # benchmark_winsorize_normal(50000, 50, 20)
-    # benchmark_winsorize_normal_with_group(50000, 50, 20, 50)
+    benchmark_neutralize(3000, 10, 1000)
+    benchmark_neutralize_with_groups(3000, 10, 1000, 30)
+    benchmark_neutralize(30, 3, 50000)
+    benchmark_neutralize_with_groups(30, 3, 50000, 3)
+    benchmark_neutralize(50000, 50, 20)
+    benchmark_neutralize_with_groups(50000, 50, 20, 50)
+    benchmark_standardize(3000, 10, 1000)
+    benchmark_standardize_with_group(3000, 10, 1000, 30)
+    benchmark_standardize(30, 10, 50000)
+    benchmark_standardize_with_group(30, 10, 5000, 5)
+    benchmark_standardize(50000, 50, 20)
+    benchmark_standardize_with_group(50000, 50, 20, 50)
+    benchmark_winsorize_normal(3000, 10, 1000)
+    benchmark_winsorize_normal_with_group(3000, 10, 1000, 30)
+    benchmark_winsorize_normal(30, 10, 50000)
+    benchmark_winsorize_normal_with_group(30, 10, 5000, 5)
+    benchmark_winsorize_normal(50000, 50, 20)
+    benchmark_winsorize_normal_with_group(50000, 50, 20, 50)
     benchmark_build_rank(3000, 1000, 300)
     benchmark_build_rank_with_group(3000, 1000, 10, 30)
     benchmark_build_rank(30, 50000, 3)
     benchmark_build_rank_with_group(30, 50000, 1, 3)
     benchmark_build_rank(50000, 20, 3000)
     benchmark_build_rank_with_group(50000, 20, 10, 300)
-    # benchmark_simple_settle(3000, 10, 1000)
-    # benchmark_simple_settle_with_group(3000, 10, 1000, 30)
-    # benchmark_simple_settle(30, 10, 50000)
-    # benchmark_simple_settle_with_group(30, 10, 50000, 5)
-    # benchmark_simple_settle(50000, 50, 20)
-    # benchmark_simple_settle_with_group(50000, 50, 20, 50)
+    benchmark_simple_settle(3000, 10, 1000)
+    benchmark_simple_settle_with_group(3000, 10, 1000, 30)
+    benchmark_simple_settle(30, 10, 50000)
+    benchmark_simple_settle_with_group(30, 10, 50000, 5)
+    benchmark_simple_settle(50000, 50, 20)
+    benchmark_simple_settle_with_group(50000, 50, 20, 50)

alphamind/data/standardize.py

@@ -6,15 +6,14 @@ Created on 2017-4-25
 """
 
 import numpy as np
-from numpy import ascontiguousarray
-from alphamind.aggregate import transform
+from alphamind.aggregate import group_mapping
+from alphamind.impl import transform
 
 
 def standardize(x: np.ndarray, groups: np.ndarray=None) -> np.ndarray:
 
-    x = ascontiguousarray(x)
-
     if groups is not None:
+        groups = group_mapping(groups)
         mean_values = transform(groups, x, 'mean')
         std_values = transform(groups, x, 'std')
 

alphamind/data/winsorize.py

@@ -6,14 +6,14 @@ Created on 2017-4-25
 """
 
 import numpy as np
-from numpy import ascontiguousarray
-from alphamind.aggregate import transform
+from alphamind.aggregate import group_mapping
+from alphamind.impl import transform
 
 
 def winsorize_normal(x: np.ndarray, num_stds: int=3, groups: np.ndarray=None) -> np.ndarray:
-    x = ascontiguousarray(x)
 
     if groups is not None:
+        groups = group_mapping(groups)
         mean_values = transform(groups, x, 'mean')
         std_values = transform(groups, x, 'std')
     else:

alphamind/settlement/simplesettle.py

@@ -6,7 +6,8 @@ Created on 2017-4-28
 """
 
 import numpy as np
-from alphamind.aggregate import aggregate
+from alphamind.aggregate import group_mapping
+from alphamind.impl import aggregate
 
 
 def simple_settle(weights: np.ndarray, ret_series: np.ndarray, groups: np.ndarray=None) -> np.ndarray:
@@ -16,6 +17,7 @@ def simple_settle(weights: np.ndarray, ret_series: np.ndarray, groups: np.ndarra
 
     ret_mat = (ret_series * weights.T).T
     if groups is not None:
+        groups = group_mapping(groups)
         return aggregate(groups, ret_mat, 'sum')
     else:
         return ret_mat.sum(axis=0)