Use Cython with Jupyter Notebook
It's a note because I found that I could try Cython with Jupyter Notebook (iPython Notebook) more easily than I expected. Cython speeds up processing by compiling before execution and statically typing.
(The .ipynb file for this article has been uploaded to [here] on Github (https://github.com/matsuken92/Qiita_Contents/blob/master/General/Cython_test.ipynb).)
</ i> Environment
The environment I tried is as follows. I'm trying it on Mac and Anaconda. If you have Anaconda installed, no special preparation is required.
Python 3.5.1 |Anaconda custom (x86_64)| (default, Jun 15 2016, 16:14:02)
[GCC 4.2.1 Compatible Apple LLVM 4.2 (clang-425.0.28)] on darwin
IPython 5.0.0
</ i> Let's try it
Execute magic command for Cython compilation
#Allow cython files to be compiled on Jupyter Notebook
%load_ext Cython
Declare Cython functions
Write the cython code with %% cython at the beginning.
The example is used from Cython Tutorial Basics.
# ↓ -n <file name>By adding, it will be easier to check the file later.
%%cython -n test_cython_code
def fib(int n):
cdef int i
cdef double a=0.0, b=1.0
for i in range(n):
a, b = a+b, a
return a
def primes(int kmax):
cdef int n, k, i
cdef int p[1000]
result = []
if kmax > 1000:
kmax = 1000
k = 0
n = 2
while k < kmax:
i = 0
while i < k and n % p[i] != 0:
i += 1
if i == k:
p[k] = n
k += 1
result.append(n)
n += 1
return result
Try out
print(fib(90))
print(primes(20))
out
2.880067194370816e+18
[2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71]
did it!
</ i> Speed comparison with raw Python
Let's write the same process in raw Python and compare the execution times.
import numpy as np
#Python function for performance comparison
def pyfib(n):
a, b = 0.0, 1.0
for i in range(n):
a, b = a+b, a
return a
def pyprimes(kmax):
p = np.zeros(1000)
result = []
#Maximum number is 1000
if kmax > 1000:
kmax = 1000
k = 0
n = 2
while k < kmax:
i = 0
while i < k and n % p[i] != 0:
i += 1
if i == k:
p[k] = n
k += 1
result.append(n)
n += 1
return result
Fibonacci number
#Repeatedly generate and measure the 1000th Fibonacci number
%timeit fib(1000)
%timeit pyfib(1000)
cython is about 50 times faster!
out
1000000 loops, best of 3: 786 ns per loop
10000 loops, best of 3: 42.5 µs per loop
Extract prime numbers
%timeit primes(1000)
%timeit pyprimes(1000)
out
100 loops, best of 3: 2.12 ms per loop
1 loop, best of 3: 218 ms per loop
This calculation is about 100 times faster!
</ i> Try using it for Pandas apply
1000 integers
df = pd.DataFrame(np.arange(1, 10**4), columns=['num'] )
You can use it just by specifying the function in the apply function: blush:
%timeit df['fib'] = df.num.apply(fib)
%timeit df['pyfib'] = df.num.apply(pyfib)
out
10 loops, best of 3: 39.2 ms per loop
1 loop, best of 3: 2.02 s per loop
print(df.head())
out
num fib pyfib
0 1 1.0 1.0
1 2 1.0 1.0
2 3 2.0 2.0
3 4 3.0 3.0
4 5 5.0 5.0
The compiled cython file is stored in ~ / .ipython / cython.
If a file name was specified with %% cython -n <file name> at compile time, it is stored here with that file name.
</ i> Handle ndarray
#Create data
rd.seed(71)
n_data = 10**5
X = pd.DataFrame(rd.normal(size=3*n_data).reshape((n_data,3)), columns=["a", "b", "c"])
print(X.shape)
print(X.head())
out
(100000, 3)
a b c
0 -0.430603 -1.193928 -0.444299
1 0.489412 -0.451557 0.585696
2 1.177320 -0.965009 0.218278
3 -0.866144 -0.323006 1.412919
4 -0.712651 -1.362191 -1.705966
Write Cython code that takes an ndarray as an argument
%%cython -n sample_calc
import numpy as np
cimport numpy as np
cpdef np.ndarray[double] sample_calc(np.ndarray col_a, np.ndarray col_b, np.ndarray col_c):
#Type check for each column
assert (col_a.dtype == np.float and col_b.dtype == np.float and col_c.dtype == np.float)
#Check that the size of each column is the same
cdef Py_ssize_t n = len(col_c)
assert (len(col_a) == len(col_b) == n)
cdef np.ndarray[double] res = np.empty(n)
# (a-b)/Do the calculation c
for i in range(n):
res[i] = (col_a[i] - col_b[i])/col_c[i]
return res
Call from Python side
sample_calc(X.a.values, X.b.values, X.c.values)
out
array([-1.71804336, 1.60658332, 9.81468496, ..., -0.44683095,
0.46970409, -0.28352272])
#For comparison
def pysample_calc(col_a, col_b, col_c):
#Type check for each column
assert (col_a.dtype == np.float and col_b.dtype == np.float and col_c.dtype == np.float)
#Check that the size of each column is the same
n = len(col_c)
assert (len(col_a) == len(col_b) == n)
res = np.empty(n)
# (a-b)/Do the calculation c
for i in range(n):
res[i] = (col_a[i] - col_b[i])/col_c[i]
return res
%timeit sample_calc(X.a.values, X.b.values, X.c.values)
%timeit pysample_calc(X.a.values, X.b.values, X.c.values)
out
100 loops, best of 3: 16.7 ms per loop
10 loops, best of 3: 37.2 ms per loop
</ i> Calculate pi by Monte Carlo method
#Data generation
rd.seed(71)
n_data = 10**7
X2 = rd.random(size=(n_data,2)).astype(np.float)
X2.dtype
Definition of Cython function
%%cython -n calc_pi
import numpy as np
cimport numpy as np
cpdef np.ndarray[long] calc_pi(np.ndarray[double, ndim=2] data):
cdef Py_ssize_t n = len(data)
cdef np.ndarray[long] res = np.empty(n, dtype=np.int)
for i in range(n):
res[i] = 1 if (data[i,0]**2 + data[i,1]**2) < 1 else 0
return res
Python function for comparison
#Python function for comparison
def pycalc_pi(data):
n = len(data)
res = [1 if (data[i,0]**2 + data[i,1]**2) < 1 else 0 for i in range(n)]
return res
I will measure it.
%time calc_pi(X2)
%time pycalc_pi(X2)
out
CPU times: user 25.2 ms, sys: 5.98 ms, total: 31.2 ms
Wall time: 31.1 ms
CPU times: user 7.7 s, sys: 46.1 ms, total: 7.75 s
Wall time: 7.75 s
Cython is much faster!
#Check if the results are the same
np.all(res == respy)
correct!
out
True
#Calculation of pi
np.sum(res)/n_data*4
out
3.1413555999999998
Try drawing.
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns
from matplotlib.colors import LinearSegmentedColormap
sns.set(style="darkgrid", palette="muted", color_codes=True)
#draw
n_plot = 10**4 #Number of points to draw
plt.figure(figsize=(8,8))
plt.scatter(X2[:n_plot,0], X2[:n_plot,1], c=res[:n_plot], s=10)
You are properly judging the inside and outside of the circle.
reference
Cython Tutorial Basics http://omake.accense.com/static/doc-ja/cython/src/userguide/tutorial.html
O'Reilly "Cython" https://www.oreilly.co.jp/books/9784873117270/
pandas 0.18.1 documentation Enhancing Performance http://pandas.pydata.org/pandas-docs/stable/enhancingperf.html
Recommended Posts