import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
#Generate arithmetic progression corresponding to p
xx = np.linspace(0, 1, 50) #Start value 0, end value 1, number of elements 50
plt.figure(figsize=(10, 8))
#Calculate each index
gini = [2 * x * (1-x) for x in xx]
entropy = [-x * np.log2(x) - (1-x) * np.log2(1-x) for x in xx]
misclass = [1 - max(x, 1-x) for x in xx]
#Show graph
plt.plot(xx, gini, label='gini', lw=3, color='b')
plt.plot(xx, entropy, label='entropy', lw=3, color='r')
plt.plot(xx, misclass, label='misclass', lw=3, color='g')
plt.xlabel('p', fontsize=15)
plt.ylabel('criterion', fontsize=15)
plt.legend(fontsize=15)
plt.xticks(fontsize=12)
plt.yticks(fontsize=12)
plt.grid()
#Generate arithmetic progression corresponding to p
xx = np.linspace(0, 1, 50) #Start value 0, end value 1, number of elements 50
plt.figure(figsize=(10, 8))
#Calculate each index
gini = [2 * x * (1-x) for x in xx]
entropy = [(x * np.log((1-x)/x) - np.log(1-x)) / (np.log(2)) for x in xx]
entropy_scaled = [(x * np.log((1-x)/x) - np.log(1-x)) / (2*np.log(2)) for x in xx]
misclass = [1 - max(x, 1-x) for x in xx]
#Show graph
plt.plot(xx, gini, label='gini', lw=3, color='b')
plt.plot(xx, entropy, label='entropy', lw=3, color='r', linestyle='dashed')
plt.plot(xx, entropy_scaled, label='entropy(scaled)', lw=3, color='r')
plt.plot(xx, misclass, label='misclass', lw=3, color='g')
plt.xlabel('p', fontsize=15)
plt.ylabel('criterion', fontsize=15)
plt.legend(fontsize=15)
plt.xticks(fontsize=12)
plt.yticks(fontsize=12)
plt.grid()
#Gini purity of parent node
IGg_p = 2 * 1/2 * (1-(1/2))
#Gini purity of child node A
IGg_A_l = 2 * 3/4 * (1-(3/4)) #left
IGg_A_r = 2 * 1/4 * (1-(1/4)) #right
#Gini impureness of child node B
IGg_B_l = 2 * 2/6 * (1-(2/6)) #left
IGg_B_r = 2 * 2/2 * (1-(2/2)) #right
#Information gain of each branch
IG_gini_A = IGg_p - 4/8 * IGg_A_l - 4/8 * IGg_A_r
IG_gini_B = IGg_p - 6/8 * IGg_B_l - 2/8 * IGg_B_r
print("Information gain of branch A:", IG_gini_A)
print("Information gain of branch B:", IG_gini_B)
#Parent node entropy
IGe_p = (4/8 * np.log((1-4/8)/(4/8)) - np.log(1-4/8)) / (np.log(2))
#Entropy of child node A
IGe_A_l = (3/4 * np.log((1-3/4)/(3/4)) - np.log(1-3/4)) / (np.log(2)) #left
IGe_A_r = (1/4 * np.log((1-1/4)/(1/4)) - np.log(1-1/4)) / (np.log(2)) #right
#Entropy of child node B
IGe_B_l = (2/6 * np.log((1-2/6)/(2/6)) - np.log(1-2/6)) / (np.log(2)) #left
IGe_B_r = (2/2 * np.log((1-2/2+1e-7)/(2/2)) - np.log(1-2/2+1e-7)) / (np.log(2)) #right,+1e-7 adds a small value to avoid division by zero
#Information gain of each branch
IG_entropy_A = IGe_p - 4/8 * IGe_A_l - 4/8 * IGe_A_r
IG_entropy_B = IGe_p - 6/8 * IGe_B_l - 2/8 * IGe_B_r
print("Information gain of branch A:", IG_entropy_A)
print("Information gain of branch B:", IG_entropy_B)
#Misclassification rate of parent node
IGm_p = 1 - np.maximum(4/8, 1-4/8)
#Misclassification rate of child node A
IGm_A_l = 1 - np.maximum(3/4, 1-3/4) #left
IGm_A_r = 1 - np.maximum(1/4, 1-1/4) #right
#Misclassification rate of child node B
IGm_B_l = 1 - np.maximum(2/6, 1-2/6) #left
IGm_B_r = 1 - np.maximum(2/2, 1-2/2) #right
#Information gain of each branch
IG_misclass_A = IGm_p - 4/8 * IGm_A_l - 4/8 * IGm_A_r
IG_misclass_B = IGm_p - 6/8 * IGm_B_l - 2/8 * IGm_B_r
print("Information gain of branch A:", IG_misclass_A)
print("Information gain of branch B:", IG_misclass_B)
Classification condition A | Classification condition B | |
---|---|---|
Gini Impure | 0.125 | 0.167 |
Entropy | 0.189 | 0.311 |
Misclassification rate | 0.250 | 0.250 |
Recommended Posts