Tuesday, 21 June 2022

# Tutorial on Scipy intro
# SciPy is a scientific computation library above NumPy.
# SciPy stands for Scientific Python.
# Ref w3schools.com


from scipy import constants
print(constants.liter) # 0.001
print(constants.pi) # 3.141592653589793


import scipy
print(scipy.__version__) # 1.8.1


from scipy import constants
print(constants.yotta)    #1e+24
print(constants.zetta)    #1e+21
print(constants.exa)      #1e+18
print(constants.peta)     #1000000000000000.0
print(constants.tera)     #1000000000000.0
print(constants.giga)     #1000000000.0
print(constants.mega)     #1000000.0
print(constants.kilo)     #1000.0
print(constants.hecto)    #100.0
print(constants.deka)     #10.0
print(constants.deci)     #0.1
print(constants.centi)    #0.01
print(constants.milli)    #0.001
print(constants.micro)    #1e-06
print(constants.nano)     #1e-09
print(constants.pico)     #1e-12
print(constants.femto)    #1e-15
print(constants.atto)     #1e-18
print(constants.zepto)    #1e-21
#
# # Root of an equation ..optimizer
from scipy.optimize import root
from math import cos

def eqn(x):
  return x + cos(x)
myroot = root(eqn, 0)
print(myroot.x)         #[-0.73908513]

#Sparse Matrix to avoid unused elements
import numpy as np
from scipy.sparse import csr_matrix
arr = np.array([0, 0, 0, 0, 0, 1, 1, 0, 2])
print(csr_matrix(arr))
'''
  (0, 5)   1
  (0, 6)   1
  (0, 8)   2
'''

import numpy as np
from scipy.sparse import csr_matrix

arr = np.array([[0, 0, 0], [0, 0, 1], [1, 0, 2]])

mat = csr_matrix(arr)
mat.eliminate_zeros()

print(mat)
'''
 (0, 5)    1
  (0, 6)   1
  (0, 8)   2
  (1, 2)   1
  (2, 0)   1
  (2, 2)   2
'''


#SCIPY sIGNIFICANt aspects

import numpy as np
from scipy.stats import ttest_ind

x1 = np.random.normal(size=50)
x2 = np.random.normal(size=50)
#T-Test
print( ttest_ind(x1, x2))
 #Ttest_indResult(statistic=-0.4602568443314745, pvalue=0.6463514741682714)

# pvalue printing
print(ttest_ind(x1, x2).pvalue) #0.3861625359272315


import numpy as np
from scipy.stats import kstest
x = np.random.normal(size=50)
print(kstest(x, 'norm'))
#KstestResult(statistic=0.07440265161334647, pvalue=0.9254263997475219)


#Statistical Distribuition

import numpy as np
from scipy.stats import describe
x = np.random.normal(size=50)
print(describe(x))
# DescribeResult(nobs=50, minmax=(-2.252010033738992, 1.694245094326367),
# mean=-0.34129189546033056, variance=0.7245922182755077, skewness=0.4445284516357619,
# kurtosis=0.11748436830854025)

# Normality Tests (Skewness and Kurtosis)
'''
Skewness:
A measure of symmetry in data, For normal distributions it is 0.
If it is negative, it means the data is skewed left.
If it is positive it means the data is skewed right.
'''
import numpy as np
from scipy.stats import skew, kurtosis

x = np.random.normal(size=50)

print(skew(x))            #0.25667677411004536   skewed right
print(skew(x))            #-0.49188030505407804  skewed left

'''
Kurtosis:
A measure of whether the data is heavy or lightly tailed with 
respect  to a normal distribution.
Positive kurtosis : heavily tailed.
Negative kurtosis : lightly tailed.
'''
print(kurtosis(x))        #-0.37504350194077585 lightly tailed
print(kurtosis(x))        #0.010624259343263276  heavily tailed

from scipy.stats import normaltest
# To check normal distribution of data
print(normaltest(x)) #NormaltestResult(statistic=7.187162674258679,
# pvalue=0.027499668274458256)

Tuesday, 31 May 2022

Let us how Visualization helps in Exploratory Data Analysis or
An EDA Case Study

Lab exercise for Course work

1.  Python Environment setup and Essentials.
2.  Mathematical computing with Python (NumPy).
3.  Scientific Computing with Python (SciPy).
4.  Data Manipulation with Pandas.
5.  Prediction using Scikit-Learn
6.  Data Visualization in python using matplotlib

Q&A

Difference among between data Analysis, data Analytics and Data Science

Data Analytics	Data Analysis	Data Science
Super set Process of Data Analysis	Sub Set Process of Data Analytics	Set Process
Past and Present study and future e prediction	Past discovery
What will happen in future	What happened
Tools: Tableau, Zoho Analytics,Talend, Hadoop, Xplenty, Kafka,Python,R-Language, Cassandra, MongoDB, HPCC, Spark, Datawrapper, PowerBI etc.	Python, R, PowerBI, Tableau, Excel
Data analytics predicts ‘what will happen next or what is going to be next?’	Data analysis is actually studying past data to understand ‘what happened?’
All analysis and prediction. Data analytics life cycle consists of Business Case Evaluation, Data Identification, Data Acquisition & Filtering, Data Extraction, Data Validation & Cleansing, Data Aggregation & Representation, Data Analysis, Data Visualization, and Utilization of Analysis Results	DA involves Querying, wrangling, statistical modelling, analysis and visualization.	DS involves data sourcing, cleansing, modelling, result evaluation and result testing and deployment

Data Analytics and data Science

Data Science is a combination of multiple disciplines – Mathematics, Statistics, Computer Science, Information Science, Machine Learning, and Artificial Intelligence.

2. Difference between mutable and immutable objects in python

In Python, Built-in projects like int, float, bool, string, tuple are immutable.

In Python, Built-in projects like list, dict, set are mutable.

Friday, 27 May 2022

Distribution with Seaborn and Random

Distributions

This can be visually with the help of seaborn and numpy random.

Look at the following sample code:


from numpy import random
import matplotlib.pyplot as plt
import seaborn as sns

# normal Distribuition
sns.distplot(random.normal(size=1000), hist=False)
plt.title(' Normal Distribuition')
plt.savefig('nd.jpg')
plt.show()

#binomial
sns.distplot(random.binomial(n=10, p=0.5, size=100), hist=True, kde=False)
plt.title(' Binomial Distribuition')
plt.savefig('bnd.jpg')
plt.show()


# poisson distribuition
sns.distplot(random.poisson(lam=2, size=100), kde=False)
plt.title('Poisson Distribuition')
plt.savefig('pois.jpg')
plt.show()


# All together
sns.distplot(random.normal(loc=50, scale=5, size=100), hist=False, label='normal')
sns.distplot(random.binomial(n=100, p=0.5, size=100), hist=False, label='binomial')
plt.title('Comparisosn between Normal and Binomial Distribuition')
plt.savefig('n-bnd.jpg')
plt.show()

sns.distplot(random.poisson(lam=2, size=100), kde=False)
plt.title('Poisson Distribuition')
plt.savefig('poisson.jpg')
plt.show()


sns.distplot(random.uniform(size=100), hist=False)
plt.title('Uniform Distribuition')
plt.savefig('uniform.jpg')
plt.show()

sns.distplot(random.logistic(size=1000), hist=False)
plt.title('Logistic Distribuition ')
plt.savefig('logistic.jpg')
plt.show()

sns.distplot(random.multinomial(n=6, pvals=[1/6, 1/6, 1/6, 1/6, 1/6, 1/6]))
plt.title('Multimodal Distribuition ')
plt.savefig('multimodal.jpg')
plt.show()


sns.distplot(random.exponential(size=1000), hist=False)
plt.title('Exponential Distribuition ')
plt.savefig('exponential.jpg')
plt.show()


sns.distplot(random.chisquare(df=1, size=1000), hist=False)
plt.title('Chisquare Distribuition ')
plt.savefig('chisquare.jpg')
plt.show()

sns.distplot(random.rayleigh(size=1000), hist=False)
plt.title('Rayleigh Distribuition')
plt.savefig('Rayleigh.jpg')
plt.show()

sns.distplot(random.pareto(a=2, size=1000), kde=False)
plt.title('Pareto Distribuition')
plt.savefig('Pareto.jpg')
plt.show()


x = random.zipf(a=2, size=1000)
sns.distplot(x[x<10], kde=False)
plt.title('Zipf Distribuition')
plt.savefig('zipf.jpg')
plt.show()