Numpy Pandas Basics

Tutorial

Author

R Yeeshu Dhurandhar, Nipun Batra

Published

December 31, 2023

Open In Colab

# importing necessary libraries
import numpy as np
import pandas as pd
import time
import matplotlib.pyplot as plt
%matplotlib inline
%config InlineBackend.figure_format = 'retina'

Lists

# Creating lists
list_a = [1, 2, 3, 4, 5]
list_b = [6, 7, 8, 9, 10]

# Operations on lists
# Adding lists
list_sum = [a + b for a, b in zip(list_a, list_b)]
print("List Sum:", list_sum)

# Vector product using lists    
vector_product = [a * b for a, b in zip(list_a, list_b)]
print("Vector Product:", vector_product)

List Sum: [7, 9, 11, 13, 15]
Vector Product: [6, 14, 24, 36, 50]

Numpy Array

# Creating numpy arrays
numpy_array_a = np.array(list_a)
numpy_array_b = np.array(list_b)

# Operations on numpy arrays
# Adding numpy arrays
numpy_sum = numpy_array_a + numpy_array_b
print("Numpy Sum:", numpy_sum)

# Vector product using numpy arrays
numpy_vector_product = np.multiply(numpy_array_a, numpy_array_b)
print("Numpy Vector Product:", numpy_vector_product)

Numpy Sum: [ 7  9 11 13 15]
Numpy Vector Product: [ 6 14 24 36 50]

np.allclose(list_sum, numpy_sum), np.allclose(vector_product, numpy_vector_product)

(True, True)

Time comparison between list and numpy array

# Creating large arrays and lists for time comparison
numpy_array_a = np.random.randint(0, 100, size=10000)
numpy_array_b = np.random.randint(0, 100, size=10000)

list_a = list(numpy_array_a)
list_b = list(numpy_array_b)

# Time for list addition
start_time = time.time()
for _ in range(1000):
    list_sum = [a + b for a, b in zip(list_a, list_b)]
end_time = time.time()
print("Time taken for lists addition:", end_time - start_time)

# Time for numpy addition
start_time = time.time()
for _ in range(1000):
    numpy_sum = numpy_array_a + numpy_array_b
end_time = time.time()
print("Time taken for numpy addition:", end_time - start_time)

Time taken for lists addition: 0.5500102043151855
Time taken for numpy addition: 0.0038487911224365234

# Time for list vector product
start_time = time.time()
for _ in range(10000):
    list_product = [a * b for a, b in zip(list_a, list_b)]

end_time = time.time()
print("Time taken for list vector product:", end_time - start_time)

# Time for numpy vector product 
start_time = time.time()
for _ in range(10000):
    numpy_product = np.multiply(numpy_array_a, numpy_array_b)

end_time = time.time()
print("Time taken for numpy vector product:", end_time - start_time)

Time taken for list vector product: 5.371699571609497
Time taken for numpy vector product: 0.047417640686035156

np.allclose(list_sum, numpy_sum), np.allclose(vector_product, numpy_vector_product)

(True, True)

timeit_add_list = %timeit -o [a + b for a, b in zip(list_a, list_b)]

542 µs ± 593 ns per loop (mean ± std. dev. of 7 runs, 1,000 loops each)

timeit_add_numpy = %timeit -o numpy_array_a + numpy_array_b

3.5 µs ± 6.1 ns per loop (mean ± std. dev. of 7 runs, 100,000 loops each)

Code clarity

# Numpy code is often more concise and readable than list comprehensions
# Example: Calculate the element-wise product of two lists
list_product = [a * b for a, b in zip(list_a, list_b)]
numpy_product = np.multiply(numpy_array_a, numpy_array_b)

numpy_product

array([5950, 1995,  264, ..., 2436,  928,  665])

numpy_array_a@numpy_array_b

24470992

Reading CSV file using Numpy

!head ../datasets/tennis-discrete-output.csv

Day,Outlook,Temp,Humidity,Windy,Play
D1,Sunny,Hot,High,Weak,No
D2,Sunny,Hot,High,Strong,No
D3,Overcast,Hot,High,Weak,Yes
D4,Rain,Mild,High,Weak,Yes
D5,Rain,Cool,Normal,Weak,Yes
D6,Rain,Cool,Normal,Strong,No
D7,Overcast,Cool,Normal,Strong,Yes
D8,Sunny,Mild,High,Weak,No
D9,Sunny,Cool,Normal,Weak,Yes

np.genfromtxt?

Signature:

np.genfromtxt(

    fname,

    dtype=<class 'float'>,

    comments='#',

    delimiter=None,

    skip_header=0,

    skip_footer=0,

    converters=None,

    missing_values=None,

    filling_values=None,

    usecols=None,

    names=None,

    excludelist=None,

    deletechars=" !#$%&'()*+,-./:;<=>?@[\\]^{|}~",

    replace_space='_',

    autostrip=False,

    case_sensitive=True,

    defaultfmt='f%i',

    unpack=None,

    usemask=False,

    loose=True,

    invalid_raise=True,

    max_rows=None,

    encoding='bytes',

    *,

    ndmin=0,

    like=None,

)

Docstring:

Load data from a text file, with missing values handled as specified.



Each line past the first `skip_header` lines is split at the `delimiter`

character, and characters following the `comments` character are discarded.



Parameters

----------

fname : file, str, pathlib.Path, list of str, generator

    File, filename, list, or generator to read.  If the filename

    extension is ``.gz`` or ``.bz2``, the file is first decompressed. Note

    that generators must return bytes or strings. The strings

    in a list or produced by a generator are treated as lines.

dtype : dtype, optional

    Data type of the resulting array.

    If None, the dtypes will be determined by the contents of each

    column, individually.

comments : str, optional

    The character used to indicate the start of a comment.

    All the characters occurring on a line after a comment are discarded.

delimiter : str, int, or sequence, optional

    The string used to separate values.  By default, any consecutive

    whitespaces act as delimiter.  An integer or sequence of integers

    can also be provided as width(s) of each field.

skiprows : int, optional

    `skiprows` was removed in numpy 1.10. Please use `skip_header` instead.

skip_header : int, optional

    The number of lines to skip at the beginning of the file.

skip_footer : int, optional

    The number of lines to skip at the end of the file.

converters : variable, optional

    The set of functions that convert the data of a column to a value.

    The converters can also be used to provide a default value

    for missing data: ``converters = {3: lambda s: float(s or 0)}``.

missing : variable, optional

    `missing` was removed in numpy 1.10. Please use `missing_values`

    instead.

missing_values : variable, optional

    The set of strings corresponding to missing data.

filling_values : variable, optional

    The set of values to be used as default when the data are missing.

usecols : sequence, optional

    Which columns to read, with 0 being the first.  For example,

    ``usecols = (1, 4, 5)`` will extract the 2nd, 5th and 6th columns.

names : {None, True, str, sequence}, optional

    If `names` is True, the field names are read from the first line after

    the first `skip_header` lines. This line can optionally be preceded

    by a comment delimiter. If `names` is a sequence or a single-string of

    comma-separated names, the names will be used to define the field names

    in a structured dtype. If `names` is None, the names of the dtype

    fields will be used, if any.

excludelist : sequence, optional

    A list of names to exclude. This list is appended to the default list

    ['return','file','print']. Excluded names are appended with an

    underscore: for example, `file` would become `file_`.

deletechars : str, optional

    A string combining invalid characters that must be deleted from the

    names.

defaultfmt : str, optional

    A format used to define default field names, such as "f%i" or "f_%02i".

autostrip : bool, optional

    Whether to automatically strip white spaces from the variables.

replace_space : char, optional

    Character(s) used in replacement of white spaces in the variable

    names. By default, use a '_'.

case_sensitive : {True, False, 'upper', 'lower'}, optional

    If True, field names are case sensitive.

    If False or 'upper', field names are converted to upper case.

    If 'lower', field names are converted to lower case.

unpack : bool, optional

    If True, the returned array is transposed, so that arguments may be

    unpacked using ``x, y, z = genfromtxt(...)``.  When used with a

    structured data-type, arrays are returned for each field.

    Default is False.

usemask : bool, optional

    If True, return a masked array.

    If False, return a regular array.

loose : bool, optional

    If True, do not raise errors for invalid values.

invalid_raise : bool, optional

    If True, an exception is raised if an inconsistency is detected in the

    number of columns.

    If False, a warning is emitted and the offending lines are skipped.

max_rows : int,  optional

    The maximum number of rows to read. Must not be used with skip_footer

    at the same time.  If given, the value must be at least 1. Default is

    to read the entire file.



    .. versionadded:: 1.10.0

encoding : str, optional

    Encoding used to decode the inputfile. Does not apply when `fname` is

    a file object.  The special value 'bytes' enables backward compatibility

    workarounds that ensure that you receive byte arrays when possible

    and passes latin1 encoded strings to converters. Override this value to

    receive unicode arrays and pass strings as input to converters.  If set

    to None the system default is used. The default value is 'bytes'.



    .. versionadded:: 1.14.0

ndmin : int, optional

    Same parameter as `loadtxt`



    .. versionadded:: 1.23.0

like : array_like, optional

    Reference object to allow the creation of arrays which are not

    NumPy arrays. If an array-like passed in as ``like`` supports

    the ``__array_function__`` protocol, the result will be defined

    by it. In this case, it ensures the creation of an array object

    compatible with that passed in via this argument.



    .. versionadded:: 1.20.0



Returns

-------

out : ndarray

    Data read from the text file. If `usemask` is True, this is a

    masked array.



See Also

--------

numpy.loadtxt : equivalent function when no data is missing.



Notes

-----

* When spaces are used as delimiters, or when no delimiter has been given

  as input, there should not be any missing data between two fields.

* When the variables are named (either by a flexible dtype or with `names`),

  there must not be any header in the file (else a ValueError

  exception is raised).

* Individual values are not stripped of spaces by default.

  When using a custom converter, make sure the function does remove spaces.



References

----------

.. [1] NumPy User Guide, section `I/O with NumPy

       <https://docs.scipy.org/doc/numpy/user/basics.io.genfromtxt.html>`_.



Examples

--------

>>> from io import StringIO

>>> import numpy as np



Comma delimited file with mixed dtype



>>> s = StringIO(u"1,1.3,abcde")

>>> data = np.genfromtxt(s, dtype=[('myint','i8'),('myfloat','f8'),

... ('mystring','S5')], delimiter=",")

>>> data

array((1, 1.3, b'abcde'),

      dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', 'S5')])



Using dtype = None



>>> _ = s.seek(0) # needed for StringIO example only

>>> data = np.genfromtxt(s, dtype=None,

... names = ['myint','myfloat','mystring'], delimiter=",")

>>> data

array((1, 1.3, b'abcde'),

      dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', 'S5')])



Specifying dtype and names



>>> _ = s.seek(0)

>>> data = np.genfromtxt(s, dtype="i8,f8,S5",

... names=['myint','myfloat','mystring'], delimiter=",")

>>> data

array((1, 1.3, b'abcde'),

      dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', 'S5')])



An example with fixed-width columns



>>> s = StringIO(u"11.3abcde")

>>> data = np.genfromtxt(s, dtype=None, names=['intvar','fltvar','strvar'],

...     delimiter=[1,3,5])

>>> data

array((1, 1.3, b'abcde'),

      dtype=[('intvar', '<i8'), ('fltvar', '<f8'), ('strvar', 'S5')])



An example to show comments



>>> f = StringIO('''

... text,# of chars

... hello world,11

... numpy,5''')

>>> np.genfromtxt(f, dtype='S12,S12', delimiter=',')

array([(b'text', b''), (b'hello world', b'11'), (b'numpy', b'5')],

  dtype=[('f0', 'S12'), ('f1', 'S12')])

File:      ~/miniforge3/lib/python3.9/site-packages/numpy/lib/npyio.py

Type:      function

data = np.genfromtxt('../datasets/tennis-discrete-output.csv', delimiter=',')
data

array([[nan, nan, nan, nan, nan, nan],
       [nan, nan, nan, nan, nan, nan],
       [nan, nan, nan, nan, nan, nan],
       [nan, nan, nan, nan, nan, nan],
       [nan, nan, nan, nan, nan, nan],
       [nan, nan, nan, nan, nan, nan],
       [nan, nan, nan, nan, nan, nan],
       [nan, nan, nan, nan, nan, nan],
       [nan, nan, nan, nan, nan, nan],
       [nan, nan, nan, nan, nan, nan],
       [nan, nan, nan, nan, nan, nan],
       [nan, nan, nan, nan, nan, nan],
       [nan, nan, nan, nan, nan, nan],
       [nan, nan, nan, nan, nan, nan],
       [nan, nan, nan, nan, nan, nan]])

Wait! What happened?

data = np.genfromtxt('../datasets/tennis-discrete-output.csv', delimiter=',', dtype=str)
data

array([['Day', 'Outlook', 'Temp', 'Humidity', 'Windy', 'Play'],
       ['D1', 'Sunny', 'Hot', 'High', 'Weak', 'No'],
       ['D2', 'Sunny', 'Hot', 'High', 'Strong', 'No'],
       ['D3', 'Overcast', 'Hot', 'High', 'Weak', 'Yes'],
       ['D4', 'Rain', 'Mild', 'High', 'Weak', 'Yes'],
       ['D5', 'Rain', 'Cool', 'Normal', 'Weak', 'Yes'],
       ['D6', 'Rain', 'Cool', 'Normal', 'Strong', 'No'],
       ['D7', 'Overcast', 'Cool', 'Normal', 'Strong', 'Yes'],
       ['D8', 'Sunny', 'Mild', 'High', 'Weak', 'No'],
       ['D9', 'Sunny', 'Cool', 'Normal', 'Weak', 'Yes'],
       ['D10', 'Rain', 'Mild', 'Normal', 'Weak', 'Yes'],
       ['D11', 'Sunny', 'Mild', 'Normal', 'Strong', 'Yes'],
       ['D12', 'Overcast', 'Mild', 'High', 'Strong', 'Yes'],
       ['D13', 'Overcast', 'Hot', 'Normal', 'Weak', 'Yes'],
       ['D14', 'Rain', 'Mild', 'High', 'Strong', 'No']], dtype='<U8')

data.shape

(15, 6)

Question: Find the outlook on D11

idx = np.argwhere(data[:, 0] == 'D11')[0, 0]
idx

data[idx]

array(['D11', 'Sunny', 'Mild', 'Normal', 'Strong', 'Yes'], dtype='<U8')

data[idx][1]

'Sunny'

Reading CSV file using Pandas

df = pd.read_csv('../datasets/tennis-discrete-output.csv')

df

	Day	Outlook	Temp	Humidity	Windy	Play
0	D1	Sunny	Hot	High	Weak	No
1	D2	Sunny	Hot	High	Strong	No
2	D3	Overcast	Hot	High	Weak	Yes
3	D4	Rain	Mild	High	Weak	Yes
4	D5	Rain	Cool	Normal	Weak	Yes
5	D6	Rain	Cool	Normal	Strong	No
6	D7	Overcast	Cool	Normal	Strong	Yes
7	D8	Sunny	Mild	High	Weak	No
8	D9	Sunny	Cool	Normal	Weak	Yes
9	D10	Rain	Mild	Normal	Weak	Yes
10	D11	Sunny	Mild	Normal	Strong	Yes
11	D12	Overcast	Mild	High	Strong	Yes
12	D13	Overcast	Hot	Normal	Weak	Yes
13	D14	Rain	Mild	High	Strong	No

df['Day'] == 'D11'

0     False
1     False
2     False
3     False
4     False
5     False
6     False
7     False
8     False
9     False
10     True
11    False
12    False
13    False
Name: Day, dtype: bool

df[df['Day'] == 'D11']

	Day	Outlook	Temp	Humidity	Windy	Play
10	D11	Sunny	Mild	Normal	Strong	Yes

df[df['Day'] == 'D11']['Outlook']

10    Sunny
Name: Outlook, dtype: object

df.query('Day == "D11"')['Outlook']

10    Sunny
Name: Outlook, dtype: object

df.shape

(14, 6)

Question. How many times do we play v/s not play tennis

ser = df['Play']
ser

0      No
1      No
2     Yes
3     Yes
4     Yes
5      No
6     Yes
7      No
8     Yes
9     Yes
10    Yes
11    Yes
12    Yes
13     No
Name: Play, dtype: object

unique_play_options = df['Play'].unique()
unique_play_options

array(['No', 'Yes'], dtype=object)

for option in unique_play_options:
    print(option, (df['Play'] == option).sum())

No 5
Yes 9

df['Play'].value_counts()

Play
Yes    9
No     5
Name: count, dtype: int64

df.groupby('Play').size()

Play
No     5
Yes    9
dtype: int64

gby = df.groupby('Play')

{k: len(v) for k, v in gby.groups.items()}

{'No': 5, 'Yes': 9}

pd.crosstab(index=df['Play'], columns='count')

col_0	count
Play
No	5
Yes	9

What is the distribution of any given attribute?

def distribution(df, attribute):
    return df[attribute].value_counts()

ser = distribution(df, 'Outlook')

ser

Outlook
Sunny       5
Rain        5
Overcast    4
Name: count, dtype: int64

type(ser)

pandas.core.series.Series

ser.values

array([5, 5, 4])

ser.index

Index(['Sunny', 'Rain', 'Overcast'], dtype='object', name='Outlook')

distribution(df, 'Temp')

Temp
Mild    6
Hot     4
Cool    4
Name: count, dtype: int64

Finding entropy for target variable

target_attribute = 'Play'
dist_target = distribution(df, target_attribute)

dist_target

Play
Yes    9
No     5
Name: count, dtype: int64

Normalize distribution

dist_target/dist_target.sum()

Play
Yes    0.642857
No     0.357143
Name: count, dtype: float64

df['Play'].value_counts(normalize=True)

Play
Yes    0.642857
No     0.357143
Name: proportion, dtype: float64

normalized_dist_target = dist_target/dist_target.sum()

For loop way of calculating entropy

e = 0.0
for value, p in normalized_dist_target.items():
    e = e - p * np.log2(p + 1e-6) # 1e-6 is added to avoid log(0)
print(e)

0.9402830732836911

normalized_dist_target.apply(lambda x: -x * np.log2(x + 1e-6))

Play
Yes    0.409775
No     0.530508
Name: count, dtype: float64

normalized_dist_target.apply(lambda x: -x * np.log2(x + 1e-6)).sum()

0.9402830732836911

Outlook	Overcast	Rain	Sunny
Play
No	NaN	2.0	3.0
Yes	4.0	3.0	2.0

Outlook	Overcast	Rain	Sunny
Play
No	0.000000	0.528770	0.442178
Yes	-0.000001	0.442178	0.528770