ThirdEyeData/Customer-Conversion-Prediction
1
1#!/usr/local/bin/python32 3# Author: Pranab Ghosh4# 5# Licensed under the Apache License, Version 2.0 (the "License"); you6# may not use this file except in compliance with the License. You may7# obtain a copy of the License at8#9# http://www.apache.org/licenses/LICENSE-2.0 10#11# Unless required by applicable law or agreed to in writing, software12# distributed under the License is distributed on an "AS IS" BASIS,13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or14# implied. See the License for the specific language governing15# permissions and limitations under the License.16 17# Package imports18import os19import sys20import numpy as np21import pandas as pd22import sklearn as sk23from sklearn import preprocessing24from sklearn import metrics25import random26from math import *27from decimal import Decimal28import pprint29from statsmodels.graphics import tsaplots30from statsmodels.tsa import stattools as stt31from statsmodels.stats import stattools as sstt32from sklearn.linear_model import LinearRegression33from matplotlib import pyplot as plt34from scipy import stats as sta35from statsmodels.tsa.seasonal import seasonal_decompose36import statsmodels.api as sm37from sklearn.ensemble import IsolationForest38from sklearn.neighbors import LocalOutlierFactor39from sklearn.svm import OneClassSVM40from sklearn.covariance import EllipticEnvelope41from sklearn.mixture import GaussianMixture42from sklearn.cluster import KMeans43from sklearn.decomposition import PCA44import hurst45from .util import *46from .mlutil import *47from .sampler import *48from .stats import *49 50"""51Load data from a CSV file, data frame, numpy array or list52Each data set (array like) is given a name while loading53Perform various data exploration operation refering to the data sets by name54Save and restore workspace if needed55"""56class DataSetMetaData:57 """58 data set meta data59 """60 dtypeNum = 161 dtypeCat = 262 dtypeBin = 363 def __init__(self, dtype):64 self.notes = list()65 self.dtype = dtype66 67 def addNote(self, note):68 """69 add note70 """71 self.notes.append(note)72 73 74class DataExplorer:75 """76 various data exploration functions77 """78 def __init__(self, verbose=True):79 """80 initialize81 82 Parameters83 verbose : True for verbosity84 """85 self.dataSets = dict()86 self.metaData = dict()87 self.pp = pprint.PrettyPrinter(indent=4)88 self.verbose = verbose89 90 def setVerbose(self, verbose):91 """92 sets verbose93 94 Parameters95 verbose : True for verbosity96 """97 self.verbose = verbose98 99 def save(self, filePath):100 """101 save checkpoint102 103 Parameters104 filePath : path of file where saved105 """106 self.__printBanner("saving workspace")107 ws = dict()108 ws["data"] = self.dataSets109 ws["metaData"] = self.metaData110 saveObject(ws, filePath)111 self.__printDone()112 113 def restore(self, filePath):114 """115 restore checkpoint116 117 Parameters118 filePath : path of file from where to store119 """120 self.__printBanner("restoring workspace")121 ws = restoreObject(filePath)122 self.dataSets = ws["data"]123 self.metaData = ws["metaData"]124 self.__printDone()125 126 127 def queryFileData(self, filePath, *columns):128 """129 query column data type from a data file130 131 Parameters132 filePath : path of file with data133 columns : indexes followed by column names or column names134 """135 self.__printBanner("querying column data type from a data frame")136 lcolumns = list(columns)137 noHeader = type(lcolumns[0]) == int138 if noHeader: 139 df = pd.read_csv(filePath, header=None) 140 else:141 df = pd.read_csv(filePath, header=0) 142 return self.queryDataFrameData(df, *columns)143 144 def queryDataFrameData(self, df, *columns):145 """146 query column data type from a data frame147 148 Parameters149 df : data frame with data150 columns : indexes followed by column name or column names151 """152 self.__printBanner("querying column data type from a data frame")153 columns = list(columns)154 noHeader = type(columns[0]) == int155 dtypes = list()156 if noHeader: 157 nCols = int(len(columns) / 2)158 colIndexes = columns[:nCols]159 cnames = columns[nCols:]160 nColsDf = len(df.columns)161 for i in range(nCols):162 ci = colIndexes[i]163 assert ci < nColsDf, "col index {} outside range".format(ci)164 col = df.loc[ : , ci]165 dtypes.append(self.getDataType(col))166 else:167 cnames = columns168 for c in columns:169 col = df[c]170 dtypes.append(self.getDataType(col))171 172 nt = list(zip(cnames, dtypes))173 result = self.__printResult("columns and data types", nt)174 return result175 176 def getDataType(self, col):177 """178 get data type 179 180 Parameters181 col : contains data array like182 """183 if isBinary(col):184 dtype = "binary"185 elif isInteger(col):186 dtype = "integer"187 elif isFloat(col):188 dtype = "float"189 elif isCategorical(col):190 dtype = "categorical"191 else:192 dtype = "mixed"193 return dtype194 195 196 def addFileNumericData(self,filePath, *columns):197 """198 add numeric columns from a file199 200 Parameters201 filePath : path of file with data202 columns : indexes followed by column names or column names203 """204 self.__printBanner("adding numeric columns from a file")205 self.addFileData(filePath, True, *columns)206 self.__printDone()207 208 209 def addFileBinaryData(self,filePath, *columns):210 """211 add binary columns from a file212 213 Parameters214 filePath : path of file with data215 columns : indexes followed by column names or column names216 """217 self.__printBanner("adding binary columns from a file")218 self.addFileData(filePath, False, *columns)219 self.__printDone()220 221 def addFileData(self, filePath, numeric, *columns):222 """223 add columns from a file224 225 Parameters226 filePath : path of file with data227 numeric : True if numeric False in binary228 columns : indexes followed by column names or column names229 """230 columns = list(columns)231 noHeader = type(columns[0]) == int232 if noHeader: 233 df = pd.read_csv(filePath, header=None) 234 else:235 df = pd.read_csv(filePath, header=0) 236 self.addDataFrameData(df, numeric, *columns)237 238 def addDataFrameNumericData(self,filePath, *columns):239 """240 add numeric columns from a data frame241 242 Parameters243 filePath : path of file with data244 columns : indexes followed by column names or column names245 """246 self.__printBanner("adding numeric columns from a data frame")247 self.addDataFrameData(filePath, True, *columns)248 249 250 def addDataFrameBinaryData(self,filePath, *columns):251 """252 add binary columns from a data frame253 254 Parameters255 filePath : path of file with data256 columns : indexes followed by column names or column names257 """258 self.__printBanner("adding binary columns from a data frame")259 self.addDataFrameData(filePath, False, *columns)260 261 262 def addDataFrameData(self, df, numeric, *columns):263 """264 add columns from a data frame265 266 Parameters267 df : data frame with data268 numeric : True if numeric False in binary269 columns : indexes followed by column names or column names270 """271 columns = list(columns)272 noHeader = type(columns[0]) == int273 if noHeader: 274 nCols = int(len(columns) / 2)275 colIndexes = columns[:nCols]276 nColsDf = len(df.columns)277 for i in range(nCols):278 ci = colIndexes[i]279 assert ci < nColsDf, "col index {} outside range".format(ci)280 col = df.loc[ : , ci]281 if numeric:282 assert isNumeric(col), "data is not numeric"283 else:284 assert isBinary(col), "data is not binary"285 col = col.to_numpy()286 cn = columns[i + nCols]287 dtype = DataSetMetaData.dtypeNum if numeric else DataSetMetaData.dtypeBin288 self.__addDataSet(cn, col, dtype)289 else:290 for c in columns:291 col = df[c]292 if numeric:293 assert isNumeric(col), "data is not numeric"294 else:295 assert isBinary(col), "data is not binary"296 col = col.to_numpy()297 dtype = DataSetMetaData.dtypeNum if numeric else DataSetMetaData.dtypeBin298 self.__addDataSet(c, col, dtype)299 300 def __addDataSet(self, dsn, data, dtype):301 """302 add dada set303 304 Parameters305 dsn: data set name306 data : numpy array data 307 """308 self.dataSets[dsn] = data309 self.metaData[dsn] = DataSetMetaData(dtype)310 311 312 def addListNumericData(self, ds, name):313 """314 add numeric data from a list315 316 Parameters317 ds : list with data318 name : name of data set319 """320 self.__printBanner("add numeric data from a list")321 self.addListData(ds, True, name)322 self.__printDone()323 324 325 def addListBinaryData(self, ds, name):326 """327 add binary data from a list328 329 Parameters330 ds : list with data331 name : name of data set332 """333 self.__printBanner("adding binary data from a list")334 self.addListData(ds, False, name)335 self.__printDone()336 337 def addListData(self, ds, numeric, name):338 """339 adds list data340 341 Parameters342 ds : list with data343 numeric : True if numeric False in binary344 name : name of data set345 """346 assert type(ds) == list, "data not a list"347 if numeric:348 assert isNumeric(ds), "data is not numeric"349 else:350 assert isBinary(ds), "data is not binary"351 dtype = DataSetMetaData.dtypeNum if numeric else DataSetMetaData.dtypeBin352 self.dataSets[name] = np.array(ds)353 self.metaData[name] = DataSetMetaData(dtype)354 355 356 def addFileCatData(self, filePath, *columns):357 """358 add categorical columns from a file359 360 Parameters361 filePath : path of file with data362 columns : indexes followed by column names or column names363 """364 self.__printBanner("adding categorical columns from a file")365 columns = list(columns)366 noHeader = type(columns[0]) == int367 if noHeader: 368 df = pd.read_csv(filePath, header=None) 369 else:370 df = pd.read_csv(filePath, header=0) 371 372 self.addDataFrameCatData(df, *columns)373 self.__printDone()374 375 def addDataFrameCatData(self, df, *columns):376 """377 add categorical columns from a data frame378 379 Parameters380 df : data frame with data381 columns : indexes followed by column names or column names382 """383 self.__printBanner("adding categorical columns from a data frame")384 columns = list(columns)385 noHeader = type(columns[0]) == int386 if noHeader: 387 nCols = int(len(columns) / 2)388 colIndexes = columns[:nCols]389 nColsDf = len(df.columns)390 for i in range(nCols):391 ci = colIndexes[i]392 assert ci < nColsDf, "col index {} outside range".format(ci)393 col = df.loc[ : , ci]394 assert isCategorical(col), "data is not categorical"395 col = col.tolist()396 cn = columns[i + nCols]397 self.__addDataSet(cn, col, DataSetMetaData.dtypeCat)398 else:399 for c in columns:400 col = df[c].tolist()401 self.__addDataSet(c, col, DataSetMetaData.dtypeCat)402 403 def addListCatData(self, ds, name):404 """405 add categorical list data406 407 Parameters408 ds : list with data409 name : name of data set410 """411 self.__printBanner("adding categorical list data")412 assert type(ds) == list, "data not a list"413 assert isCategorical(ds), "data is not categorical"414 self.__addDataSet(name, ds, DataSetMetaData.dtypeCat)415 self.__printDone()416 417 def remData(self, ds):418 """419 removes data set420 421 Parameters422 ds : data set name423 """424 self.__printBanner("removing data set", ds)425 assert ds in self.dataSets, "data set {} does not exist, please add it first".format(ds)426 self.dataSets.pop(ds)427 self.metaData.pop(ds)428 names = self.showNames()429 self.__printDone() 430 return names431 432 def addNote(self, ds, note):433 """434 get data435 436 Parameters437 ds : data set name or list or numpy array with data438 note: note text439 """440 self.__printBanner("adding note")441 assert ds in self.dataSets, "data set {} does not exist, please add it first".format(ds)442 mdata = self.metaData[ds]443 mdata.addNote(note)444 self.__printDone()445 446 def getNotes(self, ds):447 """448 get data449 450 Parameters451 ds : data set name or list or numpy array with data452 """453 self.__printBanner("getting notes")454 assert ds in self.dataSets, "data set {} does not exist, please add it first".format(ds) 455 mdata = self.metaData[ds]456 dnotes = mdata.notes457 if self.verbose:458 for dn in dnotes:459 print(dn)460 return dnotes461 462 def getNumericData(self, ds):463 """464 get numeric data465 466 Parameters467 ds : data set name or list or numpy array with data468 """469 if type(ds) == str:470 assert ds in self.dataSets, "data set {} does not exist, please add it first".format(ds)471 assert self.metaData[ds].dtype == DataSetMetaData.dtypeNum, "data set {} is expected to be numerical type for this operation".format(ds)472 data = self.dataSets[ds]473 elif type(ds) == list:474 assert isNumeric(ds), "data is not numeric"475 data = np.array(ds)476 elif type(ds) == np.ndarray:477 data = ds478 else:479 raise "invalid type, expecting data set name, list or ndarray" 480 return data481 482 483 def getCatData(self, ds):484 """485 get categorical data486 487 Parameters488 ds : data set name or list with data489 """490 if type(ds) == str:491 assert ds in self.dataSets, "data set {} does not exist, please add it first".format(ds)492 assert self.metaData[ds].dtype == DataSetMetaData.dtypeCat, "data set {} is expected to be categorical type for this operation".format(ds)493 data = self.dataSets[ds]494 elif type(ds) == list:495 assert isCategorical(ds), "data is not categorical"496 data = ds497 else:498 raise "invalid type, expecting data set name or list"499 return data500 501 def getAnyData(self, ds):502 """503 get any data504 505 Parameters506 ds : data set name or list with data507 """508 if type(ds) == str:509 assert ds in self.dataSets, "data set {} does not exist, please add it first".format(ds)510 data = self.dataSets[ds]511 elif type(ds) == list:512 data = ds513 else:514 raise "invalid type, expecting data set name or list"515 return data516 517 def loadCatFloatDataFrame(self, ds1, ds2):518 """519 loads float and cat data into data frame520 521 Parameters522 ds1: data set name or list523 ds2: data set name or list or numpy array524 """525 data1 = self.getCatData(ds1)526 data2 = self.getNumericData(ds2)527 self.ensureSameSize([data1, data2])528 df1 = pd.DataFrame(data=data1)529 df2 = pd.DataFrame(data=data2)530 df = pd.concat([df1,df2], axis=1)531 df.columns = range(df.shape[1])532 return df533 534 def showNames(self):535 """536 lists data set names537 """538 self.__printBanner("listing data set names")539 names = self.dataSets.keys()540 if self.verbose:541 print("data sets")542 for ds in names:543 print(ds)544 self.__printDone()545 return names546 547 def plot(self, ds, yscale=None):548 """549 plots data550 551 Parameters552 ds: data set name or list or numpy array553 yscale: y scale554 """555 self.__printBanner("plotting data", ds)556 data = self.getNumericData(ds)557 drawLine(data, yscale)558 559 def plotZoomed(self, ds, beg, end, yscale=None):560 """561 plots zoomed data562 563 Parameters564 ds: data set name or list or numpy array565 beg: begin offset566 end: end offset567 yscale: y scale568 """569 self.__printBanner("plotting data", ds)570 data = self.getNumericData(ds)571 drawLine(data[beg:end], yscale)572 573 def scatterPlot(self, ds1, ds2):574 """575 scatter plots data576 577 Parameters578 ds1: data set name or list or numpy array579 ds2: data set name or list or numpy array580 """581 self.__printBanner("scatter plotting data", ds1, ds2)582 data1 = self.getNumericData(ds1)583 data2 = self.getNumericData(ds2)584 self.ensureSameSize([data1, data2])585 x = np.arange(1, len(data1)+1, 1)586 plt.scatter(x, data1 ,color="red")587 plt.scatter(x, data2 ,color="blue")588 plt.show()589 590 def print(self, ds):591 """592 prunt data593 594 Parameters595 ds: data set name or list or numpy array596 """597 self.__printBanner("printing data", ds)598 assert ds in self.dataSets, "data set {} does not exist, please add it first".format(ds)599 data = self.dataSets[ds]600 if self.verbore:601 print(formatAny(len(data), "size"))602 print("showing first 50 elements" )603 print(data[:50])604 605 def plotHist(self, ds, cumulative, density, nbins=20):606 """607 plots histogram608 609 Parameters610 ds: data set name or list or numpy array611 cumulative : True if cumulative612 density : True to normalize for probability density613 nbins : no of bins614 """615 self.__printBanner("plotting histogram", ds)616 data = self.getNumericData(ds)617 plt.hist(data, bins=nbins, cumulative=cumulative, density=density)618 plt.show()619 620 def isMonotonicallyChanging(self, ds):621 """622 checks if monotonically increasing or decreasing623 624 Parameters625 ds: data set name or list or numpy array626 """627 self.__printBanner("checking monotonic change", ds)628 data = self.getNumericData(ds)629 monoIncreasing = all(list(map(lambda i : data[i] >= data[i-1], range(1, len(data), 1))))630 monoDecreasing = all(list(map(lambda i : data[i] <= data[i-1], range(1, len(data), 1))))631 result = self.__printResult("monoIncreasing", monoIncreasing, "monoDecreasing", monoDecreasing)632 return result633 634 def getFreqDistr(self, ds, nbins=20):635 """636 get histogram637 638 Parameters639 ds: data set name or list or numpy array640 nbins: num of bins641 """642 self.__printBanner("getting histogram", ds)643 data = self.getNumericData(ds)644 frequency, lowLimit, binsize, extraPoints = sta.relfreq(data, numbins=nbins)645 result = self.__printResult("frequency", frequency, "lowLimit", lowLimit, "binsize", binsize, "extraPoints", extraPoints)646 return result647 648 649 def getCumFreqDistr(self, ds, nbins=20):650 """651 get cumulative freq distribution652 653 Parameters654 ds: data set name or list or numpy array655 nbins: num of bins656 """657 self.__printBanner("getting cumulative freq distribution", ds)658 data = self.getNumericData(ds)659 cumFrequency, lowLimit, binsize, extraPoints = sta.cumfreq(data, numbins=nbins)660 result = self.__printResult("cumFrequency", cumFrequency, "lowLimit", lowLimit, "binsize", binsize, "extraPoints", extraPoints)661 return result662 663 def getExtremeValue(self, ds, ensamp, nsamp, polarity, doPlotDistr, nbins=20):664 """665 get extreme values666 667 Parameters668 ds: data set name or list or numpy array669 ensamp: num of samples for extreme values670 nsamp: num of samples671 polarity: max or min672 doPlotDistr: plot distr673 nbins: num of bins674 """675 self.__printBanner("getting extreme values", ds)676 data = self.getNumericData(ds)677 evalues = list()678 for _ in range(ensamp):679 values = selectRandomSubListFromListWithRepl(data, nsamp)680 if polarity == "max":681 evalues.append(max(values))682 else:683 evalues.append(min(values))684 if doPlotDistr:685 plt.hist(evalues, bins=nbins, cumulative=False, density=True)686 plt.show()687 result = self.__printResult("extremeValues", evalues)688 return result689 690 691 def getEntropy(self, ds, nbins=20):692 """693 get entropy694 695 Parameters696 ds: data set name or list or numpy array697 nbins: num of bins698 """699 self.__printBanner("getting entropy", ds)700 data = self.getNumericData(ds)701 result = self.getFreqDistr(data, nbins)702 entropy = sta.entropy(result["frequency"])703 result = self.__printResult("entropy", entropy)704 return result705 706 def getRelEntropy(self, ds1, ds2, nbins=20):707 """708 get relative entropy or KL divergence with both data sets numeric709 710 Parameters711 ds1: data set name or list or numpy array712 ds2: data set name or list or numpy array713 nbins: num of bins714 """715 self.__printBanner("getting relative entropy or KL divergence", ds1, ds2)716 data1 = self.getNumericData(ds1)717 data2 = self.getNumericData(ds2)718 result1 = self .getFeqDistr(data1, nbins)719 freq1 = result1["frequency"]720 result2 = self .getFeqDistr(data2, nbins)721 freq2 = result2["frequency"]722 entropy = sta.entropy(freq1, freq2)723 result = self.__printResult("relEntropy", entropy)724 return result725 726 def getAnyEntropy(self, ds, dt, nbins=20):727 """728 get entropy of any data typr numeric or categorical729 730 Parameters731 ds: data set name or list or numpy array732 dt : data type num or cat733 nbins: num of bins734 """735 entropy = self.getEntropy(ds, nbins)["entropy"] if dt == "num" else self.getStatsCat(ds)["entropy"]736 result = self.__printResult("entropy", entropy)737 return result738 739 def getJointEntropy(self, ds1, ds2, nbins=20):740 """741 get joint entropy with both data sets numeric742 743 Parameters744 ds1: data set name or list or numpy array745 ds2: data set name or list or numpy array746 nbins: num of bins747 """748 self.__printBanner("getting join entropy", ds1, ds2)749 data1 = self.getNumericData(ds1)750 data2 = self.getNumericData(ds2)751 self.ensureSameSize([data1, data2])752 hist, xedges, yedges = np.histogram2d(data1, data2, bins=nbins)753 hist = hist.flatten()754 ssize = len(data1)755 hist = hist / ssize756 entropy = sta.entropy(hist)757 result = self.__printResult("jointEntropy", entropy)758 return result759 760 761 def getAllNumMutualInfo(self, ds1, ds2, nbins=20):762 """763 get mutual information for both numeric data764 765 Parameters766 ds1: data set name or list or numpy array767 ds2: data set name or list or numpy array768 nbins: num of bins769 """770 self.__printBanner("getting mutual information", ds1, ds2)771 en1 = self.getEntropy(ds1,nbins)772 en2 = self.getEntropy(ds2,nbins)773 en = self.getJointEntropy(ds1, ds2, nbins)774 775 mutInfo = en1["entropy"] + en2["entropy"] - en["jointEntropy"]776 result = self.__printResult("mutInfo", mutInfo)777 return result778 779 780 def getNumCatMutualInfo(self, nds, cds ,nbins=20):781 """782 get mutiual information between numeric and categorical data783 784 Parameters785 nds: numeric data set name or list or numpy array786 cds: categoric data set name or list 787 nbins: num of bins788 """789 self.__printBanner("getting mutual information of numerical and categorical data", nds, cds)790 ndata = self.getNumericData(nds)791 cds = self.getCatData(cds)792 nentr = self.getEntropy(nds)["entropy"]793 794 #conditional entropy795 cdistr = self.getStatsCat(cds)["distr"]796 grdata = self.getGroupByData(nds, cds, True)["groupedData"]797 cnentr = 0798 for gr, data in grdata.items():799 self.addListNumericData(data, "grdata") 800 gnentr = self.getEntropy("grdata")["entropy"]801 cnentr += gnentr * cdistr[gr]802 803 mutInfo = nentr - cnentr804 result = self.__printResult("mutInfo", mutInfo, "entropy", nentr, "condEntropy", cnentr)805 return result806 807 def getTwoCatMutualInfo(self, cds1, cds2):808 """809 get mutiual information between 2 categorical data sets810 811 Parameters812 cds1 : categoric data set name or list 813 cds2 : categoric data set name or list 814 """815 self.__printBanner("getting mutual information of two categorical data sets", cds1, cds2)816 cdata1 = self.getCatData(cds1)817 cdata2 = self.getCatData(cds1)818 centr = self.getStatsCat(cds1)["entropy"]819 820 #conditional entropy821 cdistr = self.getStatsCat(cds2)["distr"]822 grdata = self.getGroupByData(cds1, cds2, True)["groupedData"]823 ccentr = 0824 for gr, data in grdata.items():825 self.addListCatData(data, "grdata") 826 gcentr = self.getStatsCat("grdata")["entropy"]827 ccentr += gcentr * cdistr[gr]828 829 mutInfo = centr - ccentr830 result = self.__printResult("mutInfo", mutInfo, "entropy", centr, "condEntropy", ccentr)831 return result832 833 def getMutualInfo(self, dst, nbins=20):834 """835 get mutiual information between 2 data sets,any combination numerical and categorical836 837 Parameters838 dst : data source , data type, data source , data type839 nbins : num of bins840 """841 assertEqual(len(dst), 4, "invalid data source and data type list size")842 dtypes = ["num", "cat"]843 assertInList(dst[1], dtypes, "invalid data type")844 assertInList(dst[3], dtypes, "invalid data type")845 self.__printBanner("getting mutual information of any mix numerical and categorical data", dst[0], dst[2])846 847 if dst[1] == "num":848 mutInfo = self.getAllNumMutualInfo(dst[0], dst[2], nbins)["mutInfo"] if dst[3] == "num" \849 else self.getNumCatMutualInfo(dst[0], dst[2], nbins)["mutInfo"]850 else:851 mutInfo = self.getNumCatMutualInfo(dst[2], dst[0], nbins)["mutInfo"] if dst[3] == "num" \852 else self.getTwoCatMutualInfo(dst[2], dst[0])["mutInfo"]853 854 result = self.__printResult("mutInfo", mutInfo)855 return result856 857 858 def getCondMutualInfo(self, dst, nbins=20):859 """860 get conditional mutiual information between 2 data sets,any combination numerical and categorical861 862 Parameters863 dst : data source , data type, data source , data type, data source , data type864 nbins : num of bins865 """866 assertEqual(len(dst), 6, "invalid data source and data type list size")867 dtypes = ["num", "cat"]868 assertInList(dst[1], dtypes, "invalid data type")869 assertInList(dst[3], dtypes, "invalid data type")870 assertInList(dst[5], dtypes, "invalid data type")871 self.__printBanner("getting conditional mutual information of any mix numerical and categorical data", dst[0], dst[2])872 873 if dst[5] == "cat":874 cdistr = self.getStatsCat(dst[4])["distr"]875 grdata1 = self.getGroupByData(dst[0], dst[4], True)["groupedData"]876 grdata2 = self.getGroupByData(dst[2], dst[4], True)["groupedData"]877 878 else:879 gdata = self.getNumericData(dst[4])880 hist = Histogram.createWithNumBins(gdata, nbins)881 cdistr = hist.distr()882 grdata1 = self.getGroupByData(dst[0], dst[4], False)["groupedData"]883 grdata2 = self.getGroupByData(dst[2], dst[4], False)["groupedData"]884 885 886 cminfo = 0887 for gr in grdata1.keys():888 data1 = grdata1[gr]889 data2 = grdata2[gr]890 if dst[1] == "num":891 self.addListNumericData(data1, "grdata1")892 else:893 self.addListCatData(data1, "grdata1")894 895 if dst[3] == "num":896 self.addListNumericData(data2, "grdata2")897 else:898 self.addListCatData(data2, "grdata2")899 gdst = ["grdata1", dst[1], "grdata2", dst[3]]900 minfo = self.getMutualInfo(gdst, nbins)["mutInfo"] 901 cminfo += minfo * cdistr[gr]902 903 result = self.__printResult("condMutInfo", cminfo)904 return result905 906 def getPercentile(self, ds, value):907 """908 gets percentile909 910 Parameters911 ds: data set name or list or numpy array912 value: the value913 """914 self.__printBanner("getting percentile", ds)915 data = self.getNumericData(ds)916 percent = sta.percentileofscore(data, value)917 result = self.__printResult("value", value, "percentile", percent)918 return result919 920 def getValueRangePercentile(self, ds, value1, value2):921 """922 gets percentile923 924 Parameters925 ds: data set name or list or numpy array926 value1: first value927 value2: second value928 """929 self.__printBanner("getting percentile difference for value range", ds)930 if value1 < value2:931 v1 = value1932 v2 = value2933 else:934 v1 = value2935 v2 = value1936 data = self.getNumericData(ds)937 per1 = sta.percentileofscore(data, v1)938 per2 = sta.percentileofscore(data, v2)939 result = self.__printResult("valueFirst", value1, "valueSecond", value2, "percentileDiff", per2 - per1)940 return result941 942 def getValueAtPercentile(self, ds, percent):943 """944 gets value at percentile945 946 Parameters947 ds: data set name or list or numpy array948 percent: percentile949 """950 self.__printBanner("getting value at percentile", ds)951 data = self.getNumericData(ds)952 assert isInRange(percent, 0, 100), "percent should be between 0 and 100"953 value = sta.scoreatpercentile(data, percent)954 result = self.__printResult("value", value, "percentile", percent)955 return result956 957 def getLessThanValues(self, ds, cvalue):958 """959 gets values less than given value960 961 Parameters962 ds: data set name or list or numpy array963 cvalue: condition value964 """965 self.__printBanner("getting values less than", ds)966 fdata = self.__getCondValues(ds, cvalue, "lt")967 result = self.__printResult("count", len(fdata), "lessThanvalues", fdata )968 return result969 970 971 def getGreaterThanValues(self, ds, cvalue):972 """973 gets values greater than given value974 975 Parameters976 ds: data set name or list or numpy array977 cvalue: condition value978 """979 self.__printBanner("getting values greater than", ds)980 fdata = self.__getCondValues(ds, cvalue, "gt")981 result = self.__printResult("count", len(fdata), "greaterThanvalues", fdata )982 return result983 984 def __getCondValues(self, ds, cvalue, cond):985 """986 gets cinditional values987 988 Parameters989 ds: data set name or list or numpy array990 cvalue: condition value991 cond: condition992 """993 data = self.getNumericData(ds)994 if cond == "lt":995 ind = np.where(data < cvalue)996 else:997 ind = np.where(data > cvalue)998 fdata = data[ind]999 return fdata1000 1001 def getUniqueValueCounts(self, ds, maxCnt=10):1002 """1003 gets unique values and counts1004 1005 Parameters1006 ds: data set name or list or numpy array1007 maxCnt; max value count pairs to return1008 """1009 self.__printBanner("getting unique values and counts", ds)1010 data = self.getNumericData(ds)1011 values, counts = sta.find_repeats(data)1012 cardinality = len(values)1013 vc = list(zip(values, counts))1014 vc.sort(key = lambda v : v[1], reverse = True)1015 result = self.__printResult("cardinality", cardinality, "vunique alues and repeat counts", vc[:maxCnt])1016 return result1017 1018 def getCatUniqueValueCounts(self, ds, maxCnt=10):1019 """1020 gets unique categorical values and counts1021 1022 Parameters1023 ds: data set name or list or numpy array1024 maxCnt: max value count pairs to return1025 """1026 self.__printBanner("getting unique categorical values and counts", ds)1027 data = self.getCatData(ds)1028 series = pd.Series(data)1029 uvalues = series.value_counts()1030 values = uvalues.index.tolist()1031 counts = uvalues.tolist()1032 vc = list(zip(values, counts))1033 vc.sort(key = lambda v : v[1], reverse = True)1034 result = self.__printResult("cardinality", len(values), "unique values and repeat counts", vc[:maxCnt])1035 return result1036 1037 def getCatAlphaValueCounts(self, ds):1038 """1039 gets alphabetic value count1040 1041 Parameters1042 ds: data set name or list or numpy array1043 """1044 self.__printBanner("getting alphabetic value counts", ds)1045 data = self.getCatData(ds)1046 series = pd.Series(data)1047 flags = series.str.isalpha().tolist()1048 count = sum(flags)1049 result = self.__printResult("alphabeticValueCount", count)1050 return result1051 1052 1053 def getCatNumValueCounts(self, ds):1054 """1055 gets numeric value count1056 1057 Parameters1058 ds: data set name or list or numpy array1059 """1060 self.__printBanner("getting numeric value counts", ds)1061 data = self.getCatData(ds)1062 series = pd.Series(data)1063 flags = series.str.isnumeric().tolist()1064 count = sum(flags)1065 result = self.__printResult("numericValueCount", count)1066 return result1067 1068 1069 def getCatAlphaNumValueCounts(self, ds):1070 """1071 gets alpha numeric value count1072 1073 Parameters1074 ds: data set name or list or numpy array1075 """1076 self.__printBanner("getting alpha numeric value counts", ds)1077 data = self.getCatData(ds)1078 series = pd.Series(data)1079 flags = series.str.isalnum().tolist()1080 count = sum(flags)1081 result = self.__printResult("alphaNumericValueCount", count)1082 return result1083 1084 def getCatAllCharCounts(self, ds):1085 """1086 gets alphabetic, numeric and special char count list1087 1088 Parameters1089 ds: data set name or list or numpy array1090 """1091 self.__printBanner("getting alphabetic, numeric and special char counts", ds)1092 data = self.getCatData(ds)1093 counts = list()1094 for d in data:1095 r = getAlphaNumCharCount(d)1096 counts.append(r)1097 result = self.__printResult("allTypeCharCounts", counts)1098 return result1099 1100 def getCatAlphaCharCounts(self, ds):1101 """1102 gets alphabetic char count list1103 1104 Parameters1105 ds: data set name or list or numpy array1106 """1107 self.__printBanner("getting alphabetic char counts", ds)1108 data = self.getCatData(ds)1109 counts = self.getCatAllCharCounts(ds)["allTypeCharCounts"]1110 counts = list(map(lambda r : r[0], counts))1111 result = self.__printResult("alphaCharCounts", counts)1112 return result1113 1114 def getCatNumCharCounts(self, ds):1115 """1116 gets numeric char count list1117 1118 Parameters1119 ds: data set name or list or numpy array1120 """1121 self.__printBanner("getting numeric char counts", ds)1122 data = self.getCatData(ds)1123 counts = self.getCatAllCharCounts(ds)["allTypeCharCounts"]1124 counts = list(map(lambda r : r[1], counts))1125 result = self.__printResult("numCharCounts", counts)1126 return result1127 1128 def getCatSpecialCharCounts(self, ds):1129 """1130 gets special char count list1131 1132 Parameters1133 ds: data set name or list or numpy array1134 """1135 self.__printBanner("getting special char counts", ds)1136 counts = self.getCatAllCharCounts(ds)["allTypeCharCounts"]1137 counts = list(map(lambda r : r[2], counts))1138 result = self.__printResult("specialCharCounts", counts)1139 return result1140 1141 def getCatAlphaCharCountStats(self, ds):1142 """1143 gets alphabetic char count stats1144 1145 Parameters1146 ds: data set name or list or numpy array1147 """1148 self.__printBanner("getting alphabetic char count stats", ds)1149 counts = self.getCatAlphaCharCounts(ds)["alphaCharCounts"]1150 nz = counts.count(0)1151 st = self.__getBasicStats(np.array(counts))1152 result = self.__printResult("mean", st[0], "std dev", st[1], "max", st[2], "min", st[3], "zeroCount", nz)1153 return result1154 1155 def getCatNumCharCountStats(self, ds):1156 """1157 gets numeric char count stats1158 1159 Parameters1160 ds: data set name or list or numpy array1161 """1162 self.__printBanner("getting numeric char count stats", ds)1163 counts = self.getCatNumCharCounts(ds)["numCharCounts"]1164 nz = counts.count(0)1165 st = self.__getBasicStats(np.array(counts))1166 result = self.__printResult("mean", st[0], "std dev", st[1], "max", st[2], "min", st[3], "zeroCount", nz)1167 return result1168 1169 def getCatSpecialCharCountStats(self, ds):1170 """1171 gets special char count stats1172 1173 Parameters1174 ds: data set name or list or numpy array1175 """1176 self.__printBanner("getting special char count stats", ds)1177 counts = self.getCatSpecialCharCounts(ds)["specialCharCounts"]1178 nz = counts.count(0)1179 st = self.__getBasicStats(np.array(counts))1180 result = self.__printResult("mean", st[0], "std dev", st[1], "max", st[2], "min", st[3], "zeroCount", nz)1181 return result1182 1183 def getCatFldLenStats(self, ds):1184 """1185 gets field length stats1186 1187 Parameters1188 ds: data set name or list or numpy array1189 """1190 self.__printBanner("getting field length stats", ds)1191 data = self.getCatData(ds)1192 le = list(map(lambda d: len(d), data))1193 st = self.__getBasicStats(np.array(le))1194 result = self.__printResult("mean", st[0], "std dev", st[1], "max", st[2], "min", st[3])1195 return result1196 1197 def getCatCharCountStats(self, ds, ch):1198 """1199 gets specified char ocuurence count stats1200