ThirdEyeData/Customer-Conversion-Prediction

1#!/usr/local/bin/python32 3# avenir-python: Machine Learning4# Author: Pranab Ghosh5# 6# Licensed under the Apache License, Version 2.0 (the "License"); you7# may not use this file except in compliance with the License. You may8# obtain a copy of the License at9#10# http://www.apache.org/licenses/LICENSE-2.0 11#12# Unless required by applicable law or agreed to in writing, software13# distributed under the License is distributed on an "AS IS" BASIS,14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or15# implied. See the License for the specific language governing16# permissions and limitations under the License.17 18import sys19import random 20import time21import math22import numpy as np23import statistics 24from .util import *25 26"""27histogram class28"""29class Histogram:30	def __init__(self, min, binWidth):31		"""32    	initializer33    	34		Parameters35			min : min x36			binWidth : bin width37    	"""38		self.xmin = min39		self.binWidth = binWidth40		self.normalized = False41	42	@classmethod43	def createInitialized(cls, xmin, binWidth, values):44		"""45    	create histogram instance with min domain, bin width and values46    	47		Parameters48			min : min x49			binWidth : bin width50			values : y values51    	"""52		instance = cls(xmin, binWidth)53		instance.xmax = xmin + binWidth * (len(values) - 1)54		instance.ymin = 055		instance.bins = np.array(values)56		instance.fmax = 057		for v in values:58			if (v > instance.fmax):59				instance.fmax = v60		instance.ymin = 0.061		instance.ymax = instance.fmax62		return instance63 64	@classmethod65	def createWithNumBins(cls, values, numBins=20):66		"""67    	create histogram instance values and no of bins68    	69		Parameters70			values : y values71			numBins : no of bins72		"""73		xmin = min(values)74		xmax = max(values)75		binWidth = (xmax + .01 - (xmin - .01)) / numBins76		instance = cls(xmin, binWidth)77		instance.xmax = xmax78		instance.numBin = numBins79		instance.bins = np.zeros(instance.numBin)80		for v in values:81			instance.add(v)82		return instance83	84	@classmethod85	def createUninitialized(cls, xmin, xmax, binWidth):86		"""87    	create histogram instance with no y values using domain min , max and bin width88    	89		Parameters90			min : min x91			max : max x92			binWidth : bin width93    	"""94		instance = cls(xmin, binWidth)95		instance.xmax = xmax96		instance.numBin = (xmax - xmin) / binWidth + 197		instance.bins = np.zeros(instance.numBin)98		return instance99	100	def initialize(self):101		"""102    	set y values to 0103    	"""104		self.bins = np.zeros(self.numBin)105		106	def add(self, value):107		"""108    	adds a value to a bin109    	110		Parameters111			value : value112    	"""113		bin = int((value - self.xmin) / self.binWidth)114		if (bin < 0 or  bin > self.numBin - 1):115			print (bin)116			raise ValueError("outside histogram range")117		self.bins[bin] += 1.0118	119	def normalize(self):120		"""121    	normalize  bin counts122    	"""123		if not self.normalized:124			total = self.bins.sum()125			self.bins = np.divide(self.bins, total)126			self.normalized = True127	128	def cumDistr(self):129		"""130    	cumulative dists131    	"""132		self.normalize()133		self.cbins = np.cumsum(self.bins)134		return self.cbins135		136	def distr(self):137		"""138    	distr139    	"""140		self.normalize()141		return self.bins142 143		144	def percentile(self, percent):145		"""146    	return value corresponding to a percentile147    	148		Parameters149			percent : percentile value150    	"""151		if self.cbins is None:152			raise ValueError("cumulative distribution is not available")153			154		for i,cuml in enumerate(self.cbins):155			if percent > cuml:156				value = (i * self.binWidth) - (self.binWidth / 2) + \157				(percent - self.cbins[i-1]) * self.binWidth / (self.cbins[i] - self.cbins[i-1]) 158				break159		return value160		161	def max(self):162		"""163    	return max bin value 164    	"""165		return self.bins.max()166	167	def value(self, x):168		"""169    	return a bin value	170     	171		Parameters172			x : x value173   		"""174		bin = int((x - self.xmin) / self.binWidth)175		f = self.bins[bin]176		return f177 178	def bin(self, x):179		"""180    	return a bin index	181     	182		Parameters183			x : x value184   		"""185		return int((x - self.xmin) / self.binWidth)186	187	def cumValue(self, x):188		"""189    	return a cumulative bin value	190     	191		Parameters192			x : x value193   		"""194		bin = int((x - self.xmin) / self.binWidth)195		c = self.cbins[bin]196		return c197	198		199	def getMinMax(self):200		"""201    	returns x min and x max202    	"""203		return (self.xmin, self.xmax)204		205	def boundedValue(self, x):206		"""207    	return x bounde by min and max	208     	209		Parameters210			x : x value211   		"""212		if x < self.xmin:213			x = self.xmin214		elif x > self.xmax:215			x = self.xmax216		return x217 218"""219categorical histogram class220"""221class CatHistogram:222	def __init__(self):223		"""224    	initializer225    	"""226		self.binCounts = dict()227		self.counts = 0228		self.normalized = False229	230	def add(self, value):231		"""232		adds a value to a bin233		234		Parameters235			x : x value236		"""237		addToKeyedCounter(self.binCounts, value)238		self.counts += 1	239		240	def normalize(self):241		"""242		normalize243		"""244		if not self.normalized:245			self.binCounts = dict(map(lambda r : (r[0],r[1] / self.counts), self.binCounts.items()))246			self.normalized = True247	248	def getMode(self):249		"""250		get mode251		"""252		maxk = None253		maxv = 0254		#print(self.binCounts)255		for  k,v  in  self.binCounts.items():256			if v > maxv:257				maxk = k258				maxv = v259		return (maxk, maxv)	260	261	def getEntropy(self):262		"""263		get entropy264		"""265		self.normalize()266		entr = 0 267		#print(self.binCounts)268		for  k,v  in  self.binCounts.items():269			entr -= v * math.log(v)270		return entr271 272	def getUniqueValues(self):273		"""274		get unique values275		"""		276		return list(self.binCounts.keys())277 278	def getDistr(self):279		"""280		get distribution281		"""	282		self.normalize()	283		return self.binCounts.copy()284		285class RunningStat:286	"""287	running stat class288	"""289	def __init__(self):290   		"""291    	initializer	292   		"""293   		self.sum = 0.0294   		self.sumSq = 0.0295   		self.count = 0296	297	@staticmethod298	def create(count, sum, sumSq):299		"""300    	creates iinstance	301     	302		Parameters303			sum : sum of values304			sumSq : sum of valure squared305		"""306		rs = RunningStat()307		rs.sum = sum308		rs.sumSq = sumSq309		rs.count = count310		return rs311		312	def add(self, value):313		"""314		adds new value315 316		Parameters317			value : value to add318		"""319		self.sum += value320		self.sumSq += (value * value)321		self.count += 1322 323	def getStat(self):324		"""325		return mean and std deviation 326		"""327		mean = self.sum /self. count328		t = self.sumSq / (self.count - 1) - mean * mean * self.count / (self.count - 1)329		sd = math.sqrt(t)330		re = (mean, sd)331		return re332 333	def addGetStat(self,value):334		"""335		calculate mean and std deviation with new value added336 337		Parameters338			value : value to add339		"""340		self.add(value)341		re = self.getStat()342		return re343	344	def getCount(self):345		"""346		return count347		"""348		return self.count349	350	def getState(self):351		"""352		return state353		"""354		s = (self.count, self.sum, self.sumSq)355		return s356		357class SlidingWindowStat:358	"""359	sliding window stats360	"""361	def __init__(self):362		"""363		initializer364		"""365		self.sum = 0.0366		self.sumSq = 0.0367		self.count = 0368		self.values = None369	370	@staticmethod371	def create(values, sum, sumSq):372		"""373    	creates iinstance	374     	375		Parameters376			sum : sum of values377			sumSq : sum of valure squared378		"""379		sws = SlidingWindowStat()380		sws.sum = sum381		sws.sumSq = sumSq382		self.values = values.copy()383		sws.count = len(self.values)384		return sws385		386	@staticmethod387	def initialize(values):388		"""389    	creates iinstance	390     	391		Parameters392			values : list of values393		"""394		sws = SlidingWindowStat()395		sws.values = values.copy()396		for v in sws.values:397			sws.sum += v398			sws.sumSq += v * v		399		sws.count = len(sws.values)400		return sws401 402	@staticmethod403	def createEmpty(count):404		"""405    	creates iinstance	406     	407		Parameters408			count : count of values409		"""410		sws = SlidingWindowStat()411		sws.count = count412		sws.values = list()413		return sws414 415	def add(self, value):416		"""417		adds new value418		419		Parameters420			value : value to add421		"""422		self.values.append(value)		423		if len(self.values) > self.count:424			self.sum += value - self.values[0]425			self.sumSq += (value * value) - (self.values[0] * self.values[0])426			self.values.pop(0)427		else:428			self.sum += value429			self.sumSq += (value * value)430		431 432	def getStat(self):433		"""434		calculate mean and std deviation 435		"""436		mean = self.sum /self. count437		t = self.sumSq / (self.count - 1) - mean * mean * self.count / (self.count - 1)438		sd = math.sqrt(t)439		re = (mean, sd)440		return re441 442	def addGetStat(self,value):443		"""444		calculate mean and std deviation with new value added445		"""446		self.add(value)447		re = self.getStat()448		return re449	450	def getCount(self):451		"""452		return count453		"""454		return self.count455	456	def getCurSize(self):457		"""458		return count459		"""460		return len(self.values)461		462	def getState(self):463		"""464		return state465		"""466		s = (self.count, self.sum, self.sumSq)467		return s468		469 470def basicStat(ldata):471	"""472	mean and std dev473 474	Parameters475		ldata : list of values476	"""477	m = statistics.mean(ldata)478	s = statistics.stdev(ldata, xbar=m)479	r = (m, s)480	return r481 482def getFileColumnStat(filePath, col, delem=","):483	"""484	gets stats for a file column485	486	Parameters487		filePath : file path488		col : col index489		delem : field delemter490	"""491	rs = RunningStat()492	for rec in fileRecGen(filePath, delem):493		va = float(rec[col])494		rs.add(va)495		496	return rs.getStat()497