ThirdEyeData/Customer-Conversion-Prediction

1#!/usr/local/bin/python32 3# avenir-python: Machine Learning4# Author: Pranab Ghosh5# 6# Licensed under the Apache License, Version 2.0 (the "License"); you7# may not use this file except in compliance with the License. You may8# obtain a copy of the License at9#10# http://www.apache.org/licenses/LICENSE-2.0 11#12# Unless required by applicable law or agreed to in writing, software13# distributed under the License is distributed on an "AS IS" BASIS,14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or15# implied. See the License for the specific language governing16# permissions and limitations under the License.17 18# Package imports19import os20import sys21import matplotlib.pyplot as plt22import numpy as np23import matplotlib24import random25import jprops26import statistics 27from matplotlib import pyplot28from .util import *29from .mlutil import *30from .sampler import *31 32class MonteCarloSimulator(object):33	"""34	monte carlo simulator for intergation, various statistic for complex fumctions35	"""36	def __init__(self, numIter, callback, logFilePath, logLevName):37		"""38		constructor39		40		Parameters41			numIter :num of iterations42			callback : call back method43			logFilePath : log file path44			logLevName : log level45		"""46		self.samplers = list()47		self.numIter = numIter;48		self.callback = callback49		self.extraArgs = None50		self.output = list()51		self.sum = None52		self.mean = None53		self.sd = None54		self.replSamplers = dict()55		self.prSamples = None56		57		self.logger = None58		if logFilePath is not None: 		59			self.logger = createLogger(__name__, logFilePath, logLevName)60			self.logger.info("******** stating new  session of MonteCarloSimulator")61 62 63	def registerBernoulliTrialSampler(self, pr):64		"""65		bernoulli trial sampler66		67		Parameters68			pr : probability69		"""70		self.samplers.append(BernoulliTrialSampler(pr))71		72	def registerPoissonSampler(self, rateOccur, maxSamp):73		"""74		poisson sampler75		76		Parameters77			rateOccur : rate of occurence78			maxSamp : max limit on no of samples79		"""80		self.samplers.append(PoissonSampler(rateOccur, maxSamp))81 82	def registerUniformSampler(self, minv, maxv):83		"""84		uniform sampler85		86		Parameters87			minv : min value88			maxv : max value89		"""90		self.samplers.append(UniformNumericSampler(minv, maxv))91 92	def registerTriangularSampler(self, min, max, vertexValue, vertexPos=None):93		"""94		triangular sampler95		96		Parameters97			xmin : min  value98			xmax : max  value99			vertexValue : distr value at vertex100			vertexPos : vertex pposition101		"""102		self.samplers.append(TriangularRejectSampler(min, max, vertexValue, vertexPos))103 104	def registerGaussianSampler(self, mean, sd):105		"""106		gaussian sampler107 108		Parameters109			mean : mean110			sd : std deviation111		"""112		self.samplers.append(GaussianRejectSampler(mean, sd))113		114	def registerNormalSampler(self, mean, sd):115		"""116		gaussian sampler using numpy117 118		Parameters119			mean : mean120			sd : std deviation121		"""122		self.samplers.append(NormalSampler(mean, sd))123 124	def registerLogNormalSampler(self, mean, sd):125		"""126		log normal sampler using numpy127 128		Parameters129			mean : mean130			sd : std deviation131		"""132		self.samplers.append(LogNormalSampler(mean, sd))133 134	def registerParetoSampler(self, mode, shape):135		"""136		pareto sampler using numpy137 138		Parameters139			mode : mode140			shape : shape141		"""142		self.samplers.append(ParetoSampler(mode, shape))143 144	def registerGammaSampler(self, shape, scale):145		"""146		gamma sampler using numpy147 148		Parameters149			shape : shape150			scale : scale151		"""152		self.samplers.append(GammaSampler(shape, scale))153 154	def registerDiscreteRejectSampler(self, xmin, xmax, step, *values):155		"""156		disccrete int sampler157 158		Parameters159			xmin : min  value160			xmax : max  value161			step : discrete step162			values : distr values163		"""164		self.samplers.append(DiscreteRejectSampler(xmin, xmax, step, *values))165 166	def registerNonParametricSampler(self, minv, binWidth, *values):167		"""168		nonparametric sampler169 170		Parameters171			xmin : min  value172			binWidth : bin width173			values : distr values174		"""175		sampler = NonParamRejectSampler(minv, binWidth, *values)176		sampler.sampleAsFloat()177		self.samplers.append(sampler)178 179	def registerMultiVarNormalSampler(self,  numVar, *values):180		"""181		multi var gaussian sampler using numpy182 183		Parameters184			numVar : no of variables185			values : numVar mean values followed by numVar x numVar values for covar matrix186		"""187		self.samplers.append(MultiVarNormalSampler(numVar, *values))188		189	def registerJointNonParamRejectSampler(self, xmin, xbinWidth, xnbin, ymin, ybinWidth, ynbin, *values):190		"""191		joint nonparametric sampler192 193		Parameters194			xmin : min  value for x195			xbinWidth : bin width for x196			xnbin : no of bins for x197			ymin : min  value for y198			ybinWidth : bin width for y199			ynbin : no of bins for y200			values : distr values201		"""202		self.samplers.append(JointNonParamRejectSampler(xmin, xbinWidth, xnbin, ymin, ybinWidth, ynbin, *values))203 204	def registerRangePermutationSampler(self, minv, maxv, *numShuffles):205		"""206		permutation sampler with range207 208		Parameters209			minv : min of range210			maxv : max of range211			numShuffles : no of shuffles or range of no of shuffles212		"""213		self.samplers.append(PermutationSampler.createSamplerWithRange(minv, maxv, *numShuffles))214	215	def registerValuesPermutationSampler(self, values, *numShuffles):216		"""217		permutation sampler with values218 219		Parameters220			values : list data221			numShuffles : no of shuffles or range of no of shuffles222		"""223		self.samplers.append(PermutationSampler.createSamplerWithValues(values, *numShuffles))224	225	def registerNormalSamplerWithTrendCycle(self, mean, stdDev, trend, cycle,  step=1):226		"""227		normal sampler with trend and cycle228 229		Parameters230			mean : mean231			stdDev : std deviation232			dmean : trend delta233			cycle : cycle values wrt base mean234			step : adjustment step for cycle and trend235		"""236		self.samplers.append(NormalSamplerWithTrendCycle(mean, stdDev, trend, cycle,  step))237 238	def registerCustomSampler(self, sampler):239		"""240		eventsampler241		242		Parameters243			sampler : sampler with sample() method244		"""245		self.samplers.append(sampler)246	247	def registerEventSampler(self, intvSampler, valSampler=None):248		"""249		event sampler250		251		Parameters252			intvSampler : interval sampler253			valSampler : value sampler254		"""255		self.samplers.append(EventSampler(intvSampler, valSampler))256 257	def registerMetropolitanSampler(self, propStdDev, minv, binWidth, values):258		"""259		metropolitan sampler260		261		Parameters262			propStdDev : proposal distr std dev263			minv : min domain value for target distr264			binWidth : bin width265			values : target distr values266		"""267		self.samplers.append(MetropolitanSampler(propStdDev, minv, binWidth, values))268 269	def setSampler(self, var, iter, sampler):270		"""271		set sampler for some variable when iteration reaches certain point272 273		Parameters274			var : sampler index275			iter : iteration count276			sampler : new sampler277		"""278		key = (var, iter)279		self.replSamplers[key] = sampler280 281	def registerExtraArgs(self, *args):282		"""283		extra args284 285		Parameters286			args : extra argument list287		"""288		self.extraArgs = args289 290	def replSampler(self, iter):291		"""292		replace samper for this iteration293 294		Parameters295			iter : iteration number296		"""297		if len(self.replSamplers) > 0:298			for v in range(self.numVars):299				key = (v, iter)300				if key in self.replSamplers:301					sampler = self.replSamplers[key]302					self.samplers[v] = sampler303 304	def run(self):305		"""306		run simulator307		"""308		self.sum = None309		self.mean = None310		self.sd = None311		self.numVars = len(self.samplers)312		vOut = 0313 314		#print(formatAny(self.numIter, "num iterations"))315		for i in range(self.numIter):316			self.replSampler(i)317			args = list()318			for s in self.samplers:319				arg = s.sample()320				if type(arg) is list:321					args.extend(arg)322				else:323					args.append(arg)324					325			slen = len(args)326			if self.extraArgs:327				args.extend(self.extraArgs)328			args.append(self)329			args.append(i)330			vOut = self.callback(args)	331			self.output.append(vOut)332			self.prSamples = args[:slen]333	334	def getOutput(self):335		"""336		get raw output337		"""338		return self.output339	340	def setOutput(self, values):341		"""342		set raw output343 344		Parameters345			values : output values346		"""347		self.output = values348		self.numIter = len(values)349 350	def drawHist(self, myTitle, myXlabel, myYlabel):351		"""352		draw histogram353 354		Parameters355			myTitle : title356			myXlabel : label for x357			myYlabel : label for y358		"""359		pyplot.hist(self.output, density=True)360		pyplot.title(myTitle)361		pyplot.xlabel(myXlabel)362		pyplot.ylabel(myYlabel)363		pyplot.show()	364		365	def getSum(self):366		"""367		get sum368		"""369		if not self.sum:370			self.sum = sum(self.output)371		return self.sum372		373	def getMean(self):374		"""375		get average376		"""377		if self.mean is None:378			self.mean = statistics.mean(self.output)379		return self.mean 380		381	def getStdDev(self):382		"""383		get std dev384		"""385		if self.sd is None:386			self.sd = statistics.stdev(self.output, xbar=self.mean) if self.mean else statistics.stdev(self.output)387		return self.sd 388		389 390	def getMedian(self):391		"""392		get average393		"""394		med = statistics.median(self.output)395		return med396 397	def getMax(self):398		"""399		get max400		"""401		return max(self.output)402		403	def getMin(self):404		"""405		get min406		"""407		return min(self.output)408		409	def getIntegral(self, bounds):410		"""411		integral412 413		Parameters414			bounds :  bound on sum415		"""416		if not self.sum:417			self.sum = sum(self.output)418		return self.sum * bounds / self.numIter419	420	def getLowerTailStat(self, zvalue, numIntPoints=50):421		"""422		get lower tail stat423 424		Parameters425			zvalue : zscore upper bound 426			numIntPoints : no of interpolation point for cum distribution427		"""428		mean = self.getMean()429		sd = self.getStdDev()430		tailStart = self.getMin()431		tailEnd = mean - zvalue * sd432		cvaCounts = self.cumDistr(tailStart, tailEnd, numIntPoints)433		434		reqConf = floatRange(0.0, 0.150, .01)	435		msg = "p value outside interpolation range, reduce zvalue and try again {:.5f}  {:.5f}".format(reqConf[-1], cvaCounts[-1][1])436		assert reqConf[-1] < cvaCounts[-1][1], msg437		critValues = self.interpolateCritValues(reqConf, cvaCounts, True, tailStart, tailEnd)438		return critValues439		440	def getPercentile(self, cvalue):441		"""442		percentile443 444		Parameters445			cvalue : value for percentile 446		"""447		count = 0448		for v in self.output:449			if v < cvalue:450				count += 1 451		percent =  int(count * 100.0 / self.numIter)452		return percent453 454 455	def getCritValue(self, pvalue):	456		"""457		critical value for probabaility threshold458 459		Parameters460			pvalue : pvalue 461		"""462		assertWithinRange(pvalue, 0.0, 1.0, "invalid probabaility value")463		svalues = self.output.sorted()464		ppval = None465		cpval = None466		intv = 1.0 / (self.numIter - 1)467		for i in range(self.numIter - 1):468			cpval = (i + 1) / self.numIter469			if cpval > pvalue:470				sl = svalues[i] - svalues[i-1]471				cval = svalues[i-1] + sl * (pvalue - ppval)472				break473			ppval = cpval474		return cval475		476		477	def getUpperTailStat(self, zvalue, numIntPoints=50):478		"""479		upper tail stat480 481		Parameters482			zvalue : zscore upper bound 483			numIntPoints : no of interpolation point for cum distribution484		"""485		mean = self.getMean()486		sd = self.getStdDev()487		tailStart = mean + zvalue * sd488		tailEnd = self.getMax()489		cvaCounts = self.cumDistr(tailStart, tailEnd, numIntPoints)		490		491		reqConf = floatRange(0.85, 1.0, .01)	492		msg = "p value outside interpolation range, reduce zvalue and try again {:.5f}  {:.5f}".format(reqConf[0], cvaCounts[0][1])493		assert reqConf[0] > cvaCounts[0][1],  msg494		critValues = self.interpolateCritValues(reqConf, cvaCounts, False, tailStart, tailEnd)495		return critValues		496 497	def cumDistr(self, tailStart, tailEnd, numIntPoints):498		"""499		cumulative distribution at tail500		501		Parameters502			tailStart : tail start503			tailEnd : tail end504			numIntPoints : no of interpolation points505		"""506		delta = (tailEnd - tailStart) / numIntPoints507		cvalues = floatRange(tailStart, tailEnd, delta)508		cvaCounts = list()509		for cv in cvalues:510			count = 0511			for v in self.output:512				if v < cv:513					count += 1514			p = (cv, count/self.numIter)515			if self.logger is not None:516				self.logger.info("{:.3f}  {:.3f}".format(p[0], p[1]))517			cvaCounts.append(p)518		return cvaCounts519			520	def interpolateCritValues(self, reqConf, cvaCounts, lowertTail, tailStart, tailEnd):	521		"""522		interpolate for spefici confidence limits523		524		Parameters525			reqConf : confidence level values526			cvaCounts : cum values527			lowertTail : True if lower tail528			tailStart ; tail start529			tailEnd : tail end530		"""531		critValues = list()532		if self.logger is not None:533			self.logger.info("target conf limit " + str(reqConf))534		reqConfSub = reqConf[1:] if lowertTail else reqConf[:-1]535		for rc in reqConfSub:536			for i in range(len(cvaCounts) -1):537				if rc >= cvaCounts[i][1] and rc < cvaCounts[i+1][1]:538					#print("interpoltate between " + str(cvaCounts[i])  +  " and " + str(cvaCounts[i+1]))539					slope = (cvaCounts[i+1][0] - cvaCounts[i][0]) / (cvaCounts[i+1][1] - cvaCounts[i][1])540					cval = cvaCounts[i][0] + slope * (rc - cvaCounts[i][1]) 541					p = (rc, cval)542					if self.logger is not None:543						self.logger.debug("interpolated crit values {:.3f} {:.3f}".format(p[0], p[1]))544					critValues.append(p)545					break546		if lowertTail:547			p = (0.0, tailStart)548			critValues.insert(0, p)549		else:550			p = (1.0, tailEnd)551			critValues.append(p)552		return critValues553