ThirdEyeData/Customer-Conversion-Prediction
1
1#!/usr/local/bin/python32 3# Author: Pranab Ghosh4# 5# Licensed under the Apache License, Version 2.0 (the "License"); you6# may not use this file except in compliance with the License. You may7# obtain a copy of the License at8#9# http://www.apache.org/licenses/LICENSE-2.0 10#11# Unless required by applicable law or agreed to in writing, software12# distributed under the License is distributed on an "AS IS" BASIS,13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or14# implied. See the License for the specific language governing15# permissions and limitations under the License.16 17import os18import sys19from random import randint20import random21import time22import uuid23from datetime import datetime24import math25import numpy as np26import pandas as pd27import matplotlib.pyplot as plt28import numpy as np29import logging30import logging.handlers31import pickle32from contextlib import contextmanager33 34tokens = ["0","1","2","3","4","5","6","7","8","9","A","B","C","D","E","F","G","H","I","J","K","L","M",35 "N","O","P","Q","R","S","T","U","V","W","X","Y","Z","0","1","2","3","4","5","6","7","8","9"]36numTokens = tokens[:10]37alphaTokens = tokens[10:36]38loCaseChars = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k","l","m","n","o",39"p","q","r","s","t","u","v","w","x","y","z"]40 41typeInt = "int"42typeFloat = "float"43typeString = "string"44 45secInMinute = 6046secInHour = 60 * 6047secInDay = 24 * secInHour48secInWeek = 7 * secInDay49secInYear = 365 * secInDay50secInMonth = secInYear / 1251 52minInHour = 6053minInDay = 24 * minInHour54 55ftPerYard = 356ftPerMile = ftPerYard * 176057 58 59def genID(size):60 """61 generates ID62 63 Parameters64 size : size of ID65 """66 id = ""67 for i in range(size):68 id = id + selectRandomFromList(tokens)69 return id70 71def genIdList(numId, idSize):72 """73 generate list of IDs74 75 Parameters:76 numId: number of Ids77 idSize: ID size78 """79 iDs = []80 for i in range(numId):81 iDs.append(genID(idSize))82 return iDs83 84def genNumID(size):85 """86 generates ID consisting of digits onl87 88 Parameters89 size : size of ID90 """91 id = ""92 for i in range(size):93 id = id + selectRandomFromList(numTokens)94 return id95 96def genLowCaseID(size):97 """98 generates ID consisting of lower case chars99 100 Parameters101 size : size of ID102 """103 id = ""104 for i in range(size):105 id = id + selectRandomFromList(loCaseChars)106 return id107 108def genNumIdList(numId, idSize):109 """110 generate list of numeric IDs111 112 Parameters:113 numId: number of Ids114 idSize: ID size115 """116 iDs = []117 for i in range(numId):118 iDs.append(genNumID(idSize))119 return iDs120 121def genNameInitial():122 """123 generate name initial124 """125 return selectRandomFromList(alphaTokens) + selectRandomFromList(alphaTokens)126 127def genPhoneNum(arCode):128 """129 generates phone number130 131 Parameters132 arCode: area code133 """134 phNum = genNumID(7)135 return arCode + str(phNum)136 137def selectRandomFromList(ldata):138 """139 select an element randomly from a lis140 141 Parameters142 ldata : list data143 """144 return ldata[randint(0, len(ldata)-1)]145 146def selectOtherRandomFromList(ldata, cval):147 """148 select an element randomly from a list excluding the given one149 150 Parameters151 ldata : list data152 cval : value to be excluded153 """154 nval = selectRandomFromList(ldata)155 while nval == cval:156 nval = selectRandomFromList(ldata)157 return nval158 159def selectRandomSubListFromList(ldata, num):160 """161 generates random sublist from a list without replacemment162 163 Parameters164 ldata : list data165 num : output list size166 """167 assertLesser(num, len(ldata), "size of sublist to be sampled greater than or equal to main list")168 i = randint(0, len(ldata)-1)169 sel = ldata[i]170 selSet = {i}171 selList = [sel]172 while (len(selSet) < num):173 i = randint(0, len(ldata)-1)174 if (i not in selSet):175 sel = ldata[i]176 selSet.add(i)177 selList.append(sel) 178 return selList179 180def selectRandomSubListFromListWithRepl(ldata, num):181 """182 generates random sublist from a list with replacemment183 184 Parameters185 ldata : list data186 num : output list size187 188 """189 return list(map(lambda i : selectRandomFromList(ldata), range(num)))190 191def selectRandomFromDict(ddata):192 """193 select an element randomly from a dictionary194 195 Parameters196 ddata : dictionary data197 """198 dkeys = list(ddata.keys())199 dk = selectRandomFromList(dkeys)200 el = (dk, ddata[dk])201 return el202 203def setListRandomFromList(ldata, ldataRepl):204 """205 sets some elents in the first list randomly with elements from the second list206 207 Parameters208 ldata : list data209 ldataRepl : list with replacement data210 """211 l = len(ldata)212 selSet = set()213 for d in ldataRepl:214 i = randint(0, l-1)215 while i in selSet:216 i = randint(0, l-1)217 ldata[i] = d218 selSet.add(i)219 220def genIpAddress():221 """222 generates IP address223 """224 i1 = randint(0,256)225 i2 = randint(0,256)226 i3 = randint(0,256)227 i4 = randint(0,256)228 ip = "%d.%d.%d.%d" %(i1,i2,i3,i4)229 return ip230 231def curTimeMs():232 """233 current time in ms234 """235 return int((datetime.utcnow() - datetime(1970,1,1)).total_seconds() * 1000)236 237def secDegPolyFit(x1, y1, x2, y2, x3, y3):238 """239 second deg polynomial 240 241 Parameters242 x1 : 1st point x243 y1 : 1st point y244 x2 : 2nd point x245 y2 : 2nd point y246 x3 : 3rd point x247 y3 : 3rd point y248 """249 t = (y1 - y2) / (x1 - x2)250 a = t - (y2 - y3) / (x2 - x3)251 a = a / (x1 - x3)252 b = t - a * (x1 + x2)253 c = y1 - a * x1 * x1 - b * x1254 return (a, b, c)255 256def range_limit(val, minv, maxv):257 """258 range limit a value259 260 Parameters261 val : data value262 minv : minimum263 maxv : maximum264 """265 if (val < minv):266 val = minv267 elif (val > maxv):268 val = maxv269 return val 270 271def rangeLimit(val, minv, maxv):272 """273 range limit a value274 275 Parameters276 val : data value277 minv : minimum278 maxv : maximum279 """280 return range_limit(val, minv, maxv)281 282def isInRange(val, minv, maxv):283 """284 checks if within range285 286 Parameters287 val : data value288 minv : minimum289 maxv : maximum290 """291 return val >= minv and val <= maxv292 293def stripFileLines(filePath, offset):294 """295 strips number of chars from both ends296 297 Parameters298 filePath : file path299 offset : offset from both ends of line 300 """301 fp = open(filePath, "r")302 for line in fp:303 stripped = line[offset:len(line) - 1 - offset]304 print (stripped)305 fp.close()306 307def genLatLong(lat1, long1, lat2, long2):308 """309 generate lat log within limits310 311 Parameters312 lat1 : lat of 1st point313 long1 : long of 1st point314 lat2 : lat of 2nd point315 long2 : long of 2nd point316 """317 lat = lat1 + (lat2 - lat1) * random.random()318 longg = long1 + (long2 - long1) * random.random()319 return (lat, longg)320 321def geoDistance(lat1, long1, lat2, long2):322 """323 find geo distance in ft324 325 Parameters326 lat1 : lat of 1st point327 long1 : long of 1st point328 lat2 : lat of 2nd point329 long2 : long of 2nd point330 """331 latDiff = math.radians(lat1 - lat2)332 longDiff = math.radians(long1 - long2)333 l1 = math.sin(latDiff/2.0)334 l2 = math.sin(longDiff/2.0)335 l3 = math.cos(math.radians(lat1))336 l4 = math.cos(math.radians(lat2))337 a = l1 * l1 + l3 * l4 * l2 * l2338 l5 = math.sqrt(a)339 l6 = math.sqrt(1.0 - a)340 c = 2.0 * math.atan2(l5, l6)341 r = 6371008.8 * 3.280840342 return c * r343 344def minLimit(val, limit):345 """346 min limit347 Parameters348 349 """350 if (val < limit):351 val = limit352 return val;353 354def maxLimit(val, limit):355 """356 max limit357 Parameters358 359 """360 if (val > limit):361 val = limit362 return val;363 364def rangeSample(val, minLim, maxLim):365 """366 if out side range sample within range367 368 Parameters369 val : value370 minLim : minimum371 maxLim : maximum372 """373 if val < minLim or val > maxLim:374 val = randint(minLim, maxLim)375 return val376 377def genRandomIntListWithinRange(size, minLim, maxLim):378 """379 random unique list of integers within range380 381 Parameters382 size : size of returned list383 minLim : minimum384 maxLim : maximum385 """386 values = set()387 for i in range(size):388 val = randint(minLim, maxLim)389 while val not in values:390 values.add(val)391 return list(values)392 393def preturbScalar(value, vrange, distr="uniform"):394 """395 preturbs a mutiplicative value within range396 397 Parameters398 value : data value399 vrange : value delta fraction400 distr : noise distribution type401 """402 if distr == "uniform":403 scale = 1.0 - vrange + 2 * vrange * random.random() 404 elif distr == "normal":405 scale = 1.0 + np.random.normal(0, vrange)406 else:407 exisWithMsg("unknown noise distr " + distr)408 return value * scale409 410def preturbScalarAbs(value, vrange):411 """412 preturbs an absolute value within range413 414 Parameters415 value : data value416 vrange : value delta absolute417 418 """419 delta = - vrange + 2.0 * vrange * random.random() 420 return value + delta421 422def preturbVector(values, vrange):423 """424 preturbs a list within range425 426 Parameters427 values : list data428 vrange : value delta fraction429 """430 nValues = list(map(lambda va: preturbScalar(va, vrange), values))431 return nValues432 433def randomShiftVector(values, smin, smax):434 """435 shifts a list by a random quanity with a range436 437 Parameters438 values : list data439 smin : samplinf minimum440 smax : sampling maximum441 """442 shift = np.random.uniform(smin, smax)443 return list(map(lambda va: va + shift, values))444 445def floatRange(beg, end, incr):446 """447 generates float range448 449 Parameters450 beg :range begin451 end: range end452 incr : range increment453 """454 return list(np.arange(beg, end, incr))455 456def shuffle(values, *numShuffles):457 """458 in place shuffling with swap of pairs459 460 Parameters461 values : list data462 numShuffles : parameter list for number of shuffles463 """464 size = len(values)465 if len(numShuffles) == 0:466 numShuffle = int(size / 2)467 elif len(numShuffles) == 1:468 numShuffle = numShuffles[0]469 else:470 numShuffle = randint(numShuffles[0], numShuffles[1])471 print("numShuffle {}".format(numShuffle))472 for i in range(numShuffle):473 first = random.randint(0, size - 1)474 second = random.randint(0, size - 1)475 while first == second:476 second = random.randint(0, size - 1)477 tmp = values[first]478 values[first] = values[second]479 values[second] = tmp480 481 482def splitList(itms, numGr):483 """484 splits a list into sub lists of approximately equal size, with items in sublists randomly chod=sen485 486 Parameters487 itms ; list of values 488 numGr : no of groups489 """490 tcount = len(itms)491 cItems = list(itms)492 sz = int(len(cItems) / numGr)493 groups = list()494 count = 0495 for i in range(numGr):496 if (i == numGr - 1):497 csz = tcount - count498 else:499 csz = sz + randint(-2, 2)500 count += csz501 gr = list()502 for j in range(csz):503 it = selectRandomFromList(cItems)504 gr.append(it) 505 cItems.remove(it) 506 groups.append(gr)507 return groups 508 509def multVector(values, vrange):510 """511 multiplies a list within value range512 513 Parameters514 values : list of values515 vrange : fraction of vaue to be used to update516 """517 scale = 1.0 - vrange + 2 * vrange * random.random()518 nValues = list(map(lambda va: va * scale, values))519 return nValues520 521def weightedAverage(values, weights):522 """523 calculates weighted average524 525 Parameters526 values : list of values527 weights : list of weights528 """ 529 assert len(values) == len(weights), "values and weights should be same size"530 vw = zip(values, weights)531 wva = list(map(lambda e : e[0] * e[1], vw))532 #wa = sum(x * y for x, y in vw) / sum(weights)533 wav = sum(wva) / sum(weights)534 return wav535 536def extractFields(line, delim, keepIndices):537 """538 breaks a line into fields and keeps only specified fileds and returns new line539 540 Parameters541 line ; deli separated string542 delim : delemeter543 keepIndices : list of indexes to fields to be retained544 """545 items = line.split(delim)546 newLine = []547 for i in keepIndices:548 newLine.append(line[i])549 return delim.join(newLine)550 551def remFields(line, delim, remIndices):552 """553 removes fields from delim separated string554 555 Parameters556 line ; delemeter separated string557 delim : delemeter558 remIndices : list of indexes to fields to be removed559 """560 items = line.split(delim)561 newLine = []562 for i in range(len(items)):563 if not arrayContains(remIndices, i):564 newLine.append(line[i])565 return delim.join(newLine)566 567def extractList(data, indices):568 """569 extracts list from another list, given indices570 571 Parameters572 remIndices : list data573 indices : list of indexes to fields to be retained574 """575 if areAllFieldsIncluded(data, indices):576 exList = data.copy()577 #print("all indices")578 else:579 exList = list()580 le = len(data)581 for i in indices:582 assert i < le , "index {} out of bound {}".format(i, le)583 exList.append(data[i])584 585 return exList586 587def arrayContains(arr, item):588 """589 checks if array contains an item 590 591 Parameters592 arr : list data593 item : item to search594 """595 contains = True596 try:597 arr.index(item)598 except ValueError:599 contains = False600 return contains601 602def strToIntArray(line, delim=","): 603 """604 int array from delim separated string605 606 Parameters607 line ; delemeter separated string608 """609 arr = line.split(delim)610 return [int(a) for a in arr]611 612def strToFloatArray(line, delim=","): 613 """614 float array from delim separated string615 616 Parameters617 line ; delemeter separated string618 """619 arr = line.split(delim)620 return [float(a) for a in arr]621 622def strListOrRangeToIntArray(line): 623 """624 int array from delim separated string or range625 626 Parameters627 line ; delemeter separated string628 """629 varr = line.split(",")630 if (len(varr) > 1):631 iarr = list(map(lambda v: int(v), varr))632 else:633 vrange = line.split(":")634 if (len(vrange) == 2):635 lo = int(vrange[0])636 hi = int(vrange[1])637 iarr = list(range(lo, hi+1))638 else:639 iarr = [int(line)]640 return iarr641 642def toStr(val, precision):643 """644 converts any type to string 645 646 Parameters647 val : value648 precision ; precision for float value649 """650 if type(val) == float or type(val) == np.float64 or type(val) == np.float32:651 format = "%" + ".%df" %(precision)652 sVal = format %(val)653 else:654 sVal = str(val)655 return sVal656 657def toStrFromList(values, precision, delim=","):658 """659 converts list of any type to delim separated string660 661 Parameters662 values : list data663 precision ; precision for float value664 delim : delemeter665 """666 sValues = list(map(lambda v: toStr(v, precision), values))667 return delim.join(sValues)668 669def toIntList(values):670 """671 convert to int list672 673 Parameters674 values : list data675 """676 return list(map(lambda va: int(va), values))677 678def toFloatList(values):679 """680 convert to float list681 682 Parameters683 values : list data684 685 """686 return list(map(lambda va: float(va), values))687 688def toStrList(values, precision=None):689 """690 convert to string list691 692 Parameters693 values : list data694 precision ; precision for float value695 """696 return list(map(lambda va: toStr(va, precision), values))697 698def toIntFromBoolean(value):699 """700 convert to int701 702 Parameters703 value : boolean value704 """705 ival = 1 if value else 0706 return ival707 708def scaleBySum(ldata):709 """710 scales so that sum is 1711 712 Parameters713 ldata : list data714 """715 s = sum(ldata)716 return list(map(lambda e : e/s, ldata))717 718def scaleByMax(ldata):719 """720 scales so that max value is 1721 722 Parameters723 ldata : list data724 """725 m = max(ldata)726 return list(map(lambda e : e/m, ldata))727 728def typedValue(val, dtype=None):729 """730 return typed value given string, discovers data type if not specified731 732 Parameters733 val : value734 dtype : data type735 """736 tVal = None737 738 if dtype is not None:739 if dtype == "num":740 dtype = "int" if dtype.find(".") == -1 else "float"741 742 if dtype == "int":743 tVal = int(val)744 elif dtype == "float":745 tVal = float(val)746 elif dtype == "bool":747 tVal = bool(val)748 else:749 tVal = val750 else:751 if type(val) == str:752 lVal = val.lower()753 754 #int755 done = True756 try:757 tVal = int(val)758 except ValueError:759 done = False760 761 #float762 if not done: 763 done = True764 try:765 tVal = float(val)766 except ValueError:767 done = False768 769 #boolean770 if not done:771 done = True772 if lVal == "true":773 tVal = True774 elif lVal == "false":775 tVal = False776 else:777 done = False778 #None 779 if not done:780 if lVal == "none":781 tVal = None782 else:783 tVal = val784 else:785 tVal = val 786 787 return tVal788 789def isInt(val):790 """791 return true if string is int and the typed value792 793 Parameters794 val : value795 """796 valInt = True797 try:798 tVal = int(val)799 except ValueError:800 valInt = False801 tVal = None802 r = (valInt, tVal)803 return r804 805def isFloat(val):806 """807 return true if string is float808 809 Parameters810 val : value811 """812 valFloat = True813 try:814 tVal = float(val)815 except ValueError:816 valFloat = False817 tVal = None818 r = (valFloat, tVal)819 return r820 821def getAllFiles(dirPath):822 """823 get all files recursively824 825 Parameters826 dirPath : directory path827 """828 filePaths = []829 for (thisDir, subDirs, fileNames) in os.walk(dirPath):830 for fileName in fileNames:831 filePaths.append(os.path.join(thisDir, fileName))832 filePaths.sort()833 return filePaths834 835def getFileContent(fpath, verbose=False):836 """837 get file contents in directory838 839 Parameters840 fpath ; directory path841 verbose : verbosity flag842 """843 # dcument list844 docComplete = []845 filePaths = getAllFiles(fpath)846 847 # read files848 for filePath in filePaths:849 if verbose:850 print("next file " + filePath)851 with open(filePath, 'r') as contentFile:852 content = contentFile.read()853 docComplete.append(content)854 return (docComplete, filePaths)855 856def getOneFileContent(fpath):857 """858 get one file contents859 860 Parameters861 fpath : file path862 """863 with open(fpath, 'r') as contentFile:864 docStr = contentFile.read()865 return docStr866 867def getFileLines(dirPath, delim=","):868 """869 get lines from a file870 871 Parameters872 dirPath : file path873 delim : delemeter874 """875 lines = list()876 for li in fileRecGen(dirPath, delim):877 lines.append(li) 878 return lines879 880def getFileSampleLines(dirPath, percen, delim=","):881 """882 get sampled lines from a file883 884 Parameters885 dirPath : file path886 percen : sampling percentage887 delim : delemeter888 """889 lines = list()890 for li in fileRecGen(dirPath, delim):891 if randint(0, 100) < percen:892 lines.append(li) 893 return lines894 895def getFileColumnAsString(dirPath, index, delim=","):896 """897 get string column from a file898 899 Parameters900 dirPath : file path901 index : index902 delim : delemeter903 """904 fields = list()905 for rec in fileRecGen(dirPath, delim):906 fields.append(rec[index]) 907 #print(fields) 908 return fields909 910def getFileColumnsAsString(dirPath, indexes, delim=","):911 """912 get multiple string columns from a file913 914 Parameters915 dirPath : file path916 indexes : indexes of columns917 delim : delemeter918 919 """920 nindex = len(indexes)921 columns = list(map(lambda i : list(), range(nindex)))922 for rec in fileRecGen(dirPath, delim):923 for i in range(nindex):924 columns[i].append(rec[indexes[i]]) 925 return columns926 927def getFileColumnAsFloat(dirPath, index, delim=","):928 """929 get float fileds from a file930 931 Parameters932 dirPath : file path933 index : index934 delim : delemeter935 936 """937 #print("{} {}".format(dirPath, index))938 fields = getFileColumnAsString(dirPath, index, delim)939 return list(map(lambda v:float(v), fields))940 941def getFileColumnAsInt(dirPath, index, delim=","):942 """943 get float fileds from a file944 945 Parameters946 dirPath : file path947 index : index948 delim : delemeter949 """950 fields = getFileColumnAsString(dirPath, index, delim)951 return list(map(lambda v:int(v), fields))952 953def getFileAsIntMatrix(dirPath, columns, delim=","):954 """955 extracts int matrix from csv file given column indices with each row being concatenation of 956 extracted column values row size = num of columns957 958 Parameters959 dirPath : file path960 columns : indexes of columns961 delim : delemeter962 """963 mat = list()964 for rec in fileSelFieldsRecGen(dirPath, columns, delim):965 mat.append(asIntList(rec))966 return mat967 968def getFileAsFloatMatrix(dirPath, columns, delim=","):969 """970 extracts float matrix from csv file given column indices with each row being concatenation of 971 extracted column values row size = num of columns972 973 Parameters974 dirPath : file path975 columns : indexes of columns976 delim : delemeter977 """978 mat = list()979 for rec in fileSelFieldsRecGen(dirPath, columns, delim):980 mat.append(asFloatList(rec))981 return mat982 983def getFileAsFloatColumn(dirPath):984 """985 grt float list from a file with one float per row986 987 Parameters988 dirPath : file path989 """990 flist = list()991 for rec in fileRecGen(dirPath, None):992 flist.append(float(rec))993 return flist994 995def getFileAsFiltFloatMatrix(dirPath, filt, columns, delim=","):996 """997 extracts float matrix from csv file given row filter and column indices with each row being 998 concatenation of extracted column values row size = num of columns999 1000 Parameters1001 dirPath : file path1002 columns : indexes of columns1003 filt : row filter lambda1004 delim : delemeter1005 1006 """1007 mat = list()1008 for rec in fileFiltSelFieldsRecGen(dirPath, filt, columns, delim):1009 mat.append(asFloatList(rec))1010 return mat1011 1012def getFileAsTypedRecords(dirPath, types, delim=","):1013 """1014 extracts typed records from csv file with each row being concatenation of 1015 extracted column values 1016 1017 Parameters1018 dirPath : file path1019 types : data types1020 delim : delemeter1021 """1022 (dtypes, cvalues) = extractTypesFromString(types) 1023 tdata = list()1024 for rec in fileRecGen(dirPath, delim):1025 trec = list()1026 for index, value in enumerate(rec):1027 value = __convToTyped(index, value, dtypes)1028 trec.append(value)1029 tdata.append(trec)1030 return tdata1031 1032 1033def getFileColsAsTypedRecords(dirPath, columns, types, delim=","):1034 """1035 extracts typed records from csv file given column indices with each row being concatenation of 1036 extracted column values 1037 1038 Parameters1039 Parameters1040 dirPath : file path1041 columns : column indexes1042 types : data types1043 delim : delemeter1044 """1045 (dtypes, cvalues) = extractTypesFromString(types) 1046 tdata = list()1047 for rec in fileSelFieldsRecGen(dirPath, columns, delim):1048 trec = list()1049 for indx, value in enumerate(rec):1050 tindx = columns[indx]1051 value = __convToTyped(tindx, value, dtypes)1052 trec.append(value)1053 tdata.append(trec)1054 return tdata1055 1056def getFileColumnsMinMax(dirPath, columns, dtype, delim=","):1057 """1058 extracts numeric matrix from csv file given column indices. For each column return min and max1059 1060 Parameters1061 dirPath : file path1062 columns : column indexes1063 dtype : data type1064 delim : delemeter1065 """1066 dtypes = list(map(lambda c : str(c) + ":" + dtype, columns))1067 dtypes = ",".join(dtypes)1068 #print(dtypes)1069 1070 tdata = getFileColsAsTypedRecords(dirPath, columns, dtypes, delim)1071 minMax = list()1072 ncola = len(tdata[0])1073 ncole = len(columns)1074 assertEqual(ncola, ncole, "actual no of columns different from expected")1075 1076 for ci in range(ncole): 1077 vmin = sys.float_info.max1078 vmax = sys.float_info.min1079 for r in tdata:1080 cv = r[ci]1081 vmin = cv if cv < vmin else vmin1082 vmax = cv if cv > vmax else vmax1083 mm = (vmin, vmax, vmax - vmin)1084 minMax.append(mm)1085 1086 return minMax1087 1088 1089def getRecAsTypedRecord(rec, types, delim=None):1090 """1091 converts record to typed records 1092 1093 Parameters1094 rec : delemeter separate string or list of string1095 types : field data types1096 delim : delemeter1097 """ 1098 if delim is not None:1099 rec = rec.split(delim)1100 (dtypes, cvalues) = extractTypesFromString(types) 1101 #print(types)1102 #print(dtypes)1103 trec = list()1104 for ind, value in enumerate(rec):1105 tvalue = __convToTyped(ind, value, dtypes)1106 trec.append(tvalue)1107 return trec1108 1109def __convToTyped(index, value, dtypes):1110 """1111 convert to typed value 1112 1113 Parameters1114 index : index in type list1115 value : data value1116 dtypes : data type list1117 """1118 #print(index, value)1119 dtype = dtypes[index]1120 tvalue = value1121 if dtype == "int":1122 tvalue = int(value)1123 elif dtype == "float":1124 tvalue = float(value)1125 return tvalue1126 1127 1128 1129def extractTypesFromString(types):1130 """1131 extracts column data types and set values for categorical variables 1132 1133 Parameters1134 types : encoded type information1135 """1136 ftypes = types.split(",")1137 dtypes = dict()1138 cvalues = dict()1139 for ftype in ftypes:1140 items = ftype.split(":") 1141 cindex = int(items[0])1142 dtype = items[1]1143 dtypes[cindex] = dtype1144 if len(items) == 3:1145 sitems = items[2].split()1146 cvalues[cindex] = sitems1147 return (dtypes, cvalues)1148 1149def getMultipleFileAsInttMatrix(dirPathWithCol, delim=","):1150 """1151 extracts int matrix from from csv files given column index for each file. 1152 num of columns = number of rows in each file and num of rows = number of files1153 1154 Parameters1155 dirPathWithCol: list of file path and collumn index pair1156 delim : delemeter1157 """1158 mat = list()1159 minLen = -11160 for path, col in dirPathWithCol:1161 colVals = getFileColumnAsInt(path, col, delim)1162 if minLen < 0 or len(colVals) < minLen:1163 minLen = len(colVals)1164 mat.append(colVals)1165 1166 #make all same length1167 mat = list(map(lambda li:li[:minLen], mat)) 1168 return mat1169 1170def getMultipleFileAsFloatMatrix(dirPathWithCol, delim=","):1171 """1172 extracts float matrix from from csv files given column index for each file. 1173 num of columns = number of rows in each file and num of rows = number of files1174 1175 Parameters1176 dirPathWithCol: list of file path and collumn index pair1177 delim : delemeter1178 """1179 mat = list()1180 minLen = -11181 for path, col in dirPathWithCol:1182 colVals = getFileColumnAsFloat(path, col, delim)1183 if minLen < 0 or len(colVals) < minLen:1184 minLen = len(colVals)1185 mat.append(colVals)1186 1187 #make all same length1188 mat = list(map(lambda li:li[:minLen], mat)) 1189 return mat1190 1191def writeStrListToFile(ldata, filePath, delem=","):1192 """1193 writes list of dlem separated string or list of list of string to afile1194 1195 Parameters1196 ldata : list data1197 filePath : file path1198 delim : delemeter1199 """1200 with open(filePath, "w") as fh: