jarvez/Object_representation_model-api
0
1import tensorflow as tf
2from tensorflow import keras
3from keras.regularizers import Regularizer
4###############------------------- FOR AZRA ---------------------#############
5
6from tensorflow.keras.models import model_from_yaml, model_from_json
7from tensorflow import keras
8# from functions import *
9import pickle, shapely
10import scipy
11
12from scipy.ndimage import gaussian_filter
13import scipy
14from scipy import signal, ndimage
15from sklearn.model_selection import train_test_split
16import numpy as np
17import math as mt
18import tempfile
19import tensorflow as tf
20import pickle
21from scipy import misc
22import glob, csv
23from tensorflow.keras import layers
24from shapely.geometry import box, Polygon, Point, LinearRing
25#from tensorflow.keras.datasets import mnist
26from tensorflow.keras.models import Sequential
27from tensorflow.keras.models import Model
28from tensorflow.keras.layers import Dense, Activation, Flatten, Input, Reshape, Lambda
29from tensorflow.keras.layers import Conv2D, MaxPooling2D, AveragePooling2D, UpSampling2D, concatenate, Concatenate
30from keras.layers import Dropout, GlobalAveragePooling2D
31from keras.regularizers import l2
32from tensorflow.keras.losses import categorical_crossentropy
33from tensorflow.keras.constraints import max_norm
34import matplotlib.pyplot as plt
35from tensorflow.keras import backend as K
36from keras.regularizers import Regularizer
37# from keras_gcn import GraphConv
38import numpy as np
39from tensorflow.keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
40import os
41from sklearn import preprocessing
42from numpy import linalg as LA
43import pandas as pd
44from tensorflow.keras.utils import plot_model
45from numpy import matlib
46from progressbar import ProgressBar
47import tensorflow_model_optimization as tfmot
48from matplotlib import cm
49from keras.layers import Dropout, GlobalAveragePooling2D
50from keras.regularizers import l2
51from tensorflow.keras.losses import categorical_crossentropy
52from tensorflow.keras.constraints import max_norm
53# main = "Bharat_local_runs/"
54
55def rew_new(x, y, obj_boun, present=False):
56 rew = []
57 if present:
58 for i in range(len(x)):
59 kk = Point(x[i],y[i])
60 for ii in obj_boun:
61 bb = box(ii[0], ii[1], ii[2], ii[3])
62 if bb.contains(kk):
63 rew.append(1)
64 else:
65 rew.append(0)
66 else:
67 rew = [0]*len(x)
68
69 return np.asarray(rew)
70
71
72def rew(x, y, theta, objs, obj_boun, env, env_boun, present=False):
73 reward = []
74 # obj = k2
75 # obj_boun = k3
76 for i in range(len(x)):
77 if present:
78 if present:
79 for pp in range(len(objs)):
80 k2 = obj[pp]
81 k3 = obj_boun[pp]
82 if((max(k3)[0] >= x[i] >= max(k2)[0]) and (max(k2)[1] >= y[i] >= min(k2)[1]) and (90 < theta[i] < 270)):
83 reward.append(1)
84 elif((min(k3)[0] <= x[i] <= min(k2)[0]) and (max(k2)[1] >= y[i] >= min(k2)[1]) and (90>theta[i] or theta[i]>270)):
85 reward.append(1)
86 elif((min(k3)[1] <= y[i] <= min(k2)[1]) and (max(k2)[0] >= x[i] >= min(k2)[0]) and (180>theta[i]>0 )):
87 reward.append(1)
88 elif((max(k3)[1] >= y[i] >= max(k2)[1]) and (max(k2[0]) >= x[i] >= min(k2)[0]) and (360>theta[i]>180)):
89 reward.append(1)
90
91 elif((min(k3)[0] <= x[i] <= min(k2)[0]) and (max(k3)[1] >= y[i] >= max(k2)[1]) and (15>theta[i] or theta[i]>255)):
92 reward.append(1)
93 elif((min(k3)[0] <= x[i] <= min(k2)[0]) and (min(k3)[1] <= y[i] <= min(k2[1])) and (105>theta[i] or theta[i]>335)):
94 reward.append(1)
95 elif((max(k3)[0] >= x[i] >= max(k2)[0]) and (min(k3)[1] <= y[i] <= min(k2)[1]) and (195>theta[i]>75)):
96 reward.append(1)
97 elif((max(k3)[0] >= x[i] >= max(k2)[0]) and (max(k3)[1] >= y[i] >= max(k2)[1]) and (285>theta[i]>165)):
98 reward.append(1)
99
100 if len(reward) != i+1:
101 if((max(k1_env[0]) >= x[i] >= max(k1)[0]) and (90>theta[i] or theta[i]>270)):
102 reward.append(0)
103 elif((min(k1_env[0]) <= x[i] <= min(k1)[0]) and (270>theta[i]>90)):
104 reward.append(0)
105 elif((max(k1_env[1]) >= y[i] >= max(k1)[1]) and (180>theta[i]>0)):
106 reward.append(0)
107 elif((min(k1_env)[1] <= y[i] <= min(k1)[1]) and (360>theta[i]>180)):
108 reward.append(0)
109 elif len(reward) != i+1:
110 reward.append(0)
111
112 else:
113 if((max(k1_env[0]) >= x[i] >= max(k1)[0]) and (90>theta[i] or theta[i]>270)):
114 reward.append(0)
115 elif((min(k1_env[0]) <= x[i] <= min(k1)[0]) and (270>theta[i]>90)):
116 reward.append(0)
117 elif((max(k1_env[1]) >= y[i] >= max(k1)[1]) and (180>theta[i]>0)):
118 reward.append(0)
119 elif((min(k1_env)[1] <= y[i] <= min(k1)[1]) and (360>theta[i]>180)):
120 reward.append(0)
121 elif len(reward) != i+1:
122 reward.append(0)
123
124 reward = np.asarray(reward)
125 return reward
126
127
128#%% HD
129def HD(s, t):
130 with open(main + 'hd_som_wt2.pk1', 'rb') as k:
131 wt2 = pickle.load(k)
132
133 # phase1d = np.zeros((100, 1))
134 PI2d = np.zeros((10, 10))
135 k = PI2d.shape
136 trj_hd_resp = []
137
138 for j in range(len(s)):
139 if (j%10000 == 0):
140 print(j)
141 X1 = [mt.cos(mt.radians(t[0])), mt.sin(mt.radians(t[0]))]
142 X2 = [mt.cos(mt.radians(t[j])), mt.sin(mt.radians(t[j]))]
143 s1 = X2[0]*X1[1] - X1[0]*X2[1]
144 s2 = X2[0]*X1[0] + X1[1]*X2[1]
145 # print(s2)
146 X = [s1, s2]
147 y_p = repsom2dlinear(X, wt2)
148 trj_hd_resp.append(y_p)
149 print("HD response computed")
150 return trj_hd_resp
151
152#%%
153# def PI(resp, s):
154# X, Y, theta = np.zeros((100,1)), np.ones((100,1)), [[0]*100]
155# bf = 2*6*mt.pi
156# dt = np.divide(1, 100)
157# betaa, t, Xbg, Ybg = 50, 0, 1, 0
158# tarr = []
159# for ii in range(1,len(resp)):
160# if (ii%10000 == 0):
161# print(ii)
162
163
164# y_q = resp[ii]
165# inp1d = np.reshape(np.transpose(y_q),(100,1))
166# theta_dot = [(bf + betaa * s[ii] * k[0] * 10) for k in inp1d]
167# theta_dot[:] = [x*dt for x in theta_dot]
168# theta.append([i+j for i,j in zip(theta[ii-1], theta_dot)])
169
170# theta = np.transpose(np.asarray(theta))
171# # print(theta.shape)
172# Xarr = np.cos(theta)
173# PI1d = Xarr
174# return PI1d
175
176
177def repsom2dlinear(x, wt):
178 sz_wt = list(wt.shape)
179 y = np.zeros((sz_wt[0], sz_wt[1]))
180 if(sz_wt[2] != len(x)):
181 print('Invalid input size in repsom2d()\n')
182 return
183
184 for i in range(sz_wt[0]):
185 for j in range(sz_wt[1]):
186 v = wt[i][j].reshape(sz_wt[2], 1)
187 # print(v)
188 y[i][j] = np.dot(x,v)
189 return y
190
191
192def unitvec(pos_corr):
193 temp1 = np.subtract(pos_corr[1:, :], pos_corr[:-1, :])
194 temp2 = np.sqrt((temp1*temp1).sum(axis=1))
195 temp3 = temp1 / temp2.reshape(temp1.shape[0],1)
196 return temp3
197
198
199def relu(input):
200 if input > 0:
201 return input
202 else:
203 return 0
204
205
206def test_train(dat, p):
207 train_dat = np.asarray([dat[k] for k in range(len(dat)) if not k%p==0])
208 test_dat = np.asarray([dat[k] for k in range(len(dat)) if k%p==0])
209 return [train_dat, test_dat]
210
211
212def mse(data, pred_data):
213 mse_ = np.sum(np.square(data - pred_data))/len(data)
214 return mse_
215
216
217def seq_data(data, seq_len):
218 temp1 = []
219
220 for i in range(seq_len, data.shape[0]):
221 temp3 = data[i-seq_len:i]
222 temp1.append(temp3)
223 temp1 = np.asarray(temp1)
224
225 return temp1
226
227
228class FF(tf.keras.layers.Layer):
229
230 def __init__(self, units, **kwargs):
231 super(FF, self).__init__(**kwargs)
232 self.units = units
233 self.state_size = units
234 self.j_h = tf.keras.layers.Dense(self.units)
235 self.j_x = tf.keras.layers.Dense(self.units)
236 self.k_h = tf.keras.layers.Dense(self.units)
237 self.k_x = tf.keras.layers.Dense(self.units)
238
239 def build(self, input_shape):
240 self.built = True
241
242 def get_config(self):
243 return {'units': self.units}
244
245 def call(self, inputs, states):
246 #print("FF:", inputs, states)
247 prev_output = states[0]
248 j = tf.sigmoid(self.j_x(inputs) + self.j_h(prev_output))
249 k = tf.sigmoid(self.k_x(inputs) + self.k_h(prev_output))
250 output = j * (1 - prev_output) + (1 - k) * prev_output
251 return output, [output]
252
253def firing_rate_map(firposgrid, ot, firr, title):
254 res = 45
255 #firr = list(firr[0])
256 x = np.arange(-1, 1, 1/res)
257 y = np.arange(-1, 1, 1/res)
258 fx,fy = np.meshgrid(x, y)
259 firingmap = np.zeros(fx.shape)
260 #gridpoint_x = np.asarray(np.reshape(fx, np.size(fx), 1))
261 #gridpoint_y = np.asarray(np.reshape(fx, np.size(fx), 1))
262 #gridpoint = np.transpose([gridpoint_x, gridpoint_y])
263 #roundinggridpoint = np.round(gridpoint)
264 #firposround = np.round(firposgrid)
265 firingvalue = ot[firr]
266 for ii in range(len(firposgrid)):
267 q1 = np.argmin(abs(firposgrid[ii,0] - fx[1,:]))
268 q2 = np.argmin(abs(firposgrid[ii,1] - fx[1,:]))
269 firingmap[q1,q2] = firingvalue[ii]
270 firingmap = firingmap/max(np.max(firingmap),1)
271 gaussian = matlab_style_gauss2D([10, 10], 1.5)
272 spikes_smooth = scipy.signal.convolve2d(gaussian, firingmap)
273 rotated_img = ndimage.rotate(spikes_smooth, 1*270)
274 #np.rot90([spikes_smooth], 2)
275 # plt.imshow(rotated_img, origin= 'upper')
276 # plt.title(title)
277 # plt.colorbar()
278 # ax=plt.gca() # get the axis
279 # ax.set_ylim(ax.get_ylim()[::-1]) # invert the axis
280 # ax.set_xlim(ax.get_xlim()[::-1]) # invert the axis
281 # ax.xaxis.tick_bottom() # and move the X-Axis
282 # ax.set_yticklabels([])
283 # ax.set_xticklabels([])
284
285 return rotated_img
286
287def matlab_style_gauss2D(shape,sigma):
288 """
289 2D gaussian mask - should give the same result as MATLAB's
290 fspecial('gaussian',[shape],[sigma])
291 """
292 m,n = [(ss-1.)/2. for ss in shape]
293 y,x = np.ogrid[-m:m+1,-n:n+1]
294 h = np.exp( -(x*x + y*y) / (2.*sigma*sigma) )
295 h[ h < np.finfo(h.dtype).eps*h.max() ] = 0
296 sumh = h.sum()
297 if sumh != 0:
298 h /= sumh
299 return h
300
301class GraphLayer(keras.layers.Layer):
302
303 def __init__(self,
304 step_num=1,
305 activation=None,
306 **kwargs):
307 """Initialize the layer.
308 :param step_num: Two nodes are considered as connected if they could be reached in `step_num` steps.
309 :param activation: The activation function after convolution.
310 :param kwargs: Other arguments for parent class.
311 """
312 self.supports_masking = True
313 self.step_num = step_num
314 self.activation = keras.activations.get(activation)
315 self.supports_masking = True
316 super(GraphLayer, self).__init__(**kwargs)
317
318 def get_config(self):
319 config = {
320 'step_num': self.step_num,
321 'activation': self.activation,
322 }
323 base_config = super(GraphLayer, self).get_config()
324 return dict(list(base_config.items()) + list(config.items()))
325
326 def _get_walked_edges(self, edges, step_num):
327 """Get the connection graph within `step_num` steps
328 :param edges: The graph in single step.
329 :param step_num: Number of steps.
330 :return: The new graph that has the same shape with `edges`.
331 """
332 if step_num <= 1:
333 return edges
334 deeper = self._get_walked_edges(K.batch_dot(edges, edges), step_num // 2)
335 if step_num % 2 == 1:
336 deeper += edges
337 return K.cast(K.greater(deeper, 0.0), K.floatx())
338
339 def call(self, inputs, **kwargs):
340 features, edges = inputs
341 edges = K.cast(edges, K.floatx())
342 if self.step_num > 1:
343 edges = self._get_walked_edges(edges, self.step_num)
344 outputs = self.activation(self._call(features, edges))
345 return outputs
346
347 def _call(self, features, edges):
348 raise NotImplementedError('The class is not intended to be used directly.')
349
350
351class GraphConv(GraphLayer):
352 r"""Graph convolutional layer.
353 h_i^{(t)} = \sigma \left ( \frac{ G_i^T (h_i^{(t - 1)} W + b)}{\sum G_i} \right )
354 """
355
356 def __init__(self,
357 units,
358 kernel_initializer='glorot_uniform',
359 kernel_regularizer=None,
360 kernel_constraint=None,
361 use_bias=True,
362 bias_initializer='zeros',
363 bias_regularizer=None,
364 bias_constraint=None,
365 **kwargs):
366 """Initialize the layer.
367 :param units: Number of new states. If the input shape is (batch_size, node_num, feature_len), then the output
368 shape is (batch_size, node_num, units).
369 :param kernel_initializer: The initializer of the kernel weight matrix.
370 :param kernel_regularizer: The regularizer of the kernel weight matrix.
371 :param kernel_constraint: The constraint of the kernel weight matrix.
372 :param use_bias: Whether to use bias term.
373 :param bias_initializer: The initializer of the bias vector.
374 :param bias_regularizer: The regularizer of the bias vector.
375 :param bias_constraint: The constraint of the bias vector.
376 :param kwargs: Other arguments for parent class.
377 """
378 self.units = units
379 self.kernel_initializer = keras.initializers.get(kernel_initializer)
380 self.kernel_regularizer = keras.regularizers.get(kernel_regularizer)
381 self.kernel_constraint = keras.constraints.get(kernel_constraint)
382 self.use_bias = use_bias
383 self.bias_initializer = keras.initializers.get(bias_initializer)
384 self.bias_regularizer = keras.regularizers.get(bias_regularizer)
385 self.bias_constraint = keras.constraints.get(bias_constraint)
386
387 self.W, self.b = None, None
388 super(GraphConv, self).__init__(**kwargs)
389
390 def get_config(self):
391 config = {
392 'units': self.units,
393 'kernel_initializer': keras.initializers.serialize(self.kernel_initializer),
394 'kernel_regularizer': keras.regularizers.serialize(self.kernel_regularizer),
395 'kernel_constraint': keras.constraints.serialize(self.kernel_constraint),
396 'use_bias': self.use_bias,
397 'bias_initializer': keras.initializers.serialize(self.bias_initializer),
398 'bias_regularizer': keras.regularizers.serialize(self.bias_regularizer),
399 'bias_constraint': keras.constraints.serialize(self.bias_constraint),
400 }
401 base_config = super(GraphConv, self).get_config()
402 return dict(list(base_config.items()) + list(config.items()))
403
404 def build(self, input_shape):
405 feature_dim = int(input_shape[0][-1])
406 self.W = self.add_weight(
407 shape=(feature_dim, self.units),
408 initializer=self.kernel_initializer,
409 regularizer=self.kernel_regularizer,
410 constraint=self.kernel_constraint,
411 name='{}_W'.format(self.name),
412 )
413 if self.use_bias:
414 self.b = self.add_weight(
415 shape=(self.units,),
416 initializer=self.bias_initializer,
417 regularizer=self.bias_regularizer,
418 constraint=self.bias_constraint,
419 name='{}_b'.format(self.name),
420 )
421 super(GraphConv, self).build(input_shape)
422
423 def compute_output_shape(self, input_shape):
424 return input_shape[0][:2] + (self.units,)
425
426 def compute_mask(self, inputs, mask=None):
427 if mask is None:
428 mask = [None]
429 return mask[0]
430
431 def _call(self, features, edges):
432 proj = K.dot(features, self.W)
433 if self.use_bias:
434 proj += self.b
435 if self.step_num > 1:
436 edges = self._get_walked_edges(edges, self.step_num)
437 # aggr = proj/2
438 aggr = tf.math.divide((K.sum(proj, axis=1, keepdims=True) + K.epsilon()), 3)
439 # aggr = K.batch_dot(K.permute_dimensions(edges, (0, 2, 1)), proj) \
440 # / (K.sum(edges, axis=2, keepdims=True) + K.epsilon())
441 return features + aggr
442 # return features
443
444
445def obj_cent(traj_nam):
446 q1, q2, q3, q4 = [(0.4, 0.4), (-0.4, 0.4), (-0.4, -0.4), (0.4, -0.4)]
447 obj_seq = [q3, q1, q2, q4]
448 var_chk = traj_nam[11]
449 if var_chk == "1":
450 obj_c = [q1]
451 obj_c_plot = obj_seq[:-2]
452 elif var_chk == "2":
453 obj_c = [q2]
454 obj_c_plot = obj_seq[:-1]
455 elif var_chk == "3":
456 obj_c = [q3]
457 obj_c_plot = [obj_seq[0]]
458 elif var_chk == "4":
459 obj_c = [q4]
460 obj_c_plot = obj_seq
461 else:
462 obj_c = [(0.0, 0.0)]
463 obj_c_plot = [(0.4, 0.4), (-0.4, 0.4), (-0.4, -0.4), (0.4, -0.4)]
464
465 return [obj_c, obj_c_plot]
466
467
468def computer_fisher(model, imgset, img_ind):
469 with tf.GradientTape(persistent=True) as gt:
470 f_accum = []
471 for i in range(len(model.weights)):
472 f_accum.append(np.zeros(model.weights[i].shape))
473 f_accum = np.array(f_accum)
474 if len(img_ind):
475 img_ind = img_ind
476 else:
477 img_ind = np.random.randint(imgset[0].shape[0], size = 100)
478 # pbar = ProgressBar(maxval=num_sample).start()
479 for j in range(len(img_ind)):
480 # img_index = j
481 # img_index = np.random.randint(imgset[0].shape[0])
482 img_index = img_ind[j]
483 for m in range(len(model.weights)):
484 input_mod = [np.expand_dims(imgset[0][img_index],0), np.expand_dims(imgset[1][img_index],0)]
485 # print(input_mod)
486 model_output = model(input_mod)
487 grads = gt.gradient(tf.math.log(model_output), model.weights)[m]
488 f_accum[m] += np.square(grads[0])
489 if j%100 == 0:
490 print(j)
491 # pbar.update(j+1)
492 # pbar.finish()
493 f_accum /= len(img_ind)
494 del gt
495 return f_accum
496
497
498class ewc_reg(Regularizer):
499 def __init__(self, fisher, prior_weights, Lambda):
500 self.fisher = fisher
501 self.prior_weights = prior_weights
502 self.Lambda = Lambda
503
504 def __call__(self, x):
505 regularization = 0.
506 regularization += self.Lambda * K.sum(self.fisher * K.square(x - self.prior_weights))
507 return regularization
508
509 def get_config(self):
510 return {'Lambda': float(self.Lambda)}
511
512
513def custom_lr_loss(y_true, y_pred):
514 mse = tf.keras.losses.MeanSquaredError()
515 t_loss = mse(y_true, y_pred)
516
517 if y_true == 1:
518 return 1000*t_loss
519 if y_true == 0:
520 return t_loss
521
522def PI(x, y, theta_rad, beta, pi_use, n=100, ):
523 print(pi_use)
524 if pi_use == "no_osc":
525 #Calculating distance from starting point
526 print("------using PI WITHOUT oscillators-----")
527 pos = np.column_stack((x,y))
528 a = pos[0,0] * np.ones(pos[:,0].shape)
529 b = pos[0,1] * np.ones(pos[:,1].shape)
530 origin = np.transpose(np.append([a],[b],axis=0)) #for different x,y
531
532 disp = pos - origin
533
534 # Head direction parameters
535 dth = np.divide(2*np.pi, n)
536 theta_pref = np.arange(0, 2*np.pi, dth)
537 pref_dir = np.transpose([np.cos(theta_pref), np.sin(theta_pref)])
538 print(len(pos))
539 hdi = preprocessing.normalize(np.cos(np.matlib.repmat(theta_pref, len(pos),1) - np.transpose((np.matlib.repmat(theta_rad[0:len(pos)],n, 1)))), norm='l2')
540
541 # HD responses
542 hd_resp = []
543 for i in range(len(disp)):
544 for j in range(len(pref_dir)):
545 z = np.array(disp[i])
546 dj = np.array(pref_dir[j])
547 hd_resp.append(np.dot(z,dj))
548 hd_resp = np.transpose(np.reshape(hd_resp, (len(disp),len(pref_dir))))
549
550 # path integraion
551 pi_layer_beta = []
552 for i in range(len(beta)):
553 pi_layer_temp = np.sin(beta[i] * hd_resp)
554 pi_layer_beta.append((pi_layer_temp))
555 pi_layer_beta = np.asarray(pi_layer_beta)
556 pi_beta = pi_layer_beta[0]
557 for i in range(len(beta) - 1):
558 pi_beta = np.concatenate((pi_beta, pi_layer_beta[i+1]))
559 pi_lay = pi_beta.T
560
561
562 ##### ---------------------------- PI (WITH OSCILLATORS) ----------------------#########
563 if pi_use == "osc":
564 # print("-------- using PI WITH oscillators --------")
565 trj_hd_resp = HD(speed, theta)
566 PI1d = PI(trj_hd_resp, speed)
567 PI1d = np.transpose(PI1d)
568 PI1d = preprocessing.normalize(PI1d, norm='l2', axis=1)
569
570 hd_resp = [iii.T.reshape(100,1) for iii in trj_hd_resp]
571 hd_resp = np.asarray(hd_resp).reshape(len(hd_resp), hd_resp[0].shape[0])
572 hd_resp = preprocessing.normalize(hd_resp, norm='l2', axis=1)
573 num_images = PI1d.shape[0]
574
575 pi_lay = PI1d
576
577 return pi_lay
578
579
580
581def model_arch(data, nodes, activ, lr):
582 print("#########-----------------TRAINING MODEL---------------#########")
583 act = 'relu'
584 input_img = Input(shape = data[0].shape[1:], name="IMAGE")
585 input_pi = Input(shape = data[1].shape[1], name="PI")
586
587 encoder = Conv2D(8, (5, 5), padding='same', activation= act, name="CONV_1")(input_img)
588 encoder = MaxPooling2D(pool_size=(2,2), padding='same', name="MAXPOOL_1")(encoder)
589 encoder = Conv2D(4, (5, 5), padding='same', activation= act,name="CONV_2")(encoder)
590 encoder = MaxPooling2D(pool_size=(2,2), padding='same', name="MAXPOOL_2")(encoder)
591 encoder = Conv2D(2, (5, 5), padding='same', activation= act,name="CONV_3")(encoder)
592 encoder = MaxPooling2D(pool_size=(2,2), padding='same', name="MAXPOOL_3")(encoder)
593 flatencoder=Flatten()(encoder)
594 #flatencoder = GlobalAveragePooling2D()(encoder)
595 dense0 = Dense(nodes, activation = 'sigmoid', name='LEC')(flatencoder)
596 dense_pi1 = Dense(nodes, activation= 'sigmoid', name='MEC')(input_pi)
597 dense_pi1 = layers.Reshape((1,nodes))(dense_pi1)
598 dense0 = layers.Reshape((1,nodes))(dense0)
599 data_layer = layers.concatenate([dense0, dense_pi1], axis=1)
600 print(data_layer.shape)
601 edge_layer = tf.constant(np.matlib.repmat(np.asarray([[1/3,1/3], [1/3,1/3]]), 1, 1).reshape((1,2,2)))
602 conv_layer = GraphConv(units=nodes, step_num=1)([data_layer, edge_layer])
603 conv_layer0 = Flatten()(conv_layer)
604 dense1 = Dense(nodes, activation = activ, name = 'D1')(conv_layer0)
605 dense2 = Dense(nodes, activation = activ, name = 'D2')(dense1)
606 dense3 = Dense(nodes, activation = activ, name = 'D3')(dense2)
607
608 output1 = Dense(1, activation='linear', name='VALUE1')(dense3)
609 # output2 = Dense(1, activation='linear', name='VALUE2')(conv_layer[:,1,:])
610 #
611 regressor_model = Model([input_img, input_pi], output1)
612 #regressor_model = Model([input_img,], [outputx, outputy, outputz])
613 opt = tf.keras.optimizers.Adam(learning_rate= lr)
614 regressor_model.compile(optimizer=opt, loss="mse", )
615 # regressor_model.summary()
616
617 return regressor_model
618
619
620
621def model_arch_catas(data, old_mod_dict, nodes, lam, activ, lr, ):
622 tf.keras.backend.clear_session()
623
624 old_wts = old_mod_dict["weights"]
625 I = old_mod_dict["fisher"]
626
627 print("#########-----------------RETRAINING MODEL---------------#########")
628 act = 'relu'
629 input_img = Input(shape = data[0].shape[1:], name="IMAGE")
630 input_pi = Input(shape = data[1].shape[1], name="PI")
631
632 encoder = Conv2D(8, (5, 5), padding='same', activation= act, name="CONV_1", kernel_regularizer=ewc_reg(I[0], old_wts[0], Lambda = lam),
633 bias_regularizer = ewc_reg(I[1], old_wts[1], Lambda = lam))(input_img)
634 encoder = MaxPooling2D(pool_size=(2,2), padding='same', name="MAXPOOL_1")(encoder)
635 encoder = Conv2D(4, (5, 5), padding='same', activation= act, name="CONV_2", kernel_regularizer=ewc_reg(I[2], old_wts[2], Lambda = lam),
636 bias_regularizer = ewc_reg(I[3], old_wts[3], Lambda = lam))(encoder)
637 encoder = MaxPooling2D(pool_size=(2,2), padding='same', name="MAXPOOL_2")(encoder)
638 encoder = Conv2D(2, (5, 5), padding='same', activation= act, name="CONV_3", kernel_regularizer=ewc_reg(I[4], old_wts[4], Lambda = lam),
639 bias_regularizer = ewc_reg(I[5], old_wts[5], Lambda = lam))(encoder)
640 encoder = MaxPooling2D(pool_size=(2,2), padding='same', name="MAXPOOL_3")(encoder)
641 flatencoder = Flatten()(encoder)
642 #flatencoder = GlobalAveragePooling2D()(encoder)
643 dense0 = Dense(nodes, activation = 'sigmoid', name='LEC', kernel_regularizer=ewc_reg(I[6], old_wts[6], Lambda = lam),
644 bias_regularizer = ewc_reg(I[7], old_wts[7], Lambda = lam))(flatencoder)
645 # dense1 = Dense(50, activation = act, name='LEC2')(dense0)
646 # dense0 = Dropout(0.5)(dense0)
647 dense_pi1 = Dense(nodes, activation= 'sigmoid', name='MEC', kernel_regularizer=ewc_reg(I[8], old_wts[8], Lambda = lam),
648 bias_regularizer = ewc_reg(I[9], old_wts[9], Lambda = lam))(input_pi)
649 # dense_pi2 = Dense(50, activation= act, name='MEC2')(dense_pi1)
650 # concat = layers.concatenate([dense1, dense_pi2])
651 dense_pi1 = layers.Reshape((1,nodes))(dense_pi1)
652 dense0 = layers.Reshape((1,nodes))(dense0)
653 data_layer = layers.concatenate([dense0, dense_pi1], axis=1, name = "Concat")
654 print(data_layer.shape)
655 # edge_layer = layers.Input(shape=(None, None))
656 edge_layer = tf.constant(np.matlib.repmat(np.asarray([[1/3,1/3], [1/3,1/3]]), 1, 1).reshape((1,2,2)))
657 # edge_layer = tf.constant(np.asarray([[0,1], [1,0.]]).reshape((1,2,2)))
658 conv_layer = GraphConv(units=nodes, step_num=1, kernel_regularizer=ewc_reg(I[10], old_wts[10], Lambda = lam),
659 bias_regularizer = ewc_reg(I[11], old_wts[11], Lambda = lam), name = "graph_conv")([data_layer, edge_layer])
660 # conv_layer = Dropout(0.5)(conv_layer)
661 # conv_layer0 = layers.Add()([conv_layer[:,0,:],conv_layer[:,1,:]])
662 conv_layer0 = Flatten()(conv_layer)
663 dense1 = Dense(nodes, activation=activ, name = 'D1', kernel_regularizer=ewc_reg(I[12], old_wts[12], Lambda = lam),
664 bias_regularizer = ewc_reg(I[13], old_wts[13], Lambda = lam))(conv_layer0)
665 dense2 = Dense(nodes, activation=activ, name = 'D2', kernel_regularizer=ewc_reg(I[14], old_wts[14], Lambda = lam),
666 bias_regularizer = ewc_reg(I[-5], old_wts[15], Lambda = lam))(dense1)
667 dense3 = Dense(nodes, activation=activ, name = 'D3', kernel_regularizer=ewc_reg(I[16], old_wts[16], Lambda = lam),
668 bias_regularizer = ewc_reg(I[-3], old_wts[17], Lambda = lam))(dense2)
669 # dense4 = Dense(50, activation=act, name = 'D4')(dense3)
670
671 output1 = Dense(1, activation='linear', name='VALUE1', kernel_regularizer=ewc_reg(I[18], old_wts[19], Lambda = lam),
672 bias_regularizer = ewc_reg(I[19], old_wts[19], Lambda = lam))(dense3)
673 # output2 = Dense(1, activation='linear', name='VALUE2')(conv_layer[:,1,:])
674 #
675 model_r = Model([input_img, input_pi], output1)
676 opt = tf.keras.optimizers.Adam(learning_rate = 0.001)
677 model_r.compile(optimizer=opt, loss = custom_lr_loss, )
678 # model_r.summary()
679
680 return model_r