added DA in-memory generator in data.py

data.py

@@ -191,7 +191,155 @@ def frame_generator(self, batch_size, train_test, data_type):
  y.append(self.get_class_one_hot(sample[1]))
 
  yield np.array(X), np.array(y)
+
+ @threadsafe_generator
+ def in_memory_generator(self, batch_size, train_test, data_type, augmentation=False, verbose=False):
+ """COPIED:
+ Return a generator that we can use to train on. There are
+ a couple different things we can return:
+
+ data_type: 'features', 'images'
+ EDIT: this generator DOES load everything in memory. Other than get_all_sequences_in_memory(),
+ this functions performs some data augmentation (DA) on the fly. Since we want to perform DA on a single
+ sample with multiple frames at once, we cant use per-frame-DA (i.e. if we randomly rotate a frame,
+ we want to rotate every frame within the same sample with the same amount.
+ This custom data loader provides such DA,
+ yet while being faster than applying regular DA in the non-memory generator
+ """
+ # Get the right dataset for the generator.
+ # a separate test and validation set is currently not included
+ # if train_test == 'test':
+ # data = self.split_test()
+ # else:
+ train, val = self.split_train_val()
+ data = train if train_test == 'train' else val
+
+ print("\nCreating %s generator in memory, with %d samples." % (train_test, len(data)))
+
+ X, y = self.get_all_sequences_in_memory(train_test, data_type)
+
+ while 1:
+
+ idx = list(range(X.shape[0]))
+ # random.shuffle(idx)
+
+ while idx:
+
+ batch_idx = idx[:batch_size]
+ idx = idx[batch_size:]
+ X_batch, y_batch = [], []
+
+ # Generate batch_size samples.
+ for i in batch_idx:
+ # Reset to be safe.
+ sample = None
+
+ # get a copy of the original data
+ sample = copy.deepcopy(X[i])
+ label = copy.deepcopy(y[i])
+
+ if augmentation: sample, label = self.augment(sample, label)
+
+ X_batch.append(sample)
+ y_batch.append(label)
+
+ yield np.array(X_batch), np.array(y_batch)
+
+ def augment(self, sample, label):
+ """
+ Performing augmentation on the fly
+ each sample is a numpy array with shape: [# frames per sequence, H, W 3]
+ """
+
+ # sample info
+ H, W, c = sample[0].shape
+ seq_len = self.seq_length
+ assert len(sample) == seq_len, 'Somehow sequence lenght isn\'t correct!'
+
+ # augmentation hyperparams
+ angle = 20 # maximum absolute (left or right) angle for rotation
+ mu = 0 # mean for guassian noise
+ sigma = 0.1 # std for gaussian noise
+ gamma_min = 0.5 # gamma for gamma conversion MAKE RANDOM UP TO 0.5?
+ flip_h_ch = 0.2 # chance of applying horizontal flip
+ flip_v_ch = 0.2 # chance of applying vertical flip
+
+ # augmentation parameters that need to be constant for a single sample
+ rand_rot = np.random.randint(-angle, angle)
+ M = cv2.getRotationMatrix2D((W / 2, H / 2), rand_rot, 1)
+
+ flip_hor = np.random.uniform() < flip_h_ch
+ flip_ver = np.random.uniform() < flip_v_ch
+
+ # loop over each frame only once for efficiency (i.e. not in every of the augmentation functions
+ for i, frame in enumerate(sample):
+
+ # apply augmentations
+ frame = self.rotate(frame, M, H, W)
+ frame = self.gaussian_noise(frame, H, W, c, mu=mu, sigma=sigma)
+ frame = self.gamma_conversion(frame, gamma_min=gamma_min)
+ if flip_hor: frame = self.flip_horizontal(frame)
+ if flip_ver: frame = self.flip_vertical(frame)
+
+ # update the frame in the sample
+ sample[i] = frame
+
+ # flip the label in case the images gets horizontally flipped
+ if flip_ver: label = np.ones_like(label, dtype=label.dtype) - label
+
+ return sample, label
+
+ @staticmethod
+ def rotate(frame, M, H, W):
+ '''Applies same/consistent rotation to each frame in a sample'''
+
+ return cv2.warpAffine(frame, M, (W, H), borderMode=cv2.BORDER_REPLICATE)
+
+ @staticmethod
+ def gaussian_noise(frame, H, W, c, mu=0, sigma=0.1):
+ '''Applies different random/guassian noise separately to each frame of sample'''
+
+ # get noise params, okay to be different for each frame in a single sample
+ gauss = np.random.normal(mu, sigma, (H, W, c))
+ # apply noise
+ frame = np.clip(frame + gauss, 0, 1)
+
+ return frame
+
+ @staticmethod
+ def flip_vertical(frame):
+ '''Applies vertical flipping similarly/consistently over all frame of sample'''
+
+ frame = frame[:, ::-1, :]
+
+ return frame
+
+ @staticmethod
+ def flip_horizontal(frame):
+ '''Applies horizontal flipping similarly/consistently over all frame of sample'''
+
+ frame = frame[::-1, :, :]
+
+ return frame
+
+ @staticmethod
+ def translate(sample, seq_len, H, W, c, mu=0, sigma=0.1):
+ # NOT IMPLEMENTED
+ return sample
+
+ @staticmethod
+ def gamma_conversion(frame, gamma_min=0.75):
+ '''Applies gamma conversion separately to each frame of sample'''
+
+ # get conversion params, might be okay to change for each frame?
+ gamma = np.random.uniform(gamma_min, 1.)
+ gamma = 1 / gamma if np.random.uniform() < 0.5 else gamma
+
+ # apply conversion
+ frame = frame ** (1.0 / gamma)
 
+ return frame
+
  def build_image_sequence(self, frames):
  """Given a set of frames (filenames), build our sequence."""
  return [process_image(x, self.image_shape) for x in frames]