Guided Filter from SkyOptimization https://github.com/google/sky-optimization
import numpy as np import cv2 class GuidedFilter: """ Guided Filter from SkyOptimization https://github.com/google/sky-optimization """ def __init__(self, compute_confidence=True): self.compute_confidence = compute_confidence def inference(self, reference, source, kernel_size, confidence=None, eps_luma=1e-2, eps_chroma=1e-2, clip_output=True): assert reference.shape[2] == 3 if np.any(np.array(source.shape) < np.array(reference.shape[:2])): source = self.resize(source, reference.shape[:2]) if confidence is None: confidence = self.probability_to_confidence(source) assert confidence.shape == source.shape reference_small = self.weighted_downsample(reference, confidence, kernel_size) small_shape = reference_small.shape[:2] source_small = self.weighted_downsample(source, confidence, target_size=small_shape) outer_reference = self.outer_product_images(reference, reference) outer_reference = self.weighted_downsample(outer_reference, confidence, target_size=small_shape) covar = outer_reference - self.outer_product_images(reference_small, reference_small) var = self.weighted_downsample( reference * source[..., np.newaxis], confidence, target_size=small_shape ) residual_small = var - reference_small * source_small[..., np.newaxis] covar[..., 0] += eps_luma ** 2 covar[..., [3, 5]] += eps_chroma ** 2 affine = self.solve_image_ldl3(covar, residual_small) residual = source_small - (affine * reference_small).sum(axis=2) affine = self.smooth_upsample(affine, reference.shape[:2]) residual = self.smooth_upsample(residual, reference.shape[:2]) output = (affine * reference).sum(axis=2) + residual if clip_output: output = output.clip(0, 1) return output def bias(self, x, b=0.8): denom = ((1 / b) - 2) * (1 - x) + 1 return x / denom def probability_to_confidence(self, probabilty, low_thresh=0.3, high_thresh=0.5): eps = 0.01 low = probabilty < low_thresh high = probabilty > high_thresh confidence_low = self.bias((low_thresh - probabilty[low]) / low_thresh) confidence_high = self.bias((probabilty[high] - high_thresh) / (1 - high_thresh)) confidence = np.zeros_like(probabilty) confidence[low] = confidence_low confidence[high] = confidence_high confidence = np.maximum(eps, confidence) return confidence def downsample2_antialiased(self, X): kernel = np.array([1, 3, 3, 1]) / 8 dst = cv2.sepFilter2D(X, -1, kernel, kernel, anchor=(1, 1), borderType=cv2.BORDER_REPLICATE) return dst[::2, ::2] def resize_helper(self, X, shape): X = X.squeeze() while np.all(np.array(X.shape[:2]) >= np.array(shape) * 2): X = self.downsample2_antialiased(X) return cv2.resize(X, dsize=tuple(shape[1::-1]), interpolation=cv2.INTER_LINEAR) def resize(self, X, shape): if X.ndim == 2 or X.shape[2] <= 4: return self.resize_helper(X, shape) # opencv doesn't work on more than 4 channels X1 = self.resize_helper(X[..., :3], shape) X2 = self.resize_helper(X[..., 3:], shape) return np.concatenate([X1, X2], axis=2) def outer_product_images(self, X, Y): assert X.shape[-1] == 3 and Y.shape[-1] == 3 X_flat = X[..., :, np.newaxis] Y_flat = Y[..., np.newaxis, :] outer = np.matmul(X_flat, Y_flat) ind = np.triu_indices(3) outer = outer[..., ind[0], ind[1]] return outer.reshape(X.shape[:-1] + (6,)) def smooth_upsample(self, X, size, num_steps=None): if num_steps is None: log4ratio = np.max(0.5 * np.log2(np.array(size) / X.shape[:2])) num_steps = np.maximum(1, log4ratio.round().astype(np.int)) ratio = np.array(size) / X.shape[:2] ratio_per_step = np.array(X.shape[:2]) * ratio / num_steps for step in np.arange(1, num_steps + 1): target_shape_for_step = np.round(step * ratio_per_step).astype(np.int) X = self.resize(X, target_shape_for_step) return X def solve_image_ldl3(self, A, b): A11, A12, A13, A22, A23, A33 = np.split(A, A.shape[-1], axis=-1) b1, b2, b3 = np.split(b, b.shape[-1], axis=-1) d1 = A11 L_12 = A12 / d1 d2 = A22 - L_12 * A12 L_13 = A13 / d1 L_23 = (A23 - L_13 * A12) / d2 d3 = A33 - L_13 * A13 - L_23 * L_23 * d2 y1 = b1 y2 = b2 - L_12 * y1 y3 = b3 - L_13 * y1 - L_23 * y2 x3 = y3 / d3 x2 = y2 / d2 - L_23 * x3 x1 = y1 / d1 - L_12 * x2 - L_13 * x3 return np.stack([x1, x2, x3], axis=-1).squeeze() def weighted_downsample(self, X, confidence, scale=None, target_size=None): if target_size is None: target_size = (np.array(X.shape[:2]) / scale).round().astype(np.int) if X.shape[1] > confidence.shape[1]: X = self.resize(X, confidence.shape) if X.ndim == 3: confidence = confidence[..., np.newaxis] numerator = self.resize(X * confidence, target_size) denom = self.resize(confidence, target_size) if X.ndim == 3: denom = denom[..., np.newaxis] return numerator / denom
Simple Version
import numpy as np import cv2 class SimplifiedGuidedFilter: """ Guided Filter from SkyOptimization https://github.com/google/sky-optimization Simplify guided filter: 1. change smooth upsample to bilinear upsample 2. not add eps to covariance """ def __init__(self, compute_confidence=True): self.compute_confidence = compute_confidence def inference(self, reference, source, kernel_size, confidence=None, eps_luma=1e-2, eps_chroma=1e-2, clip_output=True): assert reference.shape[2] == 3 if np.any(np.array(source.shape) < np.array(reference.shape[:2])): source = self.resize(source, reference.shape[:2]) if confidence is None: confidence = self.probability_to_confidence(source) assert confidence.shape == source.shape # 1. weighted downsample reference and source reference_small = self.weighted_downsample(reference, confidence, kernel_size) # RGB image small_shape = reference_small.shape[:2] source_small = self.weighted_downsample(source, confidence, target_size=small_shape) # mask # 2. compute covariance and variance, weighted downsample outer_reference = self.outer_product_images(reference, reference) outer_reference = self.weighted_downsample(outer_reference, confidence, target_size=small_shape) covar = outer_reference - self.outer_product_images(reference_small, reference_small) var = self.weighted_downsample( reference * source[..., np.newaxis], confidence, target_size=small_shape ) residual_small = var - reference_small * source_small[..., np.newaxis] covar[..., 0] += eps_luma ** 2 covar[..., [3, 5]] += eps_chroma ** 2 # 3. LDL decomposition affine = self.solve_image_ldl3(covar, residual_small) residual = source_small - (affine * reference_small).sum(axis=2) # 4. upsample affine = self.bilinear_upsample(affine, (reference.shape[1], reference.shape[0])) residual = self.bilinear_upsample(residual, (reference.shape[1], reference.shape[0])) output = (affine * reference).sum(axis=2) + residual if clip_output: output = output.clip(0, 1) return output def bias(self, x, b=0.8): denom = ((1 / b) - 2) * (1 - x) + 1 return x / denom def probability_to_confidence(self, probabilty, low_thresh=0.3, high_thresh=0.5): eps = 0.01 low = probabilty < low_thresh high = probabilty > high_thresh confidence_low = self.bias((low_thresh - probabilty[low]) / low_thresh) confidence_high = self.bias((probabilty[high] - high_thresh) / (1 - high_thresh)) confidence = np.zeros_like(probabilty) confidence[low] = confidence_low confidence[high] = confidence_high confidence = np.maximum(eps, confidence) return confidence def downsample2_antialiased(self, X): kernel = np.array([1, 3, 3, 1]) / 8 dst = cv2.sepFilter2D(X, -1, kernel, kernel, anchor=(1, 1), borderType=cv2.BORDER_REPLICATE) return dst[::2, ::2] def resize_helper(self, X, shape): X = X.squeeze() while np.all(np.array(X.shape[:2]) >= np.array(shape) * 2): X = self.downsample2_antialiased(X) return cv2.resize(X, dsize=tuple(shape[1::-1]), interpolation=cv2.INTER_LINEAR) def resize(self, X, shape): if X.ndim == 2 or X.shape[2] <= 4: return self.resize_helper(X, shape) # opencv doesn't work on more than 4 channels X1 = self.resize_helper(X[..., :3], shape) X2 = self.resize_helper(X[..., 3:], shape) return np.concatenate([X1, X2], axis=2) def outer_product_images(self, X, Y): assert X.shape[-1] == 3 and Y.shape[-1] == 3 X_flat = X[..., :, np.newaxis] Y_flat = Y[..., np.newaxis, :] outer = np.matmul(X_flat, Y_flat) ind = np.triu_indices(3) outer = outer[..., ind[0], ind[1]] return outer.reshape(X.shape[:-1] + (6,)) def bilinear_upsample(self, X, size): X = cv2.resize(X, size, interpolation=cv2.INTER_LINEAR) return X def solve_image_ldl3(self, A, b): A11, A12, A13, A22, A23, A33 = np.split(A, A.shape[-1], axis=-1) b1, b2, b3 = np.split(b, b.shape[-1], axis=-1) d1 = A11 L_12 = A12 / d1 d2 = A22 - L_12 * A12 L_13 = A13 / d1 L_23 = (A23 - L_13 * A12) / d2 d3 = A33 - L_13 * A13 - L_23 * L_23 * d2 y1 = b1 y2 = b2 - L_12 * y1 y3 = b3 - L_13 * y1 - L_23 * y2 x3 = y3 / d3 x2 = y2 / d2 - L_23 * x3 x1 = y1 / d1 - L_12 * x2 - L_13 * x3 return np.stack([x1, x2, x3], axis=-1).squeeze() def weighted_downsample(self, X, confidence, scale=None, target_size=None): if target_size is None: target_size = (np.array(X.shape[:2]) / scale).round().astype(np.int) if X.shape[1] > confidence.shape[1]: X = self.resize(X, confidence.shape) if X.ndim == 3: confidence = confidence[..., np.newaxis] numerator = self.resize(X * confidence, target_size) denom = self.resize(confidence, target_size) if X.ndim == 3: denom = denom[..., np.newaxis] return numerator / denom