[CCV] Implemented first version of saliency system

Signed-off-by: Jim Martens <github@2martens.de>

ccv/saliency/CMakeLists.txt

@@ -0,0 +1,9 @@
+cmake_minimum_required(VERSION 3.5)
+project(saliency)
+
+set(CMAKE_CXX_STANDARD 11)
+
+find_package( OpenCV REQUIRED )
+
+add_executable(saliency fusion.cpp gauss_pyramid.cpp lab_pyramid.cpp main.cpp)
+target_link_libraries(saliency ${OpenCV_LIBS})

ccv/saliency/fusion.cpp

@@ -0,0 +1,86 @@
+#include <opencv2/opencv.hpp>
+#include "includes/fusion.h"
+
+/**
+ * Returns the mean fusion of two feature maps.
+ *
+ * @param f_on_off feature map on off
+ * @param f_off_on feature map off on
+ * @return conspicuity map
+ */
+cv::Mat mean_fusion(cv::Mat f_on_off, cv::Mat f_off_on) {
+  cv::Mat C_l = 0.5 * (f_on_off + f_off_on);
+  double max_on_off;
+  double max_off_on;
+  cv::minMaxLoc(f_on_off, nullptr, &max_on_off);
+  cv::minMaxLoc(f_off_on, nullptr, &max_off_on);
+  double max = max_on_off >= max_off_on ? max_on_off : max_off_on;
+  cv::normalize(C_l, C_l, 0, max, cv::NORM_MINMAX, -1);
+
+  return C_l.clone();
+}
+
+/**
+ * Returns the max fusion of two feature maps.
+ *
+ * @param f_on_off feature map on off
+ * @param f_off_on feature map off on
+ * @return conspicuity map
+ */
+cv::Mat max_fusion(cv::Mat f_on_off, cv::Mat f_off_on) {
+  cv::Mat C_l = cv::max(f_on_off, f_off_on);
+  double max_on_off;
+  double max_off_on;
+  cv::minMaxLoc(f_on_off, nullptr, &max_on_off);
+  cv::minMaxLoc(f_off_on, nullptr, &max_off_on);
+  double max = max_on_off >= max_off_on ? max_on_off : max_off_on;
+  cv::normalize(C_l, C_l, 0, max, cv::NORM_MINMAX, -1);
+
+  return C_l.clone();
+}
+
+/**
+ * Computes the saliency map using mean fusion.
+ *
+ * @param C_l conspicuity map for L channel
+ * @param C_a conspicuity map for A channel
+ * @param C_b conspicuity map for B channel
+ * @return saliency map
+ */
+cv::Mat mean_fusion_saliency(cv::Mat C_l, cv::Mat C_a, cv::Mat C_b) {
+  cv::Mat S = (1/3.0) * (C_l + C_a + C_b);
+  double max_C_l;
+  double max_C_a;
+  double max_C_b;
+  cv::minMaxLoc(C_l, nullptr, &max_C_l);
+  cv::minMaxLoc(C_a, nullptr, &max_C_a);
+  cv::minMaxLoc(C_b, nullptr, &max_C_b);
+  double max = max_C_l >= max_C_a ? (max_C_l >= max_C_b ? max_C_l : max_C_b) : (max_C_a >= max_C_b ? max_C_a : max_C_b);
+  cv::normalize(S, S, 0, max, cv::NORM_MINMAX, -1);
+
+  return S;
+}
+
+/**
+ * Computes the saliency map using max fusion.
+ *
+ * @param C_l conspicuity map for L channel
+ * @param C_a conspicuity map for A channel
+ * @param C_b conspicuity map for B channel
+ * @return saliency map
+ */
+cv::Mat max_fusion_saliency(cv::Mat C_l, cv::Mat C_a, cv::Mat C_b) {
+  cv::Mat S = cv::max(C_l, C_a);
+  S = cv::max(S, C_b);
+
+  double max_C_l;
+  double max_C_a;
+  double max_C_b;
+  cv::minMaxLoc(C_l, nullptr, &max_C_l);
+  cv::minMaxLoc(C_a, nullptr, &max_C_a);
+  cv::minMaxLoc(C_b, nullptr, &max_C_b);
+  double max = max_C_l >= max_C_a ? (max_C_l >= max_C_b ? max_C_l : max_C_b) : (max_C_a >= max_C_b ? max_C_a : max_C_b);
+  cv::normalize(S, S, 0, max, cv::NORM_MINMAX, -1);
+
+  return S;
+}

ccv/saliency/gauss_pyramid.cpp

@@ -0,0 +1,35 @@
+#include "includes/gauss_pyramid.h"
+
+gauss_pyramid::gauss_pyramid() {}
+
+gauss_pyramid::gauss_pyramid(cv::Mat img, float sigma, int number_of_layers)
+{
+  cv::Mat blurredImage;
+  cv::Mat resizedImage = img.clone();
+  for (int i = 0; i < number_of_layers; i++)
+  {
+    cv::GaussianBlur(resizedImage, blurredImage, cv::Size(0, 0), sigma, sigma, cv::BORDER_REPLICATE);
+    _layers.push_back(blurredImage.clone());
+    cv::resize(blurredImage, resizedImage, cv::Size(), 0.5, 0.5, cv::INTER_NEAREST);
+  }
+}
+
+cv::Mat gauss_pyramid::get(int layer) const
+{
+  return _layers.at((unsigned long) layer);
+}
+
+cv::Mat gauss_pyramid::get(int layer)
+{
+  return _layers.at((unsigned long) layer);
+}
+
+unsigned long gauss_pyramid::get_number_of_layers() const
+{
+  return _layers.size();
+}
+
+unsigned long gauss_pyramid::get_number_of_layers()
+{
+  return _layers.size();
+}

ccv/saliency/includes/fusion.h

@@ -0,0 +1,9 @@
+#ifndef SHEET6_FUSION_H
+#define SHEET6_FUSION_H
+
+cv::Mat mean_fusion(cv::Mat f_on_off, cv::Mat f_off_on);
+cv::Mat max_fusion(cv::Mat f_on_off, cv::Mat f_off_on);
+cv::Mat mean_fusion_saliency(cv::Mat C_l, cv::Mat C_a, cv::Mat C_b);
+cv::Mat max_fusion_saliency(cv::Mat C_l, cv::Mat C_a, cv::Mat C_b);
+
+#endif //SHEET6_FUSION_H

ccv/saliency/includes/gauss_pyramid.h

@@ -0,0 +1,20 @@
+#ifndef SHEET3_GAUSS_PYRAMID_H
+#define SHEET3_GAUSS_PYRAMID_H
+
+#include <opencv2/opencv.hpp>
+
+class gauss_pyramid
+{
+private:
+    std::vector<cv::Mat> _layers;
+public:
+    gauss_pyramid();
+    gauss_pyramid(cv::Mat img, float sigma, int number_of_layers);
+    cv::Mat get(int layer) const;
+    cv::Mat get(int layer);
+    unsigned long get_number_of_layers() const;
+    unsigned long get_number_of_layers();
+};
+
+
+#endif //SHEET3_GAUSS_PYRAMID_H

ccv/saliency/includes/lab_pyramid.h

@@ -0,0 +1,118 @@
+#ifndef SHEET3_LAB_PYRAMID_H
+#define SHEET3_LAB_PYRAMID_H
+
+#include <opencv2/opencv.hpp>
+#include "gauss_pyramid.h"
+
+class lab_pyramid {
+private:
+    cv::Mat _inputImage_lab;
+    cv::Mat _inputImage_float;
+    cv::Mat _imageChannels[3];
+    gauss_pyramid _pyramids[3];
+    // contrast maps
+    static std::vector<cv::Mat> _cs_contrast_l;
+    static std::vector<cv::Mat> _sc_contrast_l;
+    static std::vector<cv::Mat> _cs_contrast_a;
+    static std::vector<cv::Mat> _sc_contrast_a;
+    static std::vector<cv::Mat> _cs_contrast_b;
+    static std::vector<cv::Mat> _sc_contrast_b;
+    // feature maps
+    static cv::Mat _cs_F_l;
+    static cv::Mat _sc_F_l;
+    static cv::Mat _cs_F_a;
+    static cv::Mat _cs_F_b;
+    static cv::Mat _sc_F_a;
+    static cv::Mat _sc_F_b;
+    // conspicuity maps
+    static cv::Mat _C_l;
+    static cv::Mat _C_a;
+    static cv::Mat _C_b;
+    // number of layers
+    static int _number_of_layers;
+public:
+    const static int COLOR_L = 0
+    const static int COLOR_A = 1;
+    const static int COLOR_B = 2;
+
+    /**
+     * Initializes a LAB pyramid.
+     *
+     * @param image_filename the filename of the image that should be used
+     */
+    lab_pyramid(cv::String image_filename);
+
+    /**
+     * Initializes a LAB pyramid.
+     *
+     * @param image the image that should be used
+     */
+    lab_pyramid(cv::Mat image);
+
+    /**
+     * Creates the gaussian pyramids for all channels with the given number of layers each.
+     *
+     * @param sigma the sigma for the gaussian pyramids
+     * @param number_of_layers number of layers for gaussian pyramid
+     */
+    void create_pyramids(float sigma, int number_of_layers);
+
+    /**
+     * Before this method can be called, pyramids have to be created via create_pyramids.
+     *
+     * @param channel the channel you want to get (COLOR_L, COLOR_A or COLOR_B)
+     * @return the gaussian_pyramid for the given channel
+     */
+    gauss_pyramid get_pyramid(int channel);
+
+    /**
+     * Computes the center-surround and surround-center contrasts and stores them for later use.
+     *
+     * @param center the center pyramid
+     * @param surround the surround pyramid
+     * @param number_of_layers the number of layers used to create the two pyramids
+     */
+    void static compute_dog(lab_pyramid center, lab_pyramid surround, int number_of_layers);
+
+    /**
+     * Visualizes the center-surround and surround-center contrasts. They have to be computed first.
+     */
+    void static visualize_dog();
+
+    /**
+     * Takes the scale images, adds them up and returns the result.
+     *
+     * @param scale_images the scale images
+     * @return the sum of the scale images
+     */
+    cv::Mat static across_scale_addition(const std::vector<cv::Mat> &scale_images);
+
+    /**
+     * Computes the feature maps.
+     * Has to be called after compute_dog.
+     */
+    void static compute_feature_maps();
+
+    /**
+     * Computes the conspicuity maps.
+     * Has to be called after compute_feature_maps.
+     */
+    void static compute_conspicuity_maps();
+
+    /**
+     * Before this method can be called, the conspicuity maps must be computed via compute_conspicuity_maps.
+     *
+     * @param channel the channel you want to get (COLOR_L, COLOR_A, COLOR_B)
+     * @return the conspicuity map for the given channel
+     */
+    cv::Mat static get_conspicuity_map(int channel);
+
+    /**
+     * Visualizes the feature maps.
+     * Has to be called after compute_feature_maps.
+     */
+    void static visualize_feature_maps();
+};
+
+
+#endif //SHEET3_LAB_PYRAMID_H

ccv/saliency/lab_pyramid.cpp

@@ -0,0 +1,185 @@
+#include "includes/lab_pyramid.h"
+#include "includes/fusion.h"
+
+// number of layers
+int lab_pyramid::_number_of_layers = 0;
+// contrast maps
+std::vector<cv::Mat> lab_pyramid::_cs_contrast_l = std::vector<cv::Mat>();
+std::vector<cv::Mat> lab_pyramid::_sc_contrast_l = std::vector<cv::Mat>();
+std::vector<cv::Mat> lab_pyramid::_cs_contrast_a = std::vector<cv::Mat>();
+std::vector<cv::Mat> lab_pyramid::_sc_contrast_a = std::vector<cv::Mat>();
+std::vector<cv::Mat> lab_pyramid::_cs_contrast_b = std::vector<cv::Mat>();
+std::vector<cv::Mat> lab_pyramid::_sc_contrast_b = std::vector<cv::Mat>();
+// feature maps
+cv::Mat lab_pyramid::_cs_F_l;
+cv::Mat lab_pyramid::_sc_F_l;
+cv::Mat lab_pyramid::_cs_F_a;
+cv::Mat lab_pyramid::_sc_F_a;
+cv::Mat lab_pyramid::_cs_F_b;
+cv::Mat lab_pyramid::_sc_F_b;
+// conspicuity maps
+cv::Mat lab_pyramid::_C_l;
+cv::Mat lab_pyramid::_C_a;
+cv::Mat lab_pyramid::_C_b;
+
+lab_pyramid::lab_pyramid(cv::String image_filename) {
+  cv::Mat image_rgb = cv::imread(image_filename, cv::IMREAD_COLOR);
+  cv::cvtColor(image_rgb, _inputImage_lab, cv::COLOR_BGR2Lab);
+  cv::split(_inputImage_lab ,_imageChannels);
+};
+
+lab_pyramid::lab_pyramid(cv::Mat image) {
+  cv::cvtColor(image, _inputImage_lab, cv::COLOR_BGR2Lab);
+  _inputImage_lab.convertTo(_inputImage_float, CV_32F);
+  cv::split(_inputImage_float, _imageChannels);
+}
+
+void lab_pyramid::create_pyramids(float sigma, int number_of_layers)
+{
+  _pyramids[COLOR_L] = gauss_pyramid(_imageChannels[COLOR_L], sigma, number_of_layers);
+  _pyramids[COLOR_A] = gauss_pyramid(_imageChannels[COLOR_A], sigma, number_of_layers);
+  _pyramids[COLOR_B] = gauss_pyramid(_imageChannels[COLOR_B], sigma, number_of_layers);
+}
+
+gauss_pyramid lab_pyramid::get_pyramid(int channel)
+{
+  switch (channel)
+  {
+    case COLOR_L:
+      return _pyramids[COLOR_L];
+    case COLOR_A:
+      return _pyramids[COLOR_A];
+    case COLOR_B:
+      return _pyramids[COLOR_B];
+    default:
+      throw std::invalid_argument( "received invalid channel value, use COLOR_L, COLOR_A or COLOR_B" );
+  }
+}
+
+void lab_pyramid::compute_dog(lab_pyramid center, lab_pyramid surround, int number_of_layers) {
+  _number_of_layers = number_of_layers;
+
+  // L channel
+  gauss_pyramid center_l = center.get_pyramid(COLOR_L);
+  gauss_pyramid surround_l = surround.get_pyramid(COLOR_L);
+
+  // A channel
+  gauss_pyramid center_a = center.get_pyramid(COLOR_A);
+  gauss_pyramid surround_a = surround.get_pyramid(COLOR_A);
+
+  // A channel
+  gauss_pyramid center_b = center.get_pyramid(COLOR_B);
+  gauss_pyramid surround_b = surround.get_pyramid(COLOR_B);
+
+  for (int layer = 0; layer < number_of_layers; layer++) {
+    // L channel
+    cv::Mat center_layer_mat_L = center_l.get(layer);
+    cv::Mat surround_layer_mat_L = surround_l.get(layer);
+    cv::Mat dog_raw_cs_L = center_layer_mat_L - surround_layer_mat_L;
+    cv::Mat dog_final_cs_L;
+    cv::Mat dog_final_sc_L;
+    cv::threshold(dog_raw_cs_L, dog_final_cs_L, 0, 1, cv::THRESH_TOZERO);
+    _cs_contrast_l.push_back(dog_final_cs_L.clone());
+    cv::Mat dog_raw_sc_L = surround_layer_mat_L - center_layer_mat_L;
+    cv::threshold(dog_raw_sc_L, dog_final_sc_L, 0, 1, cv::THRESH_TOZERO);
+    _sc_contrast_l.push_back(dog_final_sc_L.clone());
+
+    // A channel
+    cv::Mat center_layer_mat_a = center_a.get(layer);
+    cv::Mat surround_layer_mat_a = surround_a.get(layer);
+    cv::Mat dog_raw_cs_a = center_layer_mat_a - surround_layer_mat_a;
+    cv::Mat dog_final_cs_a;
+    cv::Mat dog_final_sc_a;
+    cv::threshold(dog_raw_cs_a, dog_final_cs_a, 0, 1, cv::THRESH_TOZERO);
+    _cs_contrast_a.push_back(dog_final_cs_a.clone());
+    cv::Mat dog_raw_sc_a = surround_layer_mat_a - center_layer_mat_a;
+    cv::threshold(dog_raw_sc_a, dog_final_sc_a, 0, 1, cv::THRESH_TOZERO);
+    _sc_contrast_a.push_back(dog_final_sc_a.clone());
+
+    // B channel
+    cv::Mat center_layer_mat_b = center_b.get(layer);
+    cv::Mat surround_layer_mat_b = surround_b.get(layer);
+    cv::Mat dog_raw_cs_b = center_layer_mat_b - surround_layer_mat_b;
+    cv::Mat dog_final_cs_b;
+    cv::Mat dog_final_sc_b;
+    cv::threshold(dog_raw_cs_b, dog_final_cs_b, 0, 1, cv::THRESH_TOZERO);
+    _cs_contrast_b.push_back(dog_final_cs_b.clone());
+    cv::Mat dog_raw_sc_b = surround_layer_mat_b - center_layer_mat_b;
+    cv::threshold(dog_raw_sc_b, dog_final_sc_b, 0, 1, cv::THRESH_TOZERO);
+    _sc_contrast_b.push_back(dog_final_sc_b.clone());
+  }
+}
+
+void lab_pyramid::compute_feature_maps() {
+  _cs_F_l = across_scale_addition(_cs_contrast_l);
+  _sc_F_l = across_scale_addition(_sc_contrast_l);
+  _cs_F_a = across_scale_addition(_cs_contrast_a);
+  _sc_F_a = across_scale_addition(_sc_contrast_a);
+  _cs_F_b = across_scale_addition(_cs_contrast_b);
+  _sc_F_b = across_scale_addition(_sc_contrast_b);
+}
+
+void lab_pyramid::compute_conspicuity_maps() {
+  _C_l = max_fusion(_cs_F_l, _sc_F_l);
+  _C_a = max_fusion(_cs_F_a, _sc_F_a);
+  _C_b = max_fusion(_cs_F_b, _sc_F_b);
+}
+
+cv::Mat lab_pyramid::get_conspicuity_map(int channel) {
+  switch (channel)
+  {
+    case COLOR_L:
+      return _C_l;
+    case COLOR_A:
+      return _C_a;
+    case COLOR_B:
+      return _C_b;
+    default:
+      throw std::invalid_argument( "received invalid channel value, use COLOR_L, COLOR_A or COLOR_B" );
+  }
+}
+
+cv::Mat lab_pyramid::across_scale_addition(const std::vector<cv::Mat> &scale_images) {
+  cv::Mat result = scale_images.front();
+  cv::Size original_size = scale_images.front().size();
+  for (unsigned long i = 1; i < scale_images.size(); i++) {
+    cv::Mat resized_image;
+    cv::resize(scale_images.at(i), resized_image, original_size, 0, 0, cv::INTER_CUBIC);
+    result += resized_image;
+  }
+  return result;
+}
+
+void lab_pyramid::visualize_dog() {
+  for (unsigned long layer = 0; layer < _number_of_layers; layer++) {
+    cv::namedWindow("CS L");
+    cv::imshow("CS L", _cs_contrast_l.at(layer));
+    cv::namedWindow("SC L");
+    cv::imshow("SC L", _sc_contrast_l.at(layer));
+    cv::namedWindow("CS A");
+    cv::imshow("CS A", _cs_contrast_a.at(layer));
+    cv::namedWindow("SC A");
+    cv::imshow("SC A", _cs_contrast_a.at(layer));
+    cv::namedWindow("CS B");
+    cv::imshow("CS B", _cs_contrast_b.at(layer));
+    cv::namedWindow("SC B");
+    cv::imshow("SC B", _cs_contrast_b.at(layer));
+    cv::waitKey(0);
+  }
+}
+
+void lab_pyramid::visualize_feature_maps() {
+  cv::namedWindow("CS F L");
+  cv::imshow("CS F L", _cs_F_l);
+  cv::namedWindow("CS F L");
+  cv::imshow("SC F L", _sc_F_l);
+  cv::namedWindow("CS F A");
+  cv::imshow("CS F A", _cs_F_a);
+  cv::namedWindow("SC F A");
+  cv::imshow("SC F A", _sc_F_a);
+  cv::namedWindow("CS F B");
+  cv::imshow("CS F B", _cs_F_b);
+  cv::namedWindow("CS F B");
+  cv::imshow("SC F B", _sc_F_b);
+  cv::waitKey(0);
+}

ccv/saliency/main.cpp

@@ -0,0 +1,54 @@
+#include <cstdio>
+#include <opencv2/opencv.hpp>
+
+#include "includes/lab_pyramid.h"
+#include "includes/fusion.h"
+
+/**
+ * Entry point of program
+ * @param argc number of arguments
+ * @param argv CLI arguments, 0: name of program, 1: input file name, 2: output file name
+ * @return status code (0: everything OK, -1: not right amount of arguments)
+ */
+int main(int argc, char** argv) {
+  if (argc != 3) {
+    printf("usage: <input file name> <output file name>\n");
+    return -1;
+  }
+
+  // read image
+  cv::Mat image = cv::imread(argv[1], cv::ImreadModes::IMREAD_COLOR);
+
+  // tweakable factors
+  int layers = 4;
+  float sigma_center = 5;
+  float sigma_surround = 9;
+
+  // create LAB pyramids
+  lab_pyramid lab_pyr_center = lab_pyramid(image);
+  lab_pyr_center.create_pyramids(sigma_center, layers);
+  lab_pyramid lab_pyr_surround = lab_pyramid(image);
+  lab_pyr_surround.create_pyramids(sigma_surround, layers);
+
+  // create contrast maps
+  lab_pyramid::compute_dog(lab_pyr_center, lab_pyr_surround, layers);
+  // create feature maps
+  lab_pyramid::compute_feature_maps();
+  // create conspicuity maps
+  lab_pyramid::compute_conspicuity_maps();
+  // get conspicuity maps
+  cv::Mat C_l = lab_pyramid::get_conspicuity_map(lab_pyramid::COLOR_L);
+  cv::Mat C_a = lab_pyramid::get_conspicuity_map(lab_pyramid::COLOR_A);
+  cv::Mat C_b = lab_pyramid::get_conspicuity_map(lab_pyramid::COLOR_B);
+  // get saliency map
+  cv::Mat saliency = max_fusion_saliency(C_l, C_a, C_b);
+
+  // convert saliency map to correct output format
+  cv::Mat output_image;
+  saliency.convertTo(output_image, CV_8UC1);
+
+  // write output
+  cv::imwrite(argv[2], output_image);
+
+  return 0;
+}