diff --git a/ccv/saliency/CMakeLists.txt b/ccv/saliency/CMakeLists.txt new file mode 100644 index 0000000..00a32f7 --- /dev/null +++ b/ccv/saliency/CMakeLists.txt @@ -0,0 +1,9 @@ +cmake_minimum_required(VERSION 3.5) +project(saliency) + +set(CMAKE_CXX_STANDARD 11) + +find_package( OpenCV REQUIRED ) + +add_executable(saliency fusion.cpp gauss_pyramid.cpp lab_pyramid.cpp main.cpp) +target_link_libraries(saliency ${OpenCV_LIBS}) diff --git a/ccv/saliency/fusion.cpp b/ccv/saliency/fusion.cpp new file mode 100644 index 0000000..0864fb7 --- /dev/null +++ b/ccv/saliency/fusion.cpp @@ -0,0 +1,86 @@ +#include +#include "includes/fusion.h" + +/** + * Returns the mean fusion of two feature maps. + * + * @param f_on_off feature map on off + * @param f_off_on feature map off on + * @return conspicuity map + */ +cv::Mat mean_fusion(cv::Mat f_on_off, cv::Mat f_off_on) { + cv::Mat C_l = 0.5 * (f_on_off + f_off_on); + double max_on_off; + double max_off_on; + cv::minMaxLoc(f_on_off, nullptr, &max_on_off); + cv::minMaxLoc(f_off_on, nullptr, &max_off_on); + double max = max_on_off >= max_off_on ? max_on_off : max_off_on; + cv::normalize(C_l, C_l, 0, max, cv::NORM_MINMAX, -1); + + return C_l.clone(); +} + +/** + * Returns the max fusion of two feature maps. + * + * @param f_on_off feature map on off + * @param f_off_on feature map off on + * @return conspicuity map + */ +cv::Mat max_fusion(cv::Mat f_on_off, cv::Mat f_off_on) { + cv::Mat C_l = cv::max(f_on_off, f_off_on); + double max_on_off; + double max_off_on; + cv::minMaxLoc(f_on_off, nullptr, &max_on_off); + cv::minMaxLoc(f_off_on, nullptr, &max_off_on); + double max = max_on_off >= max_off_on ? max_on_off : max_off_on; + cv::normalize(C_l, C_l, 0, max, cv::NORM_MINMAX, -1); + + return C_l.clone(); +} + +/** + * Computes the saliency map using mean fusion. + * + * @param C_l conspicuity map for L channel + * @param C_a conspicuity map for A channel + * @param C_b conspicuity map for B channel + * @return saliency map + */ +cv::Mat mean_fusion_saliency(cv::Mat C_l, cv::Mat C_a, cv::Mat C_b) { + cv::Mat S = (1/3.0) * (C_l + C_a + C_b); + double max_C_l; + double max_C_a; + double max_C_b; + cv::minMaxLoc(C_l, nullptr, &max_C_l); + cv::minMaxLoc(C_a, nullptr, &max_C_a); + cv::minMaxLoc(C_b, nullptr, &max_C_b); + double max = max_C_l >= max_C_a ? (max_C_l >= max_C_b ? max_C_l : max_C_b) : (max_C_a >= max_C_b ? max_C_a : max_C_b); + cv::normalize(S, S, 0, max, cv::NORM_MINMAX, -1); + + return S; +} + +/** + * Computes the saliency map using max fusion. + * + * @param C_l conspicuity map for L channel + * @param C_a conspicuity map for A channel + * @param C_b conspicuity map for B channel + * @return saliency map + */ +cv::Mat max_fusion_saliency(cv::Mat C_l, cv::Mat C_a, cv::Mat C_b) { + cv::Mat S = cv::max(C_l, C_a); + S = cv::max(S, C_b); + + double max_C_l; + double max_C_a; + double max_C_b; + cv::minMaxLoc(C_l, nullptr, &max_C_l); + cv::minMaxLoc(C_a, nullptr, &max_C_a); + cv::minMaxLoc(C_b, nullptr, &max_C_b); + double max = max_C_l >= max_C_a ? (max_C_l >= max_C_b ? max_C_l : max_C_b) : (max_C_a >= max_C_b ? max_C_a : max_C_b); + cv::normalize(S, S, 0, max, cv::NORM_MINMAX, -1); + + return S; +} diff --git a/ccv/saliency/gauss_pyramid.cpp b/ccv/saliency/gauss_pyramid.cpp new file mode 100644 index 0000000..0311fe0 --- /dev/null +++ b/ccv/saliency/gauss_pyramid.cpp @@ -0,0 +1,35 @@ +#include "includes/gauss_pyramid.h" + +gauss_pyramid::gauss_pyramid() {} + +gauss_pyramid::gauss_pyramid(cv::Mat img, float sigma, int number_of_layers) +{ + cv::Mat blurredImage; + cv::Mat resizedImage = img.clone(); + for (int i = 0; i < number_of_layers; i++) + { + cv::GaussianBlur(resizedImage, blurredImage, cv::Size(0, 0), sigma, sigma, cv::BORDER_REPLICATE); + _layers.push_back(blurredImage.clone()); + cv::resize(blurredImage, resizedImage, cv::Size(), 0.5, 0.5, cv::INTER_NEAREST); + } +} + +cv::Mat gauss_pyramid::get(int layer) const +{ + return _layers.at((unsigned long) layer); +} + +cv::Mat gauss_pyramid::get(int layer) +{ + return _layers.at((unsigned long) layer); +} + +unsigned long gauss_pyramid::get_number_of_layers() const +{ + return _layers.size(); +} + +unsigned long gauss_pyramid::get_number_of_layers() +{ + return _layers.size(); +} diff --git a/ccv/saliency/includes/fusion.h b/ccv/saliency/includes/fusion.h new file mode 100644 index 0000000..df473a5 --- /dev/null +++ b/ccv/saliency/includes/fusion.h @@ -0,0 +1,9 @@ +#ifndef SHEET6_FUSION_H +#define SHEET6_FUSION_H + +cv::Mat mean_fusion(cv::Mat f_on_off, cv::Mat f_off_on); +cv::Mat max_fusion(cv::Mat f_on_off, cv::Mat f_off_on); +cv::Mat mean_fusion_saliency(cv::Mat C_l, cv::Mat C_a, cv::Mat C_b); +cv::Mat max_fusion_saliency(cv::Mat C_l, cv::Mat C_a, cv::Mat C_b); + +#endif //SHEET6_FUSION_H diff --git a/ccv/saliency/includes/gauss_pyramid.h b/ccv/saliency/includes/gauss_pyramid.h new file mode 100644 index 0000000..7a54d26 --- /dev/null +++ b/ccv/saliency/includes/gauss_pyramid.h @@ -0,0 +1,20 @@ +#ifndef SHEET3_GAUSS_PYRAMID_H +#define SHEET3_GAUSS_PYRAMID_H + +#include + +class gauss_pyramid +{ +private: + std::vector _layers; +public: + gauss_pyramid(); + gauss_pyramid(cv::Mat img, float sigma, int number_of_layers); + cv::Mat get(int layer) const; + cv::Mat get(int layer); + unsigned long get_number_of_layers() const; + unsigned long get_number_of_layers(); +}; + + +#endif //SHEET3_GAUSS_PYRAMID_H diff --git a/ccv/saliency/includes/lab_pyramid.h b/ccv/saliency/includes/lab_pyramid.h new file mode 100644 index 0000000..1eca8dd --- /dev/null +++ b/ccv/saliency/includes/lab_pyramid.h @@ -0,0 +1,118 @@ +#ifndef SHEET3_LAB_PYRAMID_H +#define SHEET3_LAB_PYRAMID_H + +#include +#include "gauss_pyramid.h" + +class lab_pyramid { +private: + cv::Mat _inputImage_lab; + cv::Mat _inputImage_float; + cv::Mat _imageChannels[3]; + gauss_pyramid _pyramids[3]; + // contrast maps + static std::vector _cs_contrast_l; + static std::vector _sc_contrast_l; + static std::vector _cs_contrast_a; + static std::vector _sc_contrast_a; + static std::vector _cs_contrast_b; + static std::vector _sc_contrast_b; + // feature maps + static cv::Mat _cs_F_l; + static cv::Mat _sc_F_l; + static cv::Mat _cs_F_a; + static cv::Mat _cs_F_b; + static cv::Mat _sc_F_a; + static cv::Mat _sc_F_b; + // conspicuity maps + static cv::Mat _C_l; + static cv::Mat _C_a; + static cv::Mat _C_b; + // number of layers + static int _number_of_layers; +public: + const static int COLOR_L = 0 + const static int COLOR_A = 1; + const static int COLOR_B = 2; + + /** + * Initializes a LAB pyramid. + * + * @param image_filename the filename of the image that should be used + */ + lab_pyramid(cv::String image_filename); + + /** + * Initializes a LAB pyramid. + * + * @param image the image that should be used + */ + lab_pyramid(cv::Mat image); + + /** + * Creates the gaussian pyramids for all channels with the given number of layers each. + * + * @param sigma the sigma for the gaussian pyramids + * @param number_of_layers number of layers for gaussian pyramid + */ + void create_pyramids(float sigma, int number_of_layers); + + /** + * Before this method can be called, pyramids have to be created via create_pyramids. + * + * @param channel the channel you want to get (COLOR_L, COLOR_A or COLOR_B) + * @return the gaussian_pyramid for the given channel + */ + gauss_pyramid get_pyramid(int channel); + + /** + * Computes the center-surround and surround-center contrasts and stores them for later use. + * + * @param center the center pyramid + * @param surround the surround pyramid + * @param number_of_layers the number of layers used to create the two pyramids + */ + void static compute_dog(lab_pyramid center, lab_pyramid surround, int number_of_layers); + + /** + * Visualizes the center-surround and surround-center contrasts. They have to be computed first. + */ + void static visualize_dog(); + + /** + * Takes the scale images, adds them up and returns the result. + * + * @param scale_images the scale images + * @return the sum of the scale images + */ + cv::Mat static across_scale_addition(const std::vector &scale_images); + + /** + * Computes the feature maps. + * Has to be called after compute_dog. + */ + void static compute_feature_maps(); + + /** + * Computes the conspicuity maps. + * Has to be called after compute_feature_maps. + */ + void static compute_conspicuity_maps(); + + /** + * Before this method can be called, the conspicuity maps must be computed via compute_conspicuity_maps. + * + * @param channel the channel you want to get (COLOR_L, COLOR_A, COLOR_B) + * @return the conspicuity map for the given channel + */ + cv::Mat static get_conspicuity_map(int channel); + + /** + * Visualizes the feature maps. + * Has to be called after compute_feature_maps. + */ + void static visualize_feature_maps(); +}; + + +#endif //SHEET3_LAB_PYRAMID_H diff --git a/ccv/saliency/lab_pyramid.cpp b/ccv/saliency/lab_pyramid.cpp new file mode 100644 index 0000000..d0597fc --- /dev/null +++ b/ccv/saliency/lab_pyramid.cpp @@ -0,0 +1,185 @@ +#include "includes/lab_pyramid.h" +#include "includes/fusion.h" + +// number of layers +int lab_pyramid::_number_of_layers = 0; +// contrast maps +std::vector lab_pyramid::_cs_contrast_l = std::vector(); +std::vector lab_pyramid::_sc_contrast_l = std::vector(); +std::vector lab_pyramid::_cs_contrast_a = std::vector(); +std::vector lab_pyramid::_sc_contrast_a = std::vector(); +std::vector lab_pyramid::_cs_contrast_b = std::vector(); +std::vector lab_pyramid::_sc_contrast_b = std::vector(); +// feature maps +cv::Mat lab_pyramid::_cs_F_l; +cv::Mat lab_pyramid::_sc_F_l; +cv::Mat lab_pyramid::_cs_F_a; +cv::Mat lab_pyramid::_sc_F_a; +cv::Mat lab_pyramid::_cs_F_b; +cv::Mat lab_pyramid::_sc_F_b; +// conspicuity maps +cv::Mat lab_pyramid::_C_l; +cv::Mat lab_pyramid::_C_a; +cv::Mat lab_pyramid::_C_b; + +lab_pyramid::lab_pyramid(cv::String image_filename) { + cv::Mat image_rgb = cv::imread(image_filename, cv::IMREAD_COLOR); + cv::cvtColor(image_rgb, _inputImage_lab, cv::COLOR_BGR2Lab); + cv::split(_inputImage_lab ,_imageChannels); +}; + +lab_pyramid::lab_pyramid(cv::Mat image) { + cv::cvtColor(image, _inputImage_lab, cv::COLOR_BGR2Lab); + _inputImage_lab.convertTo(_inputImage_float, CV_32F); + cv::split(_inputImage_float, _imageChannels); +} + +void lab_pyramid::create_pyramids(float sigma, int number_of_layers) +{ + _pyramids[COLOR_L] = gauss_pyramid(_imageChannels[COLOR_L], sigma, number_of_layers); + _pyramids[COLOR_A] = gauss_pyramid(_imageChannels[COLOR_A], sigma, number_of_layers); + _pyramids[COLOR_B] = gauss_pyramid(_imageChannels[COLOR_B], sigma, number_of_layers); +} + +gauss_pyramid lab_pyramid::get_pyramid(int channel) +{ + switch (channel) + { + case COLOR_L: + return _pyramids[COLOR_L]; + case COLOR_A: + return _pyramids[COLOR_A]; + case COLOR_B: + return _pyramids[COLOR_B]; + default: + throw std::invalid_argument( "received invalid channel value, use COLOR_L, COLOR_A or COLOR_B" ); + } +} + +void lab_pyramid::compute_dog(lab_pyramid center, lab_pyramid surround, int number_of_layers) { + _number_of_layers = number_of_layers; + + // L channel + gauss_pyramid center_l = center.get_pyramid(COLOR_L); + gauss_pyramid surround_l = surround.get_pyramid(COLOR_L); + + // A channel + gauss_pyramid center_a = center.get_pyramid(COLOR_A); + gauss_pyramid surround_a = surround.get_pyramid(COLOR_A); + + // A channel + gauss_pyramid center_b = center.get_pyramid(COLOR_B); + gauss_pyramid surround_b = surround.get_pyramid(COLOR_B); + + for (int layer = 0; layer < number_of_layers; layer++) { + // L channel + cv::Mat center_layer_mat_L = center_l.get(layer); + cv::Mat surround_layer_mat_L = surround_l.get(layer); + cv::Mat dog_raw_cs_L = center_layer_mat_L - surround_layer_mat_L; + cv::Mat dog_final_cs_L; + cv::Mat dog_final_sc_L; + cv::threshold(dog_raw_cs_L, dog_final_cs_L, 0, 1, cv::THRESH_TOZERO); + _cs_contrast_l.push_back(dog_final_cs_L.clone()); + cv::Mat dog_raw_sc_L = surround_layer_mat_L - center_layer_mat_L; + cv::threshold(dog_raw_sc_L, dog_final_sc_L, 0, 1, cv::THRESH_TOZERO); + _sc_contrast_l.push_back(dog_final_sc_L.clone()); + + // A channel + cv::Mat center_layer_mat_a = center_a.get(layer); + cv::Mat surround_layer_mat_a = surround_a.get(layer); + cv::Mat dog_raw_cs_a = center_layer_mat_a - surround_layer_mat_a; + cv::Mat dog_final_cs_a; + cv::Mat dog_final_sc_a; + cv::threshold(dog_raw_cs_a, dog_final_cs_a, 0, 1, cv::THRESH_TOZERO); + _cs_contrast_a.push_back(dog_final_cs_a.clone()); + cv::Mat dog_raw_sc_a = surround_layer_mat_a - center_layer_mat_a; + cv::threshold(dog_raw_sc_a, dog_final_sc_a, 0, 1, cv::THRESH_TOZERO); + _sc_contrast_a.push_back(dog_final_sc_a.clone()); + + // B channel + cv::Mat center_layer_mat_b = center_b.get(layer); + cv::Mat surround_layer_mat_b = surround_b.get(layer); + cv::Mat dog_raw_cs_b = center_layer_mat_b - surround_layer_mat_b; + cv::Mat dog_final_cs_b; + cv::Mat dog_final_sc_b; + cv::threshold(dog_raw_cs_b, dog_final_cs_b, 0, 1, cv::THRESH_TOZERO); + _cs_contrast_b.push_back(dog_final_cs_b.clone()); + cv::Mat dog_raw_sc_b = surround_layer_mat_b - center_layer_mat_b; + cv::threshold(dog_raw_sc_b, dog_final_sc_b, 0, 1, cv::THRESH_TOZERO); + _sc_contrast_b.push_back(dog_final_sc_b.clone()); + } +} + +void lab_pyramid::compute_feature_maps() { + _cs_F_l = across_scale_addition(_cs_contrast_l); + _sc_F_l = across_scale_addition(_sc_contrast_l); + _cs_F_a = across_scale_addition(_cs_contrast_a); + _sc_F_a = across_scale_addition(_sc_contrast_a); + _cs_F_b = across_scale_addition(_cs_contrast_b); + _sc_F_b = across_scale_addition(_sc_contrast_b); +} + +void lab_pyramid::compute_conspicuity_maps() { + _C_l = max_fusion(_cs_F_l, _sc_F_l); + _C_a = max_fusion(_cs_F_a, _sc_F_a); + _C_b = max_fusion(_cs_F_b, _sc_F_b); +} + +cv::Mat lab_pyramid::get_conspicuity_map(int channel) { + switch (channel) + { + case COLOR_L: + return _C_l; + case COLOR_A: + return _C_a; + case COLOR_B: + return _C_b; + default: + throw std::invalid_argument( "received invalid channel value, use COLOR_L, COLOR_A or COLOR_B" ); + } +} + +cv::Mat lab_pyramid::across_scale_addition(const std::vector &scale_images) { + cv::Mat result = scale_images.front(); + cv::Size original_size = scale_images.front().size(); + for (unsigned long i = 1; i < scale_images.size(); i++) { + cv::Mat resized_image; + cv::resize(scale_images.at(i), resized_image, original_size, 0, 0, cv::INTER_CUBIC); + result += resized_image; + } + return result; +} + +void lab_pyramid::visualize_dog() { + for (unsigned long layer = 0; layer < _number_of_layers; layer++) { + cv::namedWindow("CS L"); + cv::imshow("CS L", _cs_contrast_l.at(layer)); + cv::namedWindow("SC L"); + cv::imshow("SC L", _sc_contrast_l.at(layer)); + cv::namedWindow("CS A"); + cv::imshow("CS A", _cs_contrast_a.at(layer)); + cv::namedWindow("SC A"); + cv::imshow("SC A", _cs_contrast_a.at(layer)); + cv::namedWindow("CS B"); + cv::imshow("CS B", _cs_contrast_b.at(layer)); + cv::namedWindow("SC B"); + cv::imshow("SC B", _cs_contrast_b.at(layer)); + cv::waitKey(0); + } +} + +void lab_pyramid::visualize_feature_maps() { + cv::namedWindow("CS F L"); + cv::imshow("CS F L", _cs_F_l); + cv::namedWindow("CS F L"); + cv::imshow("SC F L", _sc_F_l); + cv::namedWindow("CS F A"); + cv::imshow("CS F A", _cs_F_a); + cv::namedWindow("SC F A"); + cv::imshow("SC F A", _sc_F_a); + cv::namedWindow("CS F B"); + cv::imshow("CS F B", _cs_F_b); + cv::namedWindow("CS F B"); + cv::imshow("SC F B", _sc_F_b); + cv::waitKey(0); +} diff --git a/ccv/saliency/main.cpp b/ccv/saliency/main.cpp new file mode 100644 index 0000000..6376e35 --- /dev/null +++ b/ccv/saliency/main.cpp @@ -0,0 +1,54 @@ +#include +#include + +#include "includes/lab_pyramid.h" +#include "includes/fusion.h" + +/** + * Entry point of program + * @param argc number of arguments + * @param argv CLI arguments, 0: name of program, 1: input file name, 2: output file name + * @return status code (0: everything OK, -1: not right amount of arguments) + */ +int main(int argc, char** argv) { + if (argc != 3) { + printf("usage: \n"); + return -1; + } + + // read image + cv::Mat image = cv::imread(argv[1], cv::ImreadModes::IMREAD_COLOR); + + // tweakable factors + int layers = 4; + float sigma_center = 5; + float sigma_surround = 9; + + // create LAB pyramids + lab_pyramid lab_pyr_center = lab_pyramid(image); + lab_pyr_center.create_pyramids(sigma_center, layers); + lab_pyramid lab_pyr_surround = lab_pyramid(image); + lab_pyr_surround.create_pyramids(sigma_surround, layers); + + // create contrast maps + lab_pyramid::compute_dog(lab_pyr_center, lab_pyr_surround, layers); + // create feature maps + lab_pyramid::compute_feature_maps(); + // create conspicuity maps + lab_pyramid::compute_conspicuity_maps(); + // get conspicuity maps + cv::Mat C_l = lab_pyramid::get_conspicuity_map(lab_pyramid::COLOR_L); + cv::Mat C_a = lab_pyramid::get_conspicuity_map(lab_pyramid::COLOR_A); + cv::Mat C_b = lab_pyramid::get_conspicuity_map(lab_pyramid::COLOR_B); + // get saliency map + cv::Mat saliency = max_fusion_saliency(C_l, C_a, C_b); + + // convert saliency map to correct output format + cv::Mat output_image; + saliency.convertTo(output_image, CV_8UC1); + + // write output + cv::imwrite(argv[2], output_image); + + return 0; +}