contour_correspondence.hpp

/*
 * eos - A 3D Morphable Model fitting library written in modern C++11/14.
 *
 * File: include/eos/fitting/contour_correspondence.hpp
 *
 * Copyright 2015, 2016, 2023 Patrik Huber
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#pragma once
 
#ifndef EOS_CONTOUR_CORRESPONDENCE_HPP
#define EOS_CONTOUR_CORRESPONDENCE_HPP
 
#include "eos/core/Landmark.hpp"
#include "eos/core/Mesh.hpp"
#include "eos/render/matrix_projection.hpp"
 
#include "cereal/types/vector.hpp"
#include "cereal/archives/json.hpp"
 
#include "toml.hpp"
 
#include "Eigen/Core"
 
#include <vector>
#include <string>
#include <algorithm>
#include <fstream>
#include <tuple>
 
namespace eos {
namespace fitting {
 
// Forward declarations of later used functions and types:
struct ModelContour;
struct ContourLandmarks;
std::pair<std::vector<std::string>, std::vector<int>>
select_contour(float yaw_angle, const ContourLandmarks& contour_landmarks, const ModelContour& model_contour,
               float frontal_range_threshold = 7.5f);
std::tuple<std::vector<Eigen::Vector2f>, std::vector<Eigen::Vector4f>, std::vector<int>>
get_nearest_contour_correspondences(const core::LandmarkCollection<Eigen::Vector2f>& landmarks,
                                    const std::vector<std::string>& landmark_contour_identifiers,
                                    const std::vector<int>& model_contour_indices, const core::Mesh& mesh,
                                    const Eigen::Matrix4f& view_model,
                                    const Eigen::Matrix4f& ortho_projection, const Eigen::Vector4f& viewport);
 
struct ModelContour
{
    // starting from right side, eyebrow-height: (I think the order matters here)
    std::vector<int> right_contour;
    /* 23 = middle, below chin - not included in the contour here */
    // starting from left side, eyebrow-height:
    std::vector<int> left_contour;
 
    // We store r/l separately because we currently only fit to the contour facing the camera.
    // Also if we were to fit to the whole contour: Be careful not to just fit to the closest. The
    // "invisible" ones behind might be closer on an e.g 90° angle. Store CNT for left/right side separately?
 
    static ModelContour load(std::string filename)
    {
        ModelContour contour;
 
        std::ifstream file(filename);
        if (!file)
        {
            throw std::runtime_error("Error opening given file: " + filename);
        }
        cereal::JSONInputArchive input_archive(file);
        input_archive(contour);
 
        return contour;
    };
 
    friend class cereal::access;
    template <class Archive>
    void serialize(Archive& archive)
    {
        archive(cereal::make_nvp("right_contour", right_contour),
                cereal::make_nvp("left_contour", left_contour));
    };
};
 
struct ContourLandmarks
{
    // starting from right side, eyebrow-height.
    std::vector<std::string> right_contour;
    // Chin point is not included in the contour here.
    // starting from left side, eyebrow-height. Order doesn't matter here.
    std::vector<std::string> left_contour;
 
    // Note: We store r/l separately because we currently only fit to the contour facing the camera.
 
    static ContourLandmarks load(std::string filename)
    {
        // parse() as well as extracting the data can throw std::runtime error or toml::exception,
        // so ideally you'd want to call this c'tor within a try-catch.
        const auto data = toml::parse(filename);
 
        // Todo: Handle the case when '[contour_landmarks]' is empty - now it just throws. Use
        // std::unordered_map::count.
        const auto& contour_table = toml::get<toml::table>(data.at("contour_landmarks"));
 
        ContourLandmarks contour;
        // There might be a vector<string> or a vector<int>, we need to check for that and convert to vector<string>.
        // Todo: Again, check whether the key exists first.
        // Read all the "right" contour landmarks:
        const auto& right_contour = toml::get<std::vector<toml::value>>(contour_table.at("right"));
        for (const auto& landmark : right_contour)
        {
            std::string value;
            switch (landmark.type())
            {
            case toml::value_t::integer:
                value = std::to_string(toml::get<int>(landmark));
                break;
            case toml::value_t::string:
                value = toml::get<std::string>(landmark);
                break;
            default:
                throw std::runtime_error("unexpected type : " + toml::stringize(landmark.type()));
            }
            contour.right_contour.push_back(value);
        }
        // Now the same for all the "left" contour landmarks:
        const auto& left_contour = toml::get<std::vector<toml::value>>(contour_table.at("left"));
        for (const auto& landmark : left_contour)
        {
            std::string value;
            switch (landmark.type())
            {
            case toml::value_t::integer:
                value = std::to_string(toml::get<int>(landmark));
                break;
            case toml::value_t::string:
                value = toml::get<std::string>(landmark);
                break;
            default:
                throw std::runtime_error("unexpected type : " + toml::stringize(landmark.type()));
            }
            contour.left_contour.push_back(value);
        }
 
        return contour;
    };
};
 
inline std::tuple<std::vector<Eigen::Vector2f>, std::vector<Eigen::Vector4f>, std::vector<int>>
get_contour_correspondences(const core::LandmarkCollection<Eigen::Vector2f>& landmarks,
                            const ContourLandmarks& contour_landmarks, const ModelContour& model_contour,
                            float yaw_angle, const core::Mesh& mesh, const Eigen::Matrix4f& view_model,
                            const Eigen::Matrix4f& ortho_projection, const Eigen::Vector4f& viewport,
                            float frontal_range_threshold = 7.5f)
{
    // Select which side of the contour we'll use:
    std::vector<int> model_contour_indices;
    std::vector<std::string> landmark_contour_identifiers;
    std::tie(landmark_contour_identifiers, model_contour_indices) =
        select_contour(yaw_angle, contour_landmarks, model_contour, frontal_range_threshold);
 
    // For each 2D contour landmark, get the corresponding 3D vertex point and vertex id:
    // Note/Todo: Loop here instead of calling this function where we have no idea what it's doing? What does
    // its documentation say?
    return get_nearest_contour_correspondences(landmarks, landmark_contour_identifiers, model_contour_indices,
                                               mesh, view_model, ortho_projection, viewport);
};
 
inline std::pair<std::vector<std::string>, std::vector<int>>
select_contour(float yaw_angle, const ContourLandmarks& contour_landmarks, const ModelContour& model_contour, float frontal_range_threshold)
{
    using std::begin;
    using std::end;
    std::vector<int> model_contour_indices;
    std::vector<std::string> contour_landmark_identifiers;
    if (yaw_angle >= -frontal_range_threshold) // positive yaw = subject looking to the left
    {
        // ==> we use the right cnt-lms
        model_contour_indices.insert(end(model_contour_indices), begin(model_contour.right_contour),
                                     end(model_contour.right_contour));
        contour_landmark_identifiers.insert(end(contour_landmark_identifiers),
                                            begin(contour_landmarks.right_contour),
                                            end(contour_landmarks.right_contour));
    }
    if (yaw_angle <= frontal_range_threshold)
    {
        // ==> we use the left cnt-lms
        model_contour_indices.insert(end(model_contour_indices), begin(model_contour.left_contour),
                                     end(model_contour.left_contour));
        contour_landmark_identifiers.insert(end(contour_landmark_identifiers),
                                            begin(contour_landmarks.left_contour),
                                            end(contour_landmarks.left_contour));
    }
    // Note there's an overlap between the angles - if a subject is between +- 7.5°, both contours get added.
    return std::make_pair(contour_landmark_identifiers, model_contour_indices);
};
 
inline std::tuple<std::vector<Eigen::Vector2f>, std::vector<Eigen::Vector4f>, std::vector<int>>
get_nearest_contour_correspondences(const core::LandmarkCollection<Eigen::Vector2f>& landmarks,
                                    const std::vector<std::string>& landmark_contour_identifiers,
                                    const std::vector<int>& model_contour_indices, const core::Mesh& mesh,
                                    const Eigen::Matrix4f& view_model,
                                    const Eigen::Matrix4f& ortho_projection, const Eigen::Vector4f& viewport)
{
    // These are the additional contour-correspondences we're going to find and then use!
    std::vector<Eigen::Vector4f> model_points_cnt; // the points in the 3D shape model
    std::vector<int> vertex_indices_cnt;           // their vertex indices
    std::vector<Eigen::Vector2f> image_points_cnt; // the corresponding 2D landmark points
 
    // For each 2D-CNT-LM, find the closest 3DMM-CNT-LM and add to correspondences:
    // Note: If we were to do this for all 3DMM vertices, then ray-casting (i.e. glm::unproject) would be
    // quicker to find the closest vertex)
    for (const auto& ibug_idx : landmark_contour_identifiers)
    {
        // Check if the contour landmark is amongst the landmarks given to us (from detector or ground truth):
        // (Note: Alternatively, we could filter landmarks beforehand and then just loop over landmarks =>
        // means one less function param here. Separate filtering from actual algorithm.)
        const auto result = std::find_if(begin(landmarks), end(landmarks), [&ibug_idx](auto&& e) {
            return e.name == ibug_idx;
        }); // => this can go outside the loop
        if (result == std::end(landmarks))
        {
            continue; // This should be okay; So it's possible that the function will not return any
                      // correspondences.
        }
 
        const auto screen_point_2d_contour_landmark = result->coordinates;
 
        std::vector<float> distances_2d;
        for (auto model_contour_vertex_idx : model_contour_indices) // we could actually pre-project them,
                                                                    // i.e. only project them once, not for
                                                                    // each landmark newly...
        {
            const Eigen::Vector3f screen_point_model_contour = render::project(
                mesh.vertices[model_contour_vertex_idx], view_model, ortho_projection, viewport);
            const double dist =
                (screen_point_model_contour.head<2>() - screen_point_2d_contour_landmark).norm();
            distances_2d.emplace_back(dist);
        }
        const auto min_ele = std::min_element(begin(distances_2d), end(distances_2d));
        // Todo: Cover the case when cnt_indices_to_use.size() is 0.
        const auto min_ele_idx = std::distance(begin(distances_2d), min_ele);
        const auto the_3dmm_vertex_id_that_is_closest = model_contour_indices[min_ele_idx];
 
        model_points_cnt.emplace_back(mesh.vertices[the_3dmm_vertex_id_that_is_closest].homogeneous());
        vertex_indices_cnt.emplace_back(the_3dmm_vertex_id_that_is_closest);
        image_points_cnt.emplace_back(screen_point_2d_contour_landmark);
    }
 
    return std::make_tuple(image_points_cnt, model_points_cnt, vertex_indices_cnt);
};
 
template <class T>
inline auto concat(const std::vector<T>& vec_a, const std::vector<T>& vec_b)
{
    std::vector<T> concatenated_vec;
    concatenated_vec.reserve(vec_a.size() + vec_b.size());
    concatenated_vec.insert(std::end(concatenated_vec), std::begin(vec_a), std::end(vec_a));
    concatenated_vec.insert(std::end(concatenated_vec), std::begin(vec_b), std::end(vec_b));
    return concatenated_vec;
};
 
} /* namespace fitting */
} /* namespace eos */
 
#endif /* EOS_CONTOUR_CORRESPONDENCE_HPP */