grouped_variances.hpp

Go to the documentation of this file.

#ifndef TATAMI_STATS_GROUPED_VARIANCES_HPP
#define TATAMI_STATS_GROUPED_VARIANCES_HPP
 
#include "utils.hpp"
#include "variances.hpp"
 
#include <vector>
#include <algorithm>
#include <cstddef>
 
#include "tatami/tatami.hpp"
#include "sanisizer/sanisizer.hpp"
 
namespace tatami_stats {
 
namespace grouped_variances {
 
struct Options {
    bool skip_nan = false;
 
    int num_threads = 1;
};
 
namespace internal {
 
template<typename Index_, typename Output_>
void finish_means(std::size_t num_groups, const Index_* group_size, Output_* output_means) {
    for (I<decltype(num_groups)> b = 0; b < num_groups; ++b) {
        if (group_size[b]) {
            output_means[b] /= group_size[b];
        } else {
            output_means[b] = std::numeric_limits<Output_>::quiet_NaN();
        }
    }
}
 
template<typename Index_, typename Output_>
void finish_variances(std::size_t num_groups, const Index_* group_size, Output_* output_variances) {
    for (I<decltype(num_groups)> b = 0; b < num_groups; ++b) {
        if (group_size[b] > 1) {
            output_variances[b] /= group_size[b] - 1;
        } else {
            output_variances[b] = std::numeric_limits<Output_>::quiet_NaN();
        }
    }
}
 
}
template<typename Value_, typename Index_, typename Group_, typename Output_>
void direct(
    const Value_* ptr, 
    Index_ num, 
    const Group_* group, 
    std::size_t num_groups, 
    const Index_* group_size, 
    Output_* output_means, 
    Output_* output_variances, 
    bool skip_nan, 
    Index_* valid_group_size)
{
    std::fill_n(output_means, num_groups, 0);
    std::fill_n(output_variances, num_groups, 0);
 
    ::tatami_stats::internal::nanable_ifelse<Value_>(
        skip_nan,
        [&]() -> void {
            std::fill_n(valid_group_size, num_groups, 0);
 
            for (Index_ j = 0; j < num; ++j) {
                auto x = ptr[j];
                if (!std::isnan(x)) {
                    auto b = group[j];
                    output_means[b] += x;
                    ++valid_group_size[b];
                }
            }
            internal::finish_means(num_groups, valid_group_size, output_means);
 
            for (Index_ j = 0; j < num; ++j) {
                auto x = ptr[j];
                if (!std::isnan(x)) {
                    auto b = group[j];
                    auto delta = x - output_means[b];
                    output_variances[b] += delta * delta;
                }
            }
            internal::finish_variances(num_groups, valid_group_size, output_variances);
        },
        [&]() -> void {
            for (Index_ j = 0; j < num; ++j) {
                output_means[group[j]] += ptr[j];
            }
            internal::finish_means(num_groups, group_size, output_means);
 
            for (Index_ j = 0; j < num; ++j) {
                auto b = group[j];
                auto delta = ptr[j] - output_means[b];
                output_variances[b] += delta * delta;
            }
            internal::finish_variances(num_groups, group_size, output_variances);
        }
    );
}
 
template<typename Value_, typename Index_, typename Group_, typename Output_>
void direct(
    const Value_* value, 
    const Index_* index, 
    Index_ num_nonzero, 
    const Group_* group, 
    std::size_t num_groups, 
    const Index_* group_size, 
    Output_* output_means, 
    Output_* output_variances, 
    Index_* output_nonzero,
    bool skip_nan, 
    Index_* valid_group_size)
{
    std::fill_n(output_means, num_groups, 0);
    std::fill_n(output_nonzero, num_groups, 0);
    std::fill_n(output_variances, num_groups, 0);
 
    ::tatami_stats::internal::nanable_ifelse<Value_>(
        skip_nan,
        [&]() -> void {
            std::copy_n(group_size, num_groups, valid_group_size);
 
            for (Index_ j = 0; j < num_nonzero; ++j) {
                auto x = value[j];
                auto b = group[index[j]];
                if (!std::isnan(x)) {
                    output_means[b] += x;
                    ++(output_nonzero[b]);
                } else {
                    --(valid_group_size[b]);
                }
            }
            internal::finish_means(num_groups, valid_group_size, output_means);
 
            for (Index_ j = 0; j < num_nonzero; ++j) {
                auto x = value[j];
                if (!std::isnan(x)) {
                    auto b = group[index[j]];
                    auto delta = x - output_means[b];
                    output_variances[b] += delta * delta;
                }
            }
            for (I<decltype(num_groups)> b = 0; b < num_groups; ++b) {
                output_variances[b] += output_means[b] * output_means[b] * (valid_group_size[b] - output_nonzero[b]);
            }
            internal::finish_variances(num_groups, valid_group_size, output_variances);
        },
        [&]() -> void {
            for (Index_ j = 0; j < num_nonzero; ++j) {
                auto b = group[index[j]];
                output_means[b] += value[j];
                ++output_nonzero[b];
            }
            internal::finish_means(num_groups, group_size, output_means);
 
            for (Index_ j = 0; j < num_nonzero; ++j) {
                auto b = group[index[j]];
                auto delta = value[j] - output_means[b];
                output_variances[b] += delta * delta;
            }
            for (I<decltype(num_groups)> b = 0; b < num_groups; ++b) {
                output_variances[b] += output_means[b] * output_means[b] * (group_size[b] - output_nonzero[b]);
            }
            internal::finish_variances(num_groups, group_size, output_variances);
        }
    );
}
 
template<typename Value_, typename Index_, typename Group_, typename Output_>
void apply(bool row, const tatami::Matrix<Value_, Index_>& mat, const Group_* group, std::size_t num_groups, const Index_* group_size, Output_** output, const Options& sopt) {
    const Index_ dim = (row ? mat.nrow() : mat.ncol());
    const Index_ otherdim = (row ? mat.ncol() : mat.nrow());
 
    if (mat.sparse()) {
        if (mat.prefer_rows() == row) {
            tatami::parallelize([&](int, Index_ start, Index_ len) -> void {
                auto ext = tatami::consecutive_extractor<true>(mat, row, start, len);
                auto xbuffer = tatami::create_container_of_Index_size<std::vector<Value_> >(otherdim);
                auto ibuffer = tatami::create_container_of_Index_size<std::vector<Index_> >(otherdim);
 
                auto tmp_means = sanisizer::create<std::vector<Output_> >(num_groups);
                auto output_variances = sanisizer::create<std::vector<Output_> >(num_groups);
                auto tmp_nonzero = sanisizer::create<std::vector<Index_> >(num_groups);
                auto valid_group_size = sanisizer::create<std::vector<Index_> >(sopt.skip_nan ? num_groups : 0);
 
                for (Index_ i = 0; i < len; ++i) {
                    auto range = ext->fetch(xbuffer.data(), ibuffer.data());
                    direct(
                        range.value, 
                        range.index, 
                        range.number, 
                        group, 
                        num_groups, 
                        group_size, 
                        tmp_means.data(), 
                        output_variances.data(), 
                        tmp_nonzero.data(), 
                        sopt.skip_nan, 
                        valid_group_size.data()
                    );
 
                    for (I<decltype(num_groups)> g = 0; g < num_groups; ++g) {
                        output[g][i + start] = output_variances[g];
                    }
                }
            }, dim, sopt.num_threads);
 
        } else {
            // Order within each observed vector doesn't affect numerical
            // precision of the outcome, as addition order for each objective
            // vector is already well-defined for a running calculation.
            tatami::Options opt;
            opt.sparse_ordered_index = false; 
 
            tatami::parallelize([&](int thread, Index_ start, Index_ len) -> void {
                std::vector<variances::RunningSparse<Output_, Value_, Index_> > runners;
                runners.reserve(num_groups);
                std::vector<LocalOutputBuffer<Output_> > local_var_output;
                local_var_output.reserve(num_groups);
                std::vector<std::vector<Output_> > local_mean_output;
                local_mean_output.reserve(num_groups);
 
                for (I<decltype(num_groups)> g = 0; g < num_groups; ++g) {
                    local_var_output.emplace_back(thread, start, len, output[g]);
                    local_mean_output.emplace_back(len);
                    runners.emplace_back(len, local_mean_output.back().data(), local_var_output.back().data(), sopt.skip_nan, start);
                }
 
                auto ext = tatami::consecutive_extractor<true>(mat, !row, static_cast<Index_>(0), otherdim, start, len, opt);
                auto xbuffer = tatami::create_container_of_Index_size<std::vector<Value_> >(len);
                auto ibuffer = tatami::create_container_of_Index_size<std::vector<Index_> >(len);
 
                for (Index_ i = 0; i < otherdim; ++i) {
                    auto range = ext->fetch(xbuffer.data(), ibuffer.data());
                    runners[group[i]].add(range.value, range.index, range.number);
                }
 
                for (I<decltype(num_groups)> g = 0; g < num_groups; ++g) {
                    runners[g].finish();
                    local_var_output[g].transfer();
                }
            }, dim, sopt.num_threads);
        }
 
    } else {
        if (mat.prefer_rows() == row) {
            tatami::parallelize([&](int, Index_ start, Index_ len) -> void {
                auto ext = tatami::consecutive_extractor<false>(mat, row, start, len);
                auto xbuffer = tatami::create_container_of_Index_size<std::vector<Value_> >(otherdim);
 
                auto tmp_means = sanisizer::create<std::vector<Output_> >(num_groups);
                auto output_variances = sanisizer::create<std::vector<Output_> >(num_groups);
                auto valid_group_size = sanisizer::create<std::vector<Index_> >(sopt.skip_nan ? num_groups : 0);
 
                for (Index_ i = 0; i < len; ++i) {
                    auto ptr = ext->fetch(xbuffer.data());
                    direct(
                        ptr, 
                        otherdim, 
                        group, 
                        num_groups, 
                        group_size, 
                        tmp_means.data(), 
                        output_variances.data(), 
                        sopt.skip_nan, 
                        valid_group_size.data()
                    );
 
                    for (I<decltype(num_groups)> g = 0; g < num_groups; ++g) {
                        output[g][i + start] = output_variances[g];
                    }
                }
            }, dim, sopt.num_threads);
 
        } else {
            tatami::parallelize([&](int thread, Index_ start, Index_ len) -> void {
                std::vector<variances::RunningDense<Output_, Value_, Index_> > runners;
                runners.reserve(num_groups);
                std::vector<LocalOutputBuffer<Output_> > local_var_output;
                local_var_output.reserve(num_groups);
                std::vector<std::vector<Output_> > local_mean_output;
                local_mean_output.reserve(num_groups);
 
                for (I<decltype(num_groups)> g = 0; g < num_groups; ++g) {
                    local_var_output.emplace_back(thread, start, len, output[g]);
                    local_mean_output.emplace_back(len);
                    runners.emplace_back(len, local_mean_output.back().data(), local_var_output.back().data(), sopt.skip_nan);
                }
 
                auto xbuffer = tatami::create_container_of_Index_size<std::vector<Value_> >(len);
                auto ext = tatami::consecutive_extractor<false>(mat, !row, static_cast<Index_>(0), otherdim, start, len);
 
                for (Index_ i = 0; i < otherdim; ++i) {
                    auto ptr = ext->fetch(xbuffer.data());
                    runners[group[i]].add(ptr);
                }
 
                for (I<decltype(num_groups)> g = 0; g < num_groups; ++g) {
                    runners[g].finish();
                    local_var_output[g].transfer();
                }
            }, dim, sopt.num_threads);
        }
    }
}
 
// Back-compatibility.
template<typename Value_, typename Index_, typename Group_, typename Output_>
void apply(bool row, const tatami::Matrix<Value_, Index_>* p, const Group_* group, std::size_t num_groups, const Index_* group_size, Output_** output, const Options& sopt) {
    apply(row, *p, group, num_groups, group_size, output, sopt);
}
template<typename Output_ = double, typename Value_, typename Index_, typename Group_>
std::vector<std::vector<Output_> > by_row(const tatami::Matrix<Value_, Index_>& mat, const Group_* group, const Options& sopt) {
    auto mydim = mat.nrow();
    auto group_size = tabulate_groups(group, mat.ncol());
    auto ngroup = group_size.size();
 
    auto output = sanisizer::create<std::vector<std::vector<Output_> > >(ngroup);
    std::vector<Output_*> ptrs;
    ptrs.reserve(output.size());
    for (auto& o : output) {
        o.resize(mydim);
        ptrs.push_back(o.data());
    }
 
    apply(true, mat, group, ngroup, group_size.data(), ptrs.data(), sopt);
    return output;
}
 
// Back-compatibility.
template<typename Output_ = double, typename Value_, typename Index_, typename Group_>
std::vector<std::vector<Output_> > by_row(const tatami::Matrix<Value_, Index_>* p, const Group_* group, const Options& sopt) {
    return by_row<Output_>(*p, group, sopt);
}
 
template<typename Output_ = double, typename Value_, typename Index_, typename Group_>
std::vector<std::vector<Output_> > by_row(const tatami::Matrix<Value_, Index_>& mat, const Group_* group) {
    return by_row<Output_>(mat, group, Options());
}
 
template<typename Output_ = double, typename Value_, typename Index_, typename Group_>
std::vector<std::vector<Output_> > by_row(const tatami::Matrix<Value_, Index_>* p, const Group_* group) {
    return by_row<Output_>(*p, group);
}
template<typename Output_ = double, typename Value_, typename Index_, typename Group_>
std::vector<std::vector<Output_> > by_column(const tatami::Matrix<Value_, Index_>& mat, const Group_* group, const Options& sopt) {
    auto mydim = mat.ncol();
    auto group_size = tabulate_groups(group, mat.nrow());
    auto ngroup = group_size.size();
 
    auto output = sanisizer::create<std::vector<std::vector<Output_> > >(ngroup);
    std::vector<Output_*> ptrs;
    ptrs.reserve(output.size());
    for (auto& o : output) {
        o.resize(mydim);
        ptrs.push_back(o.data());
    }
 
    apply(false, mat, group, ngroup, group_size.data(), ptrs.data(), sopt);
    return output;
}
 
// Back-compatibility.
template<typename Output_ = double, typename Value_, typename Index_, typename Group_>
std::vector<std::vector<Output_> > by_column(const tatami::Matrix<Value_, Index_>* p, const Group_* group, const Options& sopt) {
    return by_column<Output_>(*p, group, sopt);
}
 
template<typename Output_ = double, typename Value_, typename Index_, typename Group_>
std::vector<std::vector<Output_> > by_column(const tatami::Matrix<Value_, Index_>& mat, const Group_* group) {
    return by_column<Output_>(mat, group, Options());
}
 
template<typename Output_ = double, typename Value_, typename Index_, typename Group_>
std::vector<std::vector<Output_> > by_column(const tatami::Matrix<Value_, Index_>* p, const Group_* group) {
    return by_column<Output_>(*p, group);
}
}
 
}
 
#endif