load_matrix.hpp

Go to the documentation of this file.

#ifndef TATAMI_MTX_SIMPLE_HPP
#define TATAMI_MTX_SIMPLE_HPP
 
#include "tatami/tatami.hpp"
#include "eminem/eminem.hpp"
#include "byteme/byteme.hpp"
 
namespace tatami_mtx {
 
struct Automatic {};
 
struct Options {
    bool row = true;
 
    size_t buffer_size = 65536;
 
    bool parallel = false;
 
    int compression = 3;
};
 
namespace internal {
 
template<bool row_, typename Value_, typename Index_, typename StoredValue_, typename StoredIndex_, typename TempIndex_, typename Parser_>
std::shared_ptr<tatami::Matrix<Value_, Index_> > load_sparse_matrix_basic(Parser_& parser, eminem::Field field, size_t NR, size_t NC, size_t NL) {
    std::vector<typename std::conditional<row_, TempIndex_, StoredIndex_>::type> rows;
    std::vector<typename std::conditional<!row_, TempIndex_, StoredIndex_>::type> columns;
    rows.reserve(NL), columns.reserve(NL);
    std::vector<StoredValue_> values;
    values.reserve(NL);
 
    if (field == eminem::Field::INTEGER) {
        parser.scan_integer([&](size_t r, size_t c, int v) -> void {
            values.push_back(v);
            rows.push_back(r - 1);
            columns.push_back(c - 1);
        });
 
    } else if (field == eminem::Field::REAL || field == eminem::Field::DOUBLE) {
        parser.scan_real([&](size_t r, size_t c, double v) -> void {
            values.push_back(v);
            rows.push_back(r - 1);
            columns.push_back(c - 1);
        });
 
    } else {
        throw std::runtime_error("unsupported Matrix Market field type");
    }
 
    auto ptr = tatami::compress_sparse_triplets<row_>(NR, NC, values, rows, columns);
    std::vector<StoredIndex_> indices;
    if constexpr(row_) {
        indices.swap(columns);
    } else {
        indices.swap(rows);
    }
 
    return std::shared_ptr<tatami::Matrix<Value_, Index_> >(
        new tatami::CompressedSparseMatrix<Value_, Index_, decltype(values), decltype(indices), decltype(ptr)>(
            NR, NC, std::move(values), std::move(indices), std::move(ptr), row_, false
        )
    );
}
 
template<bool row_, typename Value_, typename Index_, typename StoredValue_, typename StoredIndex_, typename TempIndex_, typename Parser_>
std::shared_ptr<tatami::Matrix<Value_, Index_> > load_sparse_matrix_data(Parser_& parser, eminem::Field field, size_t NR, size_t NC, size_t NL) {
    if constexpr(std::is_same<StoredValue_, Automatic>::value) {
        if (field == eminem::Field::REAL || field == eminem::Field::DOUBLE) {
            return load_sparse_matrix_basic<row_, Value_, Index_, double, StoredIndex_, TempIndex_>(parser, field, NR, NC, NL);
        }
        if (field != eminem::Field::INTEGER) {
            throw std::runtime_error("unsupported Matrix Market field type");
        }
        return load_sparse_matrix_basic<row_, Value_, Index_, int, StoredIndex_, TempIndex_>(parser, field, NR, NC, NL);
    } else {
        return load_sparse_matrix_basic<row_, Value_, Index_, StoredValue_, StoredIndex_, TempIndex_>(parser, field, NR, NC, NL);
    }
}
 
template<bool row_, typename Value_, typename Index_, typename StoredValue_, typename StoredIndex_, typename TempIndex_, typename Parser_>
std::shared_ptr<tatami::Matrix<Value_, Index_> > load_sparse_matrix_index(Parser_& parser, eminem::Field field, size_t NR, size_t NC, size_t NL) {
    if constexpr(std::is_same<StoredIndex_, Automatic>::value) {
        // Automatically choosing a smaller integer type, if it fits.
        constexpr size_t limit8 = std::numeric_limits<uint8_t>::max(), limit16 = std::numeric_limits<uint16_t>::max();
        size_t target = (row_ ? NC : NR);
 
        if (target <= limit8) {
            return load_sparse_matrix_data<row_, Value_, Index_, StoredValue_, uint8_t, TempIndex_>(parser, field, NR, NC, NL);
        } else if (target <= limit16) {
            return load_sparse_matrix_data<row_, Value_, Index_, StoredValue_, uint16_t, TempIndex_>(parser, field, NR, NC, NL);
        } else {
            return load_sparse_matrix_data<row_, Value_, Index_, StoredValue_, uint32_t, TempIndex_>(parser, field, NR, NC, NL);
        }
 
    } else {
        return load_sparse_matrix_data<row_, Value_, Index_, StoredValue_, StoredIndex_, TempIndex_>(parser, field, NR, NC, NL);
    }
}
 
template<bool row_, typename Value_, typename Index_, typename StoredValue_, typename Parser_>
std::shared_ptr<tatami::Matrix<Value_, Index_> > load_dense_matrix_basic(Parser_& parser, eminem::Field field, size_t NR, size_t NC) {
    std::vector<StoredValue_> values;
    if constexpr(row_) {
        values.resize(NR * NC);
    } else {
        values.reserve(NR * NC);
    }
 
    if (field == eminem::Field::INTEGER) {
        parser.scan_integer([&](size_t r, size_t c, int v) -> void {
            if constexpr(row_) {
                values[(r - 1) * NC + (c - 1)] = v;
            } else {
                values.push_back(v); // Matrix Market ARRAY format is already column-major
            }
        });
 
    } else if (field == eminem::Field::REAL || field == eminem::Field::DOUBLE) {
        parser.scan_real([&](size_t r, size_t c, double v) -> void {
            if constexpr(row_) {
                values[(r - 1) * NC + (c - 1)] = v;
            } else {
                values.push_back(v);
            }
        });
 
    } else {
        throw std::runtime_error("unsupported Matrix Market field type");
    }
 
    return std::shared_ptr<tatami::Matrix<Value_, Index_> >(
        new tatami::DenseMatrix<Value_, Index_, decltype(values)>(NR, NC, std::move(values), row_)
    );
}
 
template<bool row_, bool parallel_, typename Value_, typename Index_, typename StoredValue_, typename StoredIndex_>
std::shared_ptr<tatami::Matrix<Value_, Index_> > load_matrix(byteme::Reader& reader) {
    eminem::Parser<parallel_> parser(&reader);
    parser.scan_preamble();
 
    const auto& banner = parser.get_banner();
    auto field = banner.field;
    auto format = banner.format;
    size_t NR = parser.get_nrows(), NC = parser.get_ncols(), NL = parser.get_nlines();
 
    if (format == eminem::Format::COORDINATE) {
        // Automatically choosing a smaller integer type for the temporary index.
        constexpr size_t limit8 = std::numeric_limits<uint8_t>::max(), limit16 = std::numeric_limits<uint16_t>::max();
        auto primary = (row_ ? NR : NC);
 
        if (primary <= limit8) {
            return load_sparse_matrix_index<row_, Value_, Index_, StoredValue_, StoredIndex_, uint8_t>(parser, field, NR, NC, NL);
        } else if (primary <= limit16) {
            return load_sparse_matrix_index<row_, Value_, Index_, StoredValue_, StoredIndex_, uint16_t>(parser, field, NR, NC, NL);
        } else {
            return load_sparse_matrix_index<row_, Value_, Index_, StoredValue_, StoredIndex_, uint32_t>(parser, field, NR, NC, NL);
        }
 
    } else {
        if constexpr(std::is_same<StoredValue_, Automatic>::value) {
            if (field == eminem::Field::REAL || field == eminem::Field::DOUBLE) {
                return load_dense_matrix_basic<row_, Value_, Index_, double>(parser, field, NR, NC);
            }
            if (field != eminem::Field::INTEGER) {
                throw std::runtime_error("unsupported Matrix Market field type");
            }
            return load_dense_matrix_basic<row_, Value_, Index_, int>(parser, field, NR, NC);
 
        } else {
            return load_dense_matrix_basic<row_, Value_, Index_, StoredValue_>(parser, field, NR, NC);
        }
    }
}
 
}
template<typename Value_, typename Index_, typename StoredValue_ = Automatic, typename StoredIndex_ = Automatic>
std::shared_ptr<tatami::Matrix<Value_, Index_> > load_matrix(byteme::Reader& reader, const Options& options) {
    if (options.row) {
        if (options.parallel) {
            return internal::load_matrix<true, true, Value_, Index_, StoredValue_, StoredIndex_>(reader);
        } else {
            return internal::load_matrix<true, false, Value_, Index_, StoredValue_, StoredIndex_>(reader);
        }
    } else {
        if (options.parallel) {
            return internal::load_matrix<false, true, Value_, Index_, StoredValue_, StoredIndex_>(reader);
        } else {
            return internal::load_matrix<false, false, Value_, Index_, StoredValue_, StoredIndex_>(reader);
        }
    }
}
 
template<typename Value_, typename Index_, typename StoredValue_ = Automatic, typename StoredIndex_ = Automatic>
std::shared_ptr<tatami::Matrix<Value_, Index_> > load_matrix_from_text_file(const char* filepath, const Options& options) {
    byteme::RawFileReader reader(filepath, options.buffer_size);
    return load_matrix<Value_, Index_, StoredValue_, StoredIndex_>(reader, options);
}
 
#if __has_include("zlib.h")
 
template<typename Value_, typename Index_, typename StoredValue_ = Automatic, typename StoredIndex_ = Automatic>
std::shared_ptr<tatami::Matrix<Value_, Index_> > load_matrix_from_gzip_file(const char* filepath, const Options& options) {
    byteme::GzipFileReader reader(filepath, options.buffer_size);
    return load_matrix<Value_, Index_, StoredValue_, StoredIndex_>(reader, options);
}
 
template<typename Value_, typename Index_, typename StoredValue_ = Automatic, typename StoredIndex_ = Automatic>
std::shared_ptr<tatami::Matrix<Value_, Index_> > load_matrix_from_some_file(const char* filepath, const Options& options) {
    byteme::SomeFileReader reader(filepath, options.buffer_size);
    return load_matrix<Value_, Index_, StoredValue_, StoredIndex_>(reader, options);
}
 
#endif
 
template<typename Value_, typename Index_, typename StoredValue_ = Automatic, typename StoredIndex_ = Automatic>
std::shared_ptr<tatami::Matrix<Value_, Index_> > load_matrix_from_text_buffer(const unsigned char* buffer, size_t n, const Options& options) {
    byteme::RawBufferReader reader(buffer, n);
    return load_matrix<Value_, Index_, StoredValue_, StoredIndex_>(reader, options);
}
 
#if __has_include("zlib.h")
 
template<typename Value_, typename Index_, typename StoredValue_ = Automatic, typename StoredIndex_ = Automatic>
std::shared_ptr<tatami::Matrix<Value_, Index_> > load_matrix_from_zlib_buffer(const unsigned char* buffer, size_t n, const Options& options) {
    byteme::ZlibBufferReader reader(buffer, n, options.compression, options.buffer_size);
    return load_matrix<Value_, Index_, StoredValue_, StoredIndex_>(reader, options);
}
 
template<typename Value_, typename Index_, typename StoredValue_ = Automatic, typename StoredIndex_ = Automatic>
std::shared_ptr<tatami::Matrix<Value_, Index_> > load_matrix_from_some_buffer(const unsigned char* buffer, size_t n, const Options& options) {
    byteme::SomeBufferReader reader(buffer, n, options.buffer_size);
    return load_matrix<Value_, Index_, StoredValue_, StoredIndex_>(reader, options);
}
 
#endif
 
}
 
#endif