tatami_hdf5
tatami bindings for HDF5-backed matrices
Loading...
Searching...
No Matches
Public Member Functions | List of all members
tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ > Class Template Reference

Dense matrix backed by a DataSet in a HDF5 file. More...

#include <DenseMatrix.hpp>

Inheritance diagram for tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >:
Inheritance graph
[legend]
Collaboration diagram for tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >:
Collaboration graph
[legend]

Public Member Functions

 DenseMatrix (std::string file, std::string name, bool transpose, const DenseMatrixOptions &options)
 
 DenseMatrix (std::string file, std::string name, bool transpose)
 
Index_ nrow () const
 
Index_ ncol () const
 
bool prefer_rows () const
 
double prefer_rows_proportion () const
 
bool uses_oracle (bool) const
 
bool is_sparse () const
 
double is_sparse_proportion () const
 
std::unique_ptr< tatami::MyopicDenseExtractor< Value_, Index_ > > dense (bool row, const tatami::Options &) const
 
std::unique_ptr< tatami::MyopicDenseExtractor< Value_, Index_ > > dense (bool row, Index_ block_start, Index_ block_length, const tatami::Options &) const
 
std::unique_ptr< tatami::MyopicDenseExtractor< Value_, Index_ > > dense (bool row, tatami::VectorPtr< Index_ > indices_ptr, const tatami::Options &) const
 
std::unique_ptr< tatami::MyopicSparseExtractor< Value_, Index_ > > sparse (bool row, const tatami::Options &opt) const
 
std::unique_ptr< tatami::MyopicSparseExtractor< Value_, Index_ > > sparse (bool row, Index_ block_start, Index_ block_length, const tatami::Options &opt) const
 
std::unique_ptr< tatami::MyopicSparseExtractor< Value_, Index_ > > sparse (bool row, tatami::VectorPtr< Index_ > indices_ptr, const tatami::Options &opt) const
 
std::unique_ptr< tatami::OracularDenseExtractor< Value_, Index_ > > dense (bool row, std::shared_ptr< const tatami::Oracle< Index_ > > oracle, const tatami::Options &) const
 
std::unique_ptr< tatami::OracularDenseExtractor< Value_, Index_ > > dense (bool row, std::shared_ptr< const tatami::Oracle< Index_ > > oracle, Index_ block_start, Index_ block_length, const tatami::Options &) const
 
std::unique_ptr< tatami::OracularDenseExtractor< Value_, Index_ > > dense (bool row, std::shared_ptr< const tatami::Oracle< Index_ > > oracle, tatami::VectorPtr< Index_ > indices_ptr, const tatami::Options &) const
 
std::unique_ptr< tatami::OracularSparseExtractor< Value_, Index_ > > sparse (bool row, std::shared_ptr< const tatami::Oracle< Index_ > > oracle, const tatami::Options &opt) const
 
std::unique_ptr< tatami::OracularSparseExtractor< Value_, Index_ > > sparse (bool row, std::shared_ptr< const tatami::Oracle< Index_ > > oracle, Index_ block_start, Index_ block_length, const tatami::Options &opt) const
 
std::unique_ptr< tatami::OracularSparseExtractor< Value_, Index_ > > sparse (bool row, std::shared_ptr< const tatami::Oracle< Index_ > > oracle, tatami::VectorPtr< Index_ > indices_ptr, const tatami::Options &opt) const
 
- Public Member Functions inherited from tatami::Matrix< Value_, Index_ >
std::unique_ptr< MyopicDenseExtractor< Value_, Index_ > > dense_row (const Options &opt) const
 
std::unique_ptr< MyopicDenseExtractor< Value_, Index_ > > dense_row (Index_ block_start, Index_ block_length, const Options &opt) const
 
std::unique_ptr< MyopicDenseExtractor< Value_, Index_ > > dense_row (VectorPtr< Index_ > indices_ptr, const Options &opt) const
 
std::unique_ptr< MyopicDenseExtractor< Value_, Index_ > > dense_row (std::vector< Index_ > indices, const Options &opt) const
 
std::unique_ptr< MyopicDenseExtractor< Value_, Index_ > > dense_row () const
 
std::unique_ptr< MyopicDenseExtractor< Value_, Index_ > > dense_row (Index_ block_start, Index_ block_length) const
 
std::unique_ptr< MyopicDenseExtractor< Value_, Index_ > > dense_row (VectorPtr< Index_ > indices_ptr) const
 
std::unique_ptr< MyopicDenseExtractor< Value_, Index_ > > dense_row (std::vector< Index_ > indices) const
 
std::unique_ptr< OracularDenseExtractor< Value_, Index_ > > dense_row (std::shared_ptr< const Oracle< Index_ > > oracle, const Options &opt) const
 
std::unique_ptr< OracularDenseExtractor< Value_, Index_ > > dense_row (std::shared_ptr< const Oracle< Index_ > > oracle, Index_ block_start, Index_ block_length, const Options &opt) const
 
std::unique_ptr< OracularDenseExtractor< Value_, Index_ > > dense_row (std::shared_ptr< const Oracle< Index_ > > oracle, VectorPtr< Index_ > indices_ptr, const Options &opt) const
 
std::unique_ptr< OracularDenseExtractor< Value_, Index_ > > dense_row (std::shared_ptr< const Oracle< Index_ > > oracle, std::vector< Index_ > indices, const Options &opt) const
 
std::unique_ptr< OracularDenseExtractor< Value_, Index_ > > dense_row (std::shared_ptr< const Oracle< Index_ > > oracle) const
 
std::unique_ptr< OracularDenseExtractor< Value_, Index_ > > dense_row (std::shared_ptr< const Oracle< Index_ > > oracle, Index_ block_start, Index_ block_length) const
 
std::unique_ptr< OracularDenseExtractor< Value_, Index_ > > dense_row (std::shared_ptr< const Oracle< Index_ > > oracle, VectorPtr< Index_ > indices_ptr) const
 
std::unique_ptr< OracularDenseExtractor< Value_, Index_ > > dense_row (std::shared_ptr< const Oracle< Index_ > > oracle, std::vector< Index_ > indices) const
 
std::unique_ptr< MyopicDenseExtractor< Value_, Index_ > > dense_column (const Options &opt) const
 
std::unique_ptr< MyopicDenseExtractor< Value_, Index_ > > dense_column (Index_ block_start, Index_ block_length, const Options &opt) const
 
std::unique_ptr< MyopicDenseExtractor< Value_, Index_ > > dense_column (VectorPtr< Index_ > indices_ptr, const Options &opt) const
 
std::unique_ptr< MyopicDenseExtractor< Value_, Index_ > > dense_column (std::vector< Index_ > indices, const Options &opt) const
 
std::unique_ptr< MyopicDenseExtractor< Value_, Index_ > > dense_column () const
 
std::unique_ptr< MyopicDenseExtractor< Value_, Index_ > > dense_column (Index_ block_start, Index_ block_length) const
 
std::unique_ptr< MyopicDenseExtractor< Value_, Index_ > > dense_column (VectorPtr< Index_ > indices_ptr) const
 
std::unique_ptr< MyopicDenseExtractor< Value_, Index_ > > dense_column (std::vector< Index_ > indices) const
 
std::unique_ptr< OracularDenseExtractor< Value_, Index_ > > dense_column (std::shared_ptr< const Oracle< Index_ > > oracle, const Options &opt) const
 
std::unique_ptr< OracularDenseExtractor< Value_, Index_ > > dense_column (std::shared_ptr< const Oracle< Index_ > > oracle, Index_ block_start, Index_ block_length, const Options &opt) const
 
std::unique_ptr< OracularDenseExtractor< Value_, Index_ > > dense_column (std::shared_ptr< const Oracle< Index_ > > oracle, VectorPtr< Index_ > indices_ptr, const Options &opt) const
 
std::unique_ptr< OracularDenseExtractor< Value_, Index_ > > dense_column (std::shared_ptr< const Oracle< Index_ > > oracle, std::vector< Index_ > indices, const Options &opt) const
 
std::unique_ptr< OracularDenseExtractor< Value_, Index_ > > dense_column (std::shared_ptr< const Oracle< Index_ > > oracle) const
 
std::unique_ptr< OracularDenseExtractor< Value_, Index_ > > dense_column (std::shared_ptr< const Oracle< Index_ > > oracle, Index_ block_start, Index_ block_length) const
 
std::unique_ptr< OracularDenseExtractor< Value_, Index_ > > dense_column (std::shared_ptr< const Oracle< Index_ > > oracle, VectorPtr< Index_ > indices_ptr) const
 
std::unique_ptr< OracularDenseExtractor< Value_, Index_ > > dense_column (std::shared_ptr< const Oracle< Index_ > > oracle, std::vector< Index_ > indices) const
 
std::unique_ptr< MyopicSparseExtractor< Value_, Index_ > > sparse_row (const Options &opt) const
 
std::unique_ptr< MyopicSparseExtractor< Value_, Index_ > > sparse_row (Index_ block_start, Index_ block_length, const Options &opt) const
 
std::unique_ptr< MyopicSparseExtractor< Value_, Index_ > > sparse_row (VectorPtr< Index_ > indices_ptr, const Options &opt) const
 
std::unique_ptr< MyopicSparseExtractor< Value_, Index_ > > sparse_row (std::vector< Index_ > indices, const Options &opt) const
 
std::unique_ptr< MyopicSparseExtractor< Value_, Index_ > > sparse_row () const
 
std::unique_ptr< MyopicSparseExtractor< Value_, Index_ > > sparse_row (Index_ block_start, Index_ block_length) const
 
std::unique_ptr< MyopicSparseExtractor< Value_, Index_ > > sparse_row (VectorPtr< Index_ > indices_ptr) const
 
std::unique_ptr< MyopicSparseExtractor< Value_, Index_ > > sparse_row (std::vector< Index_ > indices) const
 
std::unique_ptr< OracularSparseExtractor< Value_, Index_ > > sparse_row (std::shared_ptr< const Oracle< Index_ > > oracle, const Options &opt) const
 
std::unique_ptr< OracularSparseExtractor< Value_, Index_ > > sparse_row (std::shared_ptr< const Oracle< Index_ > > oracle, Index_ block_start, Index_ block_length, const Options &opt) const
 
std::unique_ptr< OracularSparseExtractor< Value_, Index_ > > sparse_row (std::shared_ptr< const Oracle< Index_ > > oracle, VectorPtr< Index_ > indices_ptr, const Options &opt) const
 
std::unique_ptr< OracularSparseExtractor< Value_, Index_ > > sparse_row (std::shared_ptr< const Oracle< Index_ > > oracle, std::vector< Index_ > indices, const Options &opt) const
 
std::unique_ptr< OracularSparseExtractor< Value_, Index_ > > sparse_row (std::shared_ptr< const Oracle< Index_ > > oracle) const
 
std::unique_ptr< OracularSparseExtractor< Value_, Index_ > > sparse_row (std::shared_ptr< const Oracle< Index_ > > oracle, Index_ block_start, Index_ block_length) const
 
std::unique_ptr< OracularSparseExtractor< Value_, Index_ > > sparse_row (std::shared_ptr< const Oracle< Index_ > > oracle, VectorPtr< Index_ > indices_ptr) const
 
std::unique_ptr< OracularSparseExtractor< Value_, Index_ > > sparse_row (std::shared_ptr< const Oracle< Index_ > > oracle, std::vector< Index_ > indices) const
 
std::unique_ptr< MyopicSparseExtractor< Value_, Index_ > > sparse_column (const Options &opt) const
 
std::unique_ptr< MyopicSparseExtractor< Value_, Index_ > > sparse_column (Index_ block_start, Index_ block_length, const Options &opt) const
 
std::unique_ptr< MyopicSparseExtractor< Value_, Index_ > > sparse_column (VectorPtr< Index_ > indices_ptr, const Options &opt) const
 
std::unique_ptr< MyopicSparseExtractor< Value_, Index_ > > sparse_column (std::vector< Index_ > indices, const Options &opt) const
 
std::unique_ptr< MyopicSparseExtractor< Value_, Index_ > > sparse_column () const
 
std::unique_ptr< MyopicSparseExtractor< Value_, Index_ > > sparse_column (Index_ block_start, Index_ block_length) const
 
std::unique_ptr< MyopicSparseExtractor< Value_, Index_ > > sparse_column (VectorPtr< Index_ > indices_ptr) const
 
std::unique_ptr< MyopicSparseExtractor< Value_, Index_ > > sparse_column (std::vector< Index_ > indices) const
 
std::unique_ptr< OracularSparseExtractor< Value_, Index_ > > sparse_column (std::shared_ptr< const Oracle< Index_ > > oracle, const Options &opt) const
 
std::unique_ptr< OracularSparseExtractor< Value_, Index_ > > sparse_column (std::shared_ptr< const Oracle< Index_ > > oracle, Index_ block_start, Index_ block_length, const Options &opt) const
 
std::unique_ptr< OracularSparseExtractor< Value_, Index_ > > sparse_column (std::shared_ptr< const Oracle< Index_ > > oracle, VectorPtr< Index_ > indices_ptr, const Options &opt) const
 
std::unique_ptr< OracularSparseExtractor< Value_, Index_ > > sparse_column (std::shared_ptr< const Oracle< Index_ > > oracle, std::vector< Index_ > indices, const Options &opt) const
 
std::unique_ptr< OracularSparseExtractor< Value_, Index_ > > sparse_column (std::shared_ptr< const Oracle< Index_ > > oracle) const
 
std::unique_ptr< OracularSparseExtractor< Value_, Index_ > > sparse_column (std::shared_ptr< const Oracle< Index_ > > oracle, Index_ block_start, Index_ block_length) const
 
std::unique_ptr< OracularSparseExtractor< Value_, Index_ > > sparse_column (std::shared_ptr< const Oracle< Index_ > > oracle, VectorPtr< Index_ > indices_ptr) const
 
std::unique_ptr< OracularSparseExtractor< Value_, Index_ > > sparse_column (std::shared_ptr< const Oracle< Index_ > > oracle, std::vector< Index_ > indices) const
 

Additional Inherited Members

- Public Types inherited from tatami::Matrix< Value_, Index_ >
typedef Value_ value_type
 
typedef Index_ index_type
 

Detailed Description

template<typename Value_, typename Index_, typename CachedValue_ = Value_>
class tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >

Dense matrix backed by a DataSet in a HDF5 file.

This class retrieves data from the HDF5 file on demand rather than loading it all in at the start. This allows us to handle very large datasets in limited memory at the cost of some speed.

We manually handle the chunk caching to speed up access for consecutive rows and columns. The policy is to minimize the number of calls to the HDF5 library by requesting large contiguous slices where possible, where each slice typically consists of multiple rows/columns that belong in the same HDF5 chunk. The size of the slice is determined by the options in the constructor.

Callers should follow the prefer_rows() suggestion when extracting data, as this tries to minimize the number of chunks that need to be read per access request. If they do not, the access pattern on disk may be slightly to highly suboptimal, depending on the chunk dimensions.

As the HDF5 library is not generally thread-safe, the HDF5-related operations should only be run in a single thread. This is normally handled automatically but developers can check out serialize() to customize the locking scheme.

Template Parameters
Value_Type of the matrix values.
Index_Type of the row/column indices.
CachedValue_Type of the matrix value to store in the cache. This can be set to a narrower type than Value_ to save memory and improve cache performance, if a smaller type is known to be able to store the values (based on their HDF5 type or other knowledge).

Constructor & Destructor Documentation

◆ DenseMatrix() [1/2]

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::DenseMatrix ( std::string  file,
std::string  name,
bool  transpose,
const DenseMatrixOptions options 
)
inline
Parameters
filePath to the file.
namePath to the dataset inside the file.
transposeWhether the dataset is transposed in its storage order, i.e., rows in the HDF5 dataset correspond to columns of this matrix. This may be true for HDF5 files generated by frameworks that use column-major matrices, where preserving the data layout between memory and disk is more efficient (see, e.g., the rhdf5 Bioconductor package).
optionsFurther options for data extraction.

◆ DenseMatrix() [2/2]

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::DenseMatrix ( std::string  file,
std::string  name,
bool  transpose 
)
inline

Overload that uses the default DenseMatrixOptions.

Parameters
filePath to the file.
namePath to the dataset inside the file.
transposeWhether the dataset is transposed in its storage order.

Member Function Documentation

◆ nrow()

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
Index_ tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::nrow ( ) const
inlinevirtual

◆ ncol()

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
Index_ tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::ncol ( ) const
inlinevirtual

◆ prefer_rows()

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
bool tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::prefer_rows ( ) const
inlinevirtual
Returns
Boolean indicating whether to prefer row extraction.

We favor extraction on the first dimension (rows by default, columns when transpose = true) as this matches the HDF5 storage order. However, for some chunking scheme and cache_limit, this might require repeated reads from file; in such cases, we switch to extraction on the second dimension.

Implements tatami::Matrix< Value_, Index_ >.

◆ prefer_rows_proportion()

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
double tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::prefer_rows_proportion ( ) const
inlinevirtual

◆ uses_oracle()

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
bool tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::uses_oracle ( bool  ) const
inlinevirtual

◆ is_sparse()

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
bool tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::is_sparse ( ) const
inlinevirtual

◆ is_sparse_proportion()

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
double tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::is_sparse_proportion ( ) const
inlinevirtual

◆ dense() [1/6]

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
std::unique_ptr< tatami::MyopicDenseExtractor< Value_, Index_ > > tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::dense ( bool  row,
const tatami::Options  
) const
inlinevirtual

◆ dense() [2/6]

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
std::unique_ptr< tatami::MyopicDenseExtractor< Value_, Index_ > > tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::dense ( bool  row,
Index_  block_start,
Index_  block_length,
const tatami::Options  
) const
inlinevirtual

◆ dense() [3/6]

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
std::unique_ptr< tatami::MyopicDenseExtractor< Value_, Index_ > > tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::dense ( bool  row,
tatami::VectorPtr< Index_ >  indices_ptr,
const tatami::Options  
) const
inlinevirtual

◆ sparse() [1/6]

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
std::unique_ptr< tatami::MyopicSparseExtractor< Value_, Index_ > > tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::sparse ( bool  row,
const tatami::Options opt 
) const
inlinevirtual

◆ sparse() [2/6]

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
std::unique_ptr< tatami::MyopicSparseExtractor< Value_, Index_ > > tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::sparse ( bool  row,
Index_  block_start,
Index_  block_length,
const tatami::Options opt 
) const
inlinevirtual

◆ sparse() [3/6]

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
std::unique_ptr< tatami::MyopicSparseExtractor< Value_, Index_ > > tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::sparse ( bool  row,
tatami::VectorPtr< Index_ >  indices_ptr,
const tatami::Options opt 
) const
inlinevirtual

◆ dense() [4/6]

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
std::unique_ptr< tatami::OracularDenseExtractor< Value_, Index_ > > tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::dense ( bool  row,
std::shared_ptr< const tatami::Oracle< Index_ > >  oracle,
const tatami::Options  
) const
inlinevirtual

◆ dense() [5/6]

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
std::unique_ptr< tatami::OracularDenseExtractor< Value_, Index_ > > tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::dense ( bool  row,
std::shared_ptr< const tatami::Oracle< Index_ > >  oracle,
Index_  block_start,
Index_  block_length,
const tatami::Options  
) const
inlinevirtual

◆ dense() [6/6]

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
std::unique_ptr< tatami::OracularDenseExtractor< Value_, Index_ > > tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::dense ( bool  row,
std::shared_ptr< const tatami::Oracle< Index_ > >  oracle,
tatami::VectorPtr< Index_ >  indices_ptr,
const tatami::Options  
) const
inlinevirtual

◆ sparse() [4/6]

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
std::unique_ptr< tatami::OracularSparseExtractor< Value_, Index_ > > tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::sparse ( bool  row,
std::shared_ptr< const tatami::Oracle< Index_ > >  oracle,
const tatami::Options opt 
) const
inlinevirtual

◆ sparse() [5/6]

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
std::unique_ptr< tatami::OracularSparseExtractor< Value_, Index_ > > tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::sparse ( bool  row,
std::shared_ptr< const tatami::Oracle< Index_ > >  oracle,
Index_  block_start,
Index_  block_length,
const tatami::Options opt 
) const
inlinevirtual

◆ sparse() [6/6]

template<typename Value_ , typename Index_ , typename CachedValue_ = Value_>
std::unique_ptr< tatami::OracularSparseExtractor< Value_, Index_ > > tatami_hdf5::DenseMatrix< Value_, Index_, CachedValue_ >::sparse ( bool  row,
std::shared_ptr< const tatami::Oracle< Index_ > >  oracle,
tatami::VectorPtr< Index_ >  indices_ptr,
const tatami::Options opt 
) const
inlinevirtual

The documentation for this class was generated from the following file: