tatami_hdf5
tatami bindings for HDF5-backed matrices
Loading...
Searching...
No Matches
write_compressed_sparse_matrix.hpp
Go to the documentation of this file.
1#ifndef TATAMI_WRITE_SPARSE_MATRIX_TO_HDF5_HPP
2#define TATAMI_WRITE_SPARSE_MATRIX_TO_HDF5_HPP
3
4#include "tatami/tatami.hpp"
5#include "utils.hpp"
6
7#include "H5Cpp.h"
8
9#include <cstdint>
10#include <string>
11#include <vector>
12#include <cmath>
13#include <limits>
14#include <optional>
15
21namespace tatami_hdf5 {
22
31 std::optional<std::string> data_name;
32
37 std::optional<std::string> index_name;
38
43 std::optional<std::string> ptr_name;
44
49 std::optional<WriteStorageLayout> columnar;
50
55 std::optional<WriteStorageType> data_type;
56
63 bool force_integer = false;
64
69 std::optional<WriteStorageType> index_type;
70
75 std::optional<WriteStorageType> ptr_type;
76
83
88 hsize_t chunk_size = sanisizer::cap<hsize_t>(10000);
89
95 bool shuffle = true;
96
102 bool two_pass = false;
103
108 int num_threads = 1;
109};
110
114inline H5::DataSet create_1d_compressed_hdf5_dataset(H5::Group& location, WriteStorageType type, const std::string& name, hsize_t length, int deflate_level, hsize_t chunk, bool shuffle) {
115 H5::DataSpace dspace(1, &length);
116 H5::DSetCreatPropList plist;
117
118 if (deflate_level >= 0 && length) {
119 plist.setDeflate(deflate_level);
120 if (chunk > length) {
121 plist.setChunk(1, &length);
122 } else {
123 plist.setChunk(1, &chunk);
124 }
125 if (shuffle) {
126 plist.setShuffle();
127 }
128 }
129
130 const auto dtype = choose_pred_type(type);
131 return location.createDataSet(name, *dtype, dspace, plist);
132}
133
134template<typename Type_>
135bool does_non_negative_integer_fit(const WriteStorageType type, const Type_ x) {
136 static_assert(std::is_integral<Type_>::value);
137
138 bool okay = false;
139 switch (type) {
140 case WriteStorageType::INT8:
141 okay = fits_upper_limit<std::int8_t>(x);
142 break;
143 case WriteStorageType::UINT8:
144 okay = fits_upper_limit<std::uint8_t>(x);
145 break;
146 case WriteStorageType::INT16:
147 okay = fits_upper_limit<std::int16_t >(x);
148 break;
149 case WriteStorageType::UINT16:
150 okay = fits_upper_limit<std::uint16_t>(x);
151 break;
152 case WriteStorageType::INT32:
153 okay = fits_upper_limit<std::int32_t >(x);
154 break;
155 case WriteStorageType::UINT32:
156 okay = fits_upper_limit<std::uint32_t>(x);
157 break;
158 case WriteStorageType::INT64:
159 okay = fits_upper_limit<std::int64_t >(x);
160 break;
161 case WriteStorageType::UINT64:
162 okay = fits_upper_limit<std::uint64_t>(x);
163 break;
164 default:
165 // okay remains false, as the x must be integer.
166 break;
167 }
168 return okay;
169}
170
171template<typename Index_>
172WriteStorageType choose_index_type(const std::optional<WriteStorageType>& index_type, Index_ upper_index) {
173 static_assert(std::is_integral<Index_>::value);
174
175 if (!index_type.has_value()) {
176 if (fits_upper_limit<std::uint8_t>(upper_index)) {
177 return WriteStorageType::UINT8;
178 } else if (fits_upper_limit<std::uint16_t>(upper_index)) {
179 return WriteStorageType::UINT16;
180 } else if (fits_upper_limit<std::uint32_t>(upper_index)) {
181 return WriteStorageType::UINT32;
182 } else if (fits_upper_limit<std::uint64_t>(upper_index)) {
183 return WriteStorageType::UINT64;
184 }
185 throw std::runtime_error("no type can store the largest index");
186 }
187
188 const auto itype = *index_type;
189 if (!does_non_negative_integer_fit(itype, upper_index)) {
190 throw std::runtime_error("specified type cannot store the largest index");
191 }
192
193 return itype;
194}
195
196inline WriteStorageType choose_ptr_type(const std::optional<WriteStorageType>& ptr_type, hsize_t nnzero) {
197 if (!ptr_type.has_value()) {
198 if (fits_upper_limit<std::uint32_t>(nnzero)) {
199 return WriteStorageType::UINT32;
200 } else if (fits_upper_limit<std::uint64_t>(nnzero)) {
201 return WriteStorageType::UINT64;
202 }
203
204 throw std::runtime_error("no type can store the number of non-zero elements");
205 }
206
207 const auto ptype = *ptr_type;
208 if (!does_non_negative_integer_fit(ptype, nnzero)) {
209 throw std::runtime_error("specified type cannot store the number of non-zero elements");
210 }
211
212 return ptype;
213}
214
215template<typename Value_, typename Index_>
216struct WriteSparseHdf5Statistics {
217 Value_ lower_data = 0;
218 Value_ upper_data = 0;
219 Index_ upper_index = 0;
220 hsize_t non_zeros = 0;
221 bool has_decimal = false;
222 bool has_nonfinite = false;
223
224 void add_value(Value_ val) {
225 if constexpr(!std::is_integral<Value_>::value) {
226 if (std::trunc(val) != val) {
227 has_decimal = true;
228 }
229 if (!std::isfinite(val)) {
230 has_nonfinite = true;
231 }
232 }
233
234 if (val < lower_data) {
235 lower_data = val;
236 } else if (val > upper_data) {
237 upper_data = val;
238 }
239 }
240
241 void add_index(Index_ idx) {
242 if (idx > upper_index) {
243 upper_index = idx;
244 }
245 }
246};
247
248template<typename Value_, typename Index_>
249void update_hdf5_stats(const tatami::SparseRange<Value_, Index_>& extracted, WriteSparseHdf5Statistics<Value_, Index_>& output) {
250 // We need to protect the addition just in case it overflows from having too many non-zero elements.
251 output.non_zeros = sanisizer::sum<hsize_t>(output.non_zeros, extracted.number);
252
253 for (Index_ i = 0; i < extracted.number; ++i) {
254 output.add_value(extracted.value[i]);
255 }
256
257 for (Index_ i = 0; i < extracted.number; ++i) {
258 output.add_index(extracted.index[i]);
259 }
260}
261
262template<typename Value_, typename Index_>
263void update_hdf5_stats(const Value_* extracted, Index_ n, WriteSparseHdf5Statistics<Value_, Index_>& output) {
264 Index_ local_nonzero = 0;
265 for (Index_ i = 0; i < n; ++i) {
266 auto val = extracted[i];
267 if (val == 0) {
268 continue;
269 }
270 ++local_nonzero;
271 output.add_value(val);
272 output.add_index(i);
273 }
274
275 // Checking that there aren't overflows, but doing so outside of the hot loop for perf.
276 output.non_zeros = sanisizer::sum<hsize_t>(output.non_zeros, local_nonzero);
277}
278
279template<typename Value_, typename Index_>
280WriteSparseHdf5Statistics<Value_, Index_> write_sparse_hdf5_statistics(const tatami::Matrix<Value_, Index_>& mat, int nthreads) {
281 const auto NR = mat.nrow(), NC = mat.ncol();
282
283 WriteSparseHdf5Statistics<Value_, Index_> output;
284 auto collected = sanisizer::create<std::vector<WriteSparseHdf5Statistics<Value_, Index_> > >(nthreads - 1); // nthreads had better be >= 1.
285 int num_used;
286
287 if (mat.sparse()) {
288 if (mat.prefer_rows()) {
289 num_used = tatami::parallelize([&](int t, Index_ start, Index_ len) -> void {
290 WriteSparseHdf5Statistics<Value_, Index_> current_output;
291
292 auto wrk = tatami::consecutive_extractor<true>(mat, true, start, len);
293 std::vector<Value_> xbuffer(NC);
294 std::vector<Index_> ibuffer(NC);
295 for (Index_ r = start, end = start + len; r < end; ++r) {
296 auto extracted = wrk->fetch(r, xbuffer.data(), ibuffer.data());
297 update_hdf5_stats(extracted, current_output);
298 }
299
300 // Only move to the result buffer at the end, to avoid false sharing between threads.
301 (t ? collected[t - 1] : output) = std::move(current_output);
302 }, NR, nthreads);
303
304 } else {
305 num_used = tatami::parallelize([&](int t, Index_ start, Index_ len) -> void {
306 WriteSparseHdf5Statistics<Value_, Index_> current_output;
307
308 auto wrk = tatami::consecutive_extractor<true>(mat, false, start, len);
309 std::vector<Value_> xbuffer(NR);
310 std::vector<Index_> ibuffer(NR);
311 for (Index_ c = start, end = start + len; c < end; ++c) {
312 auto extracted = wrk->fetch(c, xbuffer.data(), ibuffer.data());
313 update_hdf5_stats(extracted, current_output);
314 }
315
316 // Only move to the result buffer at the end, to avoid false sharing between threads.
317 (t ? collected[t - 1] : output) = std::move(current_output);
318 }, NC, nthreads);
319 }
320
321 } else {
322 if (mat.prefer_rows()) {
323 num_used = tatami::parallelize([&](int t, Index_ start, Index_ len) -> void {
324 WriteSparseHdf5Statistics<Value_, Index_> current_output;
325
326 auto wrk = tatami::consecutive_extractor<false>(mat, true, start, len);
327 std::vector<Value_> xbuffer(NC);
328 for (Index_ r = start, end = start + len; r < end; ++r) {
329 auto extracted = wrk->fetch(r, xbuffer.data());
330 update_hdf5_stats(extracted, NC, current_output);
331 }
332
333 // Only move to the result buffer at the end, to avoid false sharing between threads.
334 (t ? collected[t - 1] : output) = std::move(current_output);
335 }, NR, nthreads);
336
337 } else {
338 num_used = tatami::parallelize([&](int t, Index_ start, Index_ len) -> void {
339 WriteSparseHdf5Statistics<Value_, Index_> current_output;
340
341 auto wrk = tatami::consecutive_extractor<false>(mat, false, start, len);
342 std::vector<Value_> xbuffer(NR);
343 for (Index_ c = start, end = start + len; c < end; ++c) {
344 auto extracted = wrk->fetch(c, xbuffer.data());
345 update_hdf5_stats(extracted, NR, current_output);
346 }
347
348 // Only move to the result buffer at the end, to avoid false sharing between threads.
349 (t ? collected[t - 1] : output) = std::move(current_output);
350 }, NC, nthreads);
351 }
352 }
353
354 for (int i = 1; i < num_used; ++i) {
355 auto& current = collected[i - 1];
356 output.lower_data = std::min(output.lower_data, current.lower_data);
357 output.upper_data = std::max(output.upper_data, current.upper_data);
358 output.upper_index = std::max(output.upper_index, current.upper_index);
359 output.non_zeros = sanisizer::sum<hsize_t>(output.non_zeros, current.non_zeros);
360 output.has_decimal = output.has_decimal || current.has_decimal;
361 output.has_nonfinite = output.has_nonfinite || current.has_nonfinite;
362 }
363
364 return output;
365}
366
367template<typename Value_, typename Index_>
368void write_compressed_sparse_matrix_two_pass(
370 H5::Group& location,
371 const WriteStorageLayout layout,
372 const std::string& data_name,
373 const std::string& index_name,
374 const std::string& ptr_name,
375 const WriteCompressedSparseMatrixOptions& params
376) {
377 auto stats = write_sparse_hdf5_statistics(mat, params.num_threads);
378 const auto data_type = choose_data_type(params.data_type, stats.lower_data, stats.upper_data, stats.has_decimal, params.force_integer, stats.has_nonfinite);
379 const auto index_type = choose_index_type(params.index_type, stats.upper_index);
380
381 // And then saving it. This time we have no choice but to iterate by the desired dimension.
382 const auto non_zeros = stats.non_zeros;
383 H5::DataSet data_ds = create_1d_compressed_hdf5_dataset(location, data_type, data_name, non_zeros, params.deflate_level, params.chunk_size, params.shuffle);
384 H5::DataSet index_ds = create_1d_compressed_hdf5_dataset(location, index_type, index_name, non_zeros, params.deflate_level, params.chunk_size, params.shuffle);
385 hsize_t offset = 0;
386 H5::DataSpace inspace(1, &non_zeros);
387 H5::DataSpace outspace(1, &non_zeros);
388 const auto& dstype = define_mem_type<Value_>();
389 const auto& ixtype = define_mem_type<Index_>();
390
391 const Index_ NR = mat.nrow(), NC = mat.ncol();
392 std::vector<hsize_t> ptrs;
393
394 auto fill_datasets = [&](const Value_* vptr, const Index_* iptr, hsize_t count) -> void {
395 if (count) {
396 inspace.setExtentSimple(1, &count);
397 outspace.selectHyperslab(H5S_SELECT_SET, &count, &offset);
398 data_ds.write(vptr, dstype, inspace, outspace);
399 index_ds.write(iptr, ixtype, inspace, outspace);
400 offset += count; // sum is safe as we already know that the number of non-zeros fits in a hsize_t.
401 }
402 };
403
404 if (mat.sparse()) {
405 if (layout == WriteStorageLayout::ROW) {
406 ptrs.resize(sanisizer::sum<decltype(ptrs.size())>(NR, 1));
409
410 auto wrk = tatami::consecutive_extractor<true>(mat, true, static_cast<Index_>(0), NR);
411 for (Index_ r = 0; r < NR; ++r) {
412 auto extracted = wrk->fetch(r, xbuffer.data(), ibuffer.data());
413 fill_datasets(extracted.value, extracted.index, extracted.number);
414 ptrs[r + 1] = offset;
415 }
416
417 } else {
418 ptrs.resize(sanisizer::sum<decltype(ptrs.size())>(NC, 1));
421
422 auto wrk = tatami::consecutive_extractor<true>(mat, false, static_cast<Index_>(0), NC);
423 for (Index_ c = 0; c < NC; ++c) {
424 auto extracted = wrk->fetch(c, xbuffer.data(), ibuffer.data());
425 fill_datasets(extracted.value, extracted.index, extracted.number);
426 ptrs[c + 1] = offset;
427 }
428 }
429
430 } else {
431 std::vector<Value_> sparse_xbuffer;
432 std::vector<Index_> sparse_ibuffer;
433 auto fill_datasets_from_dense = [&](const Value_* extracted, Index_ n) -> void {
434 sparse_xbuffer.clear();
435 sparse_ibuffer.clear();
436 for (Index_ i = 0; i < n; ++i) {
437 if (extracted[i]) {
438 sparse_xbuffer.push_back(extracted[i]);
439 sparse_ibuffer.push_back(i);
440 }
441 }
442
443 hsize_t count = sparse_xbuffer.size();
444 fill_datasets(sparse_xbuffer.data(), sparse_ibuffer.data(), count);
445 };
446
447 if (layout == WriteStorageLayout::ROW) {
448 ptrs.resize(sanisizer::sum<decltype(ptrs.size())>(NR, 1));
450 auto wrk = tatami::consecutive_extractor<false>(mat, true, static_cast<Index_>(0), NR);
451 for (Index_ r = 0; r < NR; ++r) {
452 auto extracted = wrk->fetch(r, dbuffer.data());
453 fill_datasets_from_dense(extracted, NC);
454 ptrs[r + 1] = offset;
455 }
456
457 } else {
458 ptrs.resize(sanisizer::sum<decltype(ptrs.size())>(NC, 1));
460 auto wrk = tatami::consecutive_extractor<false>(mat, false, static_cast<Index_>(0), NC);
461 for (Index_ c = 0; c < NC; ++c) {
462 auto extracted = wrk->fetch(c, dbuffer.data());
463 fill_datasets_from_dense(extracted, NR);
464 ptrs[c + 1] = offset;
465 }
466 }
467 }
468
469 // Saving the pointers.
470 auto ptr_len = sanisizer::cast<hsize_t>(ptrs.size());
471 H5::DataSet ptr_ds = create_1d_compressed_hdf5_dataset(
472 location,
473 choose_ptr_type(params.ptr_type, ptrs.back()),
474 ptr_name,
475 ptr_len,
476 params.deflate_level,
477 params.chunk_size,
478 params.shuffle
479 );
480 H5::DataSpace ptr_space(1, &ptr_len);
481 ptr_ds.write(ptrs.data(), H5::PredType::NATIVE_HSIZE, ptr_space);
482
483 return;
484}
485
486inline H5::DataSet create_1d_compressed_hdf5_dataset(H5::Group& location, WriteStorageType type, const std::string& name, int deflate_level, hsize_t chunk, bool shuffle) {
487 const hsize_t length = 0;
488 constexpr auto copy = H5S_UNLIMITED; // can't directly take an address to this, guess it's a macro.
489 H5::DataSpace dspace(1, &length, &copy);
490 H5::DSetCreatPropList plist;
491
492 // Extensible datasets must be chunked.
493 if (deflate_level == 0) {
494 throw std::runtime_error("'deflate_level' must be positive if 'two_pass = false'");
495 }
496
497 plist.setDeflate(deflate_level);
498 plist.setChunk(1, &chunk);
499 if (shuffle) {
500 plist.setShuffle();
501 }
502
503 const auto dtype = choose_pred_type(type);
504 return location.createDataSet(name, *dtype, dspace, plist);
505}
506
507template<typename Value_, typename Index_>
508void write_compressed_sparse_matrix_one_pass(
510 H5::Group& location,
511 const WriteStorageLayout layout,
512 const std::string& data_name,
513 const std::string& index_name,
514 const std::string& ptr_name,
515 const WriteCompressedSparseMatrixOptions& params
516){
517 const auto requested_dtype = *(params.data_type);
518 const auto requested_itype = *(params.index_type);
519 H5::DataSet data_ds = create_1d_compressed_hdf5_dataset(location, requested_dtype, data_name, params.deflate_level, params.chunk_size, params.shuffle);
520 H5::DataSet index_ds = create_1d_compressed_hdf5_dataset(location, requested_itype, index_name, params.deflate_level, params.chunk_size, params.shuffle);
521
522 hsize_t offset = 0;
523 H5::DataSpace outspace;
524 const auto& dstype = define_mem_type<Value_>();
525 const auto& ixtype = define_mem_type<Index_>();
526
527 const Index_ NR = mat.nrow(), NC = mat.ncol();
528 std::vector<hsize_t> ptrs;
529
530 auto fill_datasets = [&](const Value_* vptr, const Index_* iptr, hsize_t count, H5::DataSpace& inspace) -> void {
531 if (count) {
532 // We need to check this because we don't know that the number of non-zeros fits in a hsize_t.
533 const hsize_t new_size = sanisizer::sum<hsize_t>(offset, count);
534 data_ds.extend(&new_size);
535 index_ds.extend(&new_size);
536
537 constexpr hsize_t zero = 0;
538 inspace.selectHyperslab(H5S_SELECT_SET, &count, &zero);
539 outspace.setExtentSimple(1, &new_size);
540 outspace.selectHyperslab(H5S_SELECT_SET, &count, &offset);
541
542 data_ds.write(vptr, dstype, inspace, outspace);
543 index_ds.write(iptr, ixtype, inspace, outspace);
544 offset = new_size;
545 }
546 };
547
548 if (mat.sparse()) {
549 auto fill_datasets_from_sparse = [&](const Value_* vptr, const Index_* iptr, Index_ n, H5::DataSpace& inspace) -> void {
550 for (Index_ i = 0; i < n; ++i) {
551 check_data_value_fit(requested_dtype, vptr[i]);
552 if (!does_non_negative_integer_fit(requested_itype, iptr[i])) {
553 throw std::runtime_error("specified type cannot store the largest index");
554 }
555 }
556 // We need to check this because we don't even know that the dimension extent fits in a hsize_t.
557 const auto count = sanisizer::cast<hsize_t>(n);
558 fill_datasets(vptr, iptr, count, inspace);
559 };
560
561 if (layout == WriteStorageLayout::ROW) {
562 ptrs.resize(sanisizer::sum<decltype(ptrs.size())>(NR, 1));
565 const hsize_t extent = NC;
566 H5::DataSpace inspace(1, &extent);
567
568 auto wrk = tatami::consecutive_extractor<true>(mat, true, static_cast<Index_>(0), NR);
569 for (Index_ r = 0; r < NR; ++r) {
570 auto extracted = wrk->fetch(r, xbuffer.data(), ibuffer.data());
571 fill_datasets_from_sparse(extracted.value, extracted.index, extracted.number, inspace);
572 ptrs[r + 1] = offset;
573 }
574
575 } else {
576 ptrs.resize(sanisizer::sum<decltype(ptrs.size())>(NC, 1));
579 const hsize_t extent = NR;
580 H5::DataSpace inspace(1, &extent);
581
582 auto wrk = tatami::consecutive_extractor<true>(mat, false, static_cast<Index_>(0), NC);
583 for (Index_ c = 0; c < NC; ++c) {
584 auto extracted = wrk->fetch(c, xbuffer.data(), ibuffer.data());
585 fill_datasets_from_sparse(extracted.value, extracted.index, extracted.number, inspace);
586 ptrs[c + 1] = offset;
587 }
588 }
589
590 } else {
591 std::vector<Value_> sparse_xbuffer;
592 std::vector<Index_> sparse_ibuffer;
593 auto fill_datasets_from_dense = [&](const Value_* extracted, Index_ n, H5::DataSpace& inspace) -> void {
594 sparse_xbuffer.clear();
595 sparse_ibuffer.clear();
596 for (Index_ i = 0; i < n; ++i) {
597 if (extracted[i]) {
598 check_data_value_fit(requested_dtype, extracted[i]);
599 sparse_xbuffer.push_back(extracted[i]);
600 if (!does_non_negative_integer_fit(requested_itype, i)) {
601 throw std::runtime_error("specified type cannot store the largest index");
602 }
603 sparse_ibuffer.push_back(i);
604 }
605 }
606
607 const auto count = sanisizer::cast<hsize_t>(sparse_xbuffer.size());
608 fill_datasets(sparse_xbuffer.data(), sparse_ibuffer.data(), count, inspace);
609 };
610
611 if (layout == WriteStorageLayout::ROW) {
612 ptrs.resize(sanisizer::sum<decltype(ptrs.size())>(NR, 1));
614 const hsize_t extent = NC;
615 H5::DataSpace inspace(1, &extent);
616
617 auto wrk = tatami::consecutive_extractor<false>(mat, true, static_cast<Index_>(0), NR);
618 for (Index_ r = 0; r < NR; ++r) {
619 auto extracted = wrk->fetch(r, dbuffer.data());
620 fill_datasets_from_dense(extracted, NC, inspace);
621 ptrs[r + 1] = offset;
622 }
623
624 } else {
625 ptrs.resize(sanisizer::sum<decltype(ptrs.size())>(NC, 1));
627 const hsize_t extent = NR;
628 H5::DataSpace inspace(1, &extent);
629
630 auto wrk = tatami::consecutive_extractor<false>(mat, false, static_cast<Index_>(0), NC);
631 for (Index_ c = 0; c < NC; ++c) {
632 auto extracted = wrk->fetch(c, dbuffer.data());
633 fill_datasets_from_dense(extracted, NR, inspace);
634 ptrs[c + 1] = offset;
635 }
636 }
637 }
638
639 // Saving the pointers.
640 auto ptr_len = sanisizer::cast<hsize_t>(ptrs.size());
641 H5::DataSet ptr_ds = create_1d_compressed_hdf5_dataset(
642 location,
643 choose_ptr_type(params.ptr_type, ptrs.back()),
644 ptr_name,
645 ptr_len,
646 params.deflate_level,
647 params.chunk_size,
648 params.shuffle
649 );
650 H5::DataSpace ptr_space(1, &ptr_len);
651 ptr_ds.write(ptrs.data(), H5::PredType::NATIVE_HSIZE, ptr_space);
652}
670template<typename Value_, typename Index_>
672 // Choosing the layout.
673 WriteStorageLayout layout;
674 if (params.columnar.has_value()) {
675 layout = *(params.columnar);
676 } else {
677 if (mat.prefer_rows()) {
678 layout = WriteStorageLayout::ROW;
679 } else {
680 layout = WriteStorageLayout::COLUMN;
681 }
682 }
683
684 // Choosing the names.
685 std::string data_name;
686 if (params.data_name.has_value()) {
687 data_name = *(params.data_name);
688 } else {
689 data_name = "data";
690 }
691
692 std::string index_name;
693 if (params.index_name.has_value()) {
694 index_name = *(params.index_name);
695 } else {
696 index_name = "indices";
697 }
698
699 std::string ptr_name;
700 if (params.ptr_name.has_value()) {
701 ptr_name = *(params.ptr_name);
702 } else {
703 ptr_name = "indptr";
704 }
705
706 // Only executing a one-pass strategy if the types are already known.
707 if (params.two_pass || !params.data_type.has_value() || !params.index_type.has_value()) {
708 write_compressed_sparse_matrix_two_pass(mat, location, layout, data_name, index_name, ptr_name, params);
709 } else {
710 write_compressed_sparse_matrix_one_pass(mat, location, layout, data_name, index_name, ptr_name, params);
711 }
712}
713
723template<typename Value_, typename Index_>
726 write_compressed_sparse_matrix(mat, location, params);
727 return;
728}
729
733template<typename Value_, typename Index_>
734void write_compressed_sparse_matrix(const tatami::Matrix<Value_, Index_>* mat, H5::Group& location, const WriteCompressedSparseMatrixOptions& params) {
735 return write_compressed_sparse_matrix(*mat, location, params);
736}
737
738template<typename Value_, typename Index_>
739void write_compressed_sparse_matrix(const tatami::Matrix<Value_, Index_>* mat, H5::Group& location) {
740 return write_compressed_sparse_matrix(*mat, location);
741}
746}
747
748#endif
virtual Index_ ncol() const=0
virtual Index_ nrow() const=0
virtual bool prefer_rows() const=0
virtual std::unique_ptr< MyopicSparseExtractor< Value_, Index_ > > sparse(bool row, const Options &opt) const=0
Representations for matrix data in HDF5 files.
Definition CompressedSparseMatrix.hpp:24
WriteStorageLayout
Definition utils.hpp:26
WriteStorageType
Definition utils.hpp:31
void write_compressed_sparse_matrix(const tatami::Matrix< Value_, Index_ > &mat, H5::Group &location, const WriteCompressedSparseMatrixOptions &params)
Definition write_compressed_sparse_matrix.hpp:671
int parallelize(Function_ fun, const Index_ tasks, const int workers)
Container_ create_container_of_Index_size(const Index_ x, Args_ &&... args)
auto consecutive_extractor(const Matrix< Value_, Index_ > &matrix, const bool row, const Index_ iter_start, const Index_ iter_length, Args_ &&... args)
const Value_ * value
const Index_ * index
Parameters for write_compressed_sparse_matrix().
Definition write_compressed_sparse_matrix.hpp:26
std::optional< WriteStorageLayout > columnar
Definition write_compressed_sparse_matrix.hpp:49
bool two_pass
Definition write_compressed_sparse_matrix.hpp:102
bool force_integer
Definition write_compressed_sparse_matrix.hpp:63
int deflate_level
Definition write_compressed_sparse_matrix.hpp:82
std::optional< WriteStorageType > ptr_type
Definition write_compressed_sparse_matrix.hpp:75
hsize_t chunk_size
Definition write_compressed_sparse_matrix.hpp:88
std::optional< std::string > data_name
Definition write_compressed_sparse_matrix.hpp:31
std::optional< std::string > ptr_name
Definition write_compressed_sparse_matrix.hpp:43
std::optional< std::string > index_name
Definition write_compressed_sparse_matrix.hpp:37
std::optional< WriteStorageType > index_type
Definition write_compressed_sparse_matrix.hpp:69
std::optional< WriteStorageType > data_type
Definition write_compressed_sparse_matrix.hpp:55
int num_threads
Definition write_compressed_sparse_matrix.hpp:108
bool shuffle
Definition write_compressed_sparse_matrix.hpp:95
Utilities for HDF5 extraction.