Alexandria 2.25.0
SDC-CH common library for the Euclid project
NpyMmap.icpp
Go to the documentation of this file.
1/*
2 * Copyright (C) 2012-2021 Euclid Science Ground Segment
3 *
4 * This library is free software; you can redistribute it and/or modify it under
5 * the terms of the GNU Lesser General Public License as published by the Free
6 * Software Foundation; either version 3.0 of the License, or (at your option)
7 * any later version.
8 *
9 * This library is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
11 * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
12 * details.
13 *
14 * You should have received a copy of the GNU Lesser General Public License
15 * along with this library; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
19#ifdef NPYMMAP_IMPL
20
21#include "NpyCommon.h"
22#include <boost/filesystem/path.hpp>
23#include <boost/iostreams/stream.hpp>
24#include <numeric>
25
26namespace Euclid {
27namespace NdArray {
28
29typedef boost::iostreams::stream<boost::iostreams::mapped_file_source> MappedStream;
30
31template <typename T>
32NdArray<T> mmapNpy(const boost::filesystem::path& path, boost::iostreams::mapped_file_base::mapmode mode,
33 size_t max_size) {
34 std::string dtype;
35 size_t n_elements = 0;
36 std::vector<size_t> shape;
37 std::vector<std::string> attrs;
38
39 boost::iostreams::mapped_file_params map_params;
40 map_params.path = path.native();
41 map_params.flags = mode;
42 if (max_size)
43 map_params.length = max_size;
44 else
45 max_size = boost::filesystem::file_size(path);
46
47 boost::iostreams::mapped_file input(map_params);
48 MappedStream stream(input.operator boost::iostreams::mapped_file_source&());
49 stream.set_auto_close(false);
50 readNpyHeader(stream, dtype, shape, attrs, n_elements);
51
52 if (dtype != NpyDtype<T>::str)
53 throw Elements::Exception() << "Can not cast " << dtype << " into " << typeid(T).name();
54
55 if (!attrs.empty()) {
56 n_elements *= attrs.size();
57 }
58
59 return {shape, attrs,
60 std::move(MappedContainer<T>(path, stream.tellg(), n_elements, attrs, std::move(input), max_size))};
61}
62
63template <typename T>
64NdArray<T> createMmapNpy(const boost::filesystem::path& path, const std::vector<size_t>& shape,
65 const std::vector<std::string>& attrs, size_t max_size) {
66 // Pre-generate header
67 std::stringstream header;
68 writeNpyHeader<T>(header, appendAttrShape(shape, attrs.size()), attrs);
69 auto header_str = header.str();
70 auto header_size = header_str.size();
71
72 assert(header_size % 64 == 0);
73
74 // Compute file expected size
75 size_t n_elements = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies<size_t>());
76 if (!attrs.empty())
77 n_elements *= attrs.size();
78 size_t data_size = n_elements * sizeof(T);
79 size_t total_size = header_size + data_size;
80
81 boost::iostreams::mapped_file_params map_params;
82 map_params.path = path.native();
83 map_params.flags = boost::iostreams::mapped_file_base::readwrite;
84 map_params.new_file_size = total_size;
85 if (max_size >= total_size)
86 map_params.length = max_size;
87 else
88 max_size = total_size;
89
90 boost::iostreams::mapped_file output(map_params);
91 std::copy(header_str.begin(), header_str.end(), output.begin());
92 return {shape, attrs,
93 std::move(MappedContainer<T>(path, header_size, n_elements, attrs, std::move(output), max_size))};
94}
95
96} // end of namespace NdArray
97} // end of namespace Euclid
98
99#endif // NPYMMAP_IMPL