Alexandria 2.25.0
SDC-CH common library for the Euclid project
FileSystemProvider.cpp
Go to the documentation of this file.
1/*
2 * Copyright (C) 2012-2021 Euclid Science Ground Segment
3 *
4 * This library is free software; you can redistribute it and/or modify it under
5 * the terms of the GNU Lesser General Public License as published by the Free
6 * Software Foundation; either version 3.0 of the License, or (at your option)
7 * any later version.
8 *
9 * This library is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
11 * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
12 * details.
13 *
14 * You should have received a copy of the GNU Lesser General Public License
15 * along with this library; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
29#include "StringFunctions.h"
30#include <boost/algorithm/string.hpp>
31#include <boost/filesystem.hpp>
32#include <fstream>
33#include <set>
34#include <string>
35#include <unordered_set>
36
37namespace fs = boost::filesystem;
38
39namespace Euclid {
40namespace XYDataset {
41
43
52static std::vector<fs::path> getOrder(const fs::path& dir) {
53 std::vector<fs::path> result{};
54
55 // First add the files in the order.txt
56 auto order_file = dir / "order.txt";
57 std::unordered_set<std::string> ordered_names{};
58 if (fs::exists(order_file)) {
59 std::ifstream in{order_file.c_str()};
60 while (in) {
61 std::string line;
62 getline(in, line);
63 size_t comment_pos = line.find('#');
64 if (comment_pos != std::string::npos) {
65 line = line.substr(0, comment_pos);
66 }
67 boost::trim(line);
68 if (!line.empty()) {
69 auto name = dir / line;
70 if (fs::exists(name)) {
71 result.emplace_back(name);
72 ordered_names.emplace(line);
73 } else {
74 logger.warn() << "Unknown name " << line << " in order.txt of " << dir << " directory";
75 }
76 }
77 }
78 }
79
80 // Now we add any other files in the directory, which were not in the order.txt
81 // file. We use a set in order to avoid sorting problem between platforms.
82 std::set<fs::path> remaining_files{};
83 for (fs::directory_iterator iter{dir}; iter != fs::directory_iterator{}; ++iter) {
84 if (ordered_names.count(iter->path().filename().string()) == 0) {
85 remaining_files.emplace(*iter);
86 }
87 }
88
89 // Put the remaining files into the result vector
90 for (auto& file : remaining_files) {
91 result.emplace_back(file);
92 }
93
94 return result;
95}
96
98 std::vector<fs::path> result{};
99 auto ordered_contents = getOrder(dir);
100 for (auto& name : ordered_contents) {
101 if (fs::is_directory(name)) {
102 auto sub_dir_contents = getRecursiveDirectoryContents(name);
103 result.insert(result.end(), sub_dir_contents.begin(), sub_dir_contents.end());
104 } else {
105 result.emplace_back(name);
106 }
107 }
108 return result;
109}
110
111//-----------------------------------------------------------------------------
112// Constructor
113//-----------------------------------------------------------------------------
114
116 : XYDatasetProvider(), m_root_path(root_path), m_parser(std::move(parser)) {
117
118 std::vector<std::string> string_vector{};
119
120 // Make sure the root path finishes with a "/" and only one
122
123 // Convert path to boost filesytem object
124 fs::path fspath(m_root_path);
125 if (!fs::exists(fspath)) {
126 throw Elements::Exception() << "From FileSystemProvider: root path not found : " << fspath;
127 }
128
129 // Get all files below the root directory
130 if (fs::is_directory(fspath)) {
131 auto dir_contents = getRecursiveDirectoryContents(fspath);
132 for (auto& file : dir_contents) {
133 if (fs::is_regular_file(file) && m_parser->isDatasetFile(file.string())) {
134 std::string dataset_name = m_parser->getName(file.string());
135 // Remove empty dataset name
136 if (dataset_name.empty()) {
137 continue;
138 }
139 // Remove the root part
140 std::string str = file.string();
141 str = str.substr(m_root_path.length(), str.length());
142 // Split by the character '/'
144 boost::split(groups, str, boost::is_any_of("/"));
145 // The last string is the file name, so we remove it
146 groups.pop_back();
147 QualifiedName qualified_name{groups, dataset_name};
148 // Fill up a map
149 auto ret = m_name_file_map.insert(make_pair(qualified_name, file.string()));
150 m_order_names.push_back(qualified_name);
151 // Check for unique record
152 if (!ret.second) {
153 throw Elements::Exception() << "Qualified name can not be inserted "
154 << "in the map. Qualify name : " << qualified_name.qualifiedName()
155 << " Path :" << file.string();
156 }
157 }
158 }
159 } else {
160 throw Elements::Exception() << " Root path : " << fspath.string() << " is not a directory!";
161 }
162}
163
164//-----------------------------------------------------------------------------
165// listContents function
166//-----------------------------------------------------------------------------
167
169
170 std::string my_group = group;
171 // Make sure the group finishes with a "/" and only one
172 while (!my_group.empty() && my_group.back() == '/') {
173 my_group.pop_back();
174 }
175 // Make sure the group do not start with a "/"
176 size_t pos = my_group.find_first_not_of("/");
177 if (!my_group.empty() && pos != 0) {
178 my_group = my_group.substr(pos);
179 }
180 if (!my_group.empty()) {
181 my_group.push_back('/');
182 }
183
184 std::vector<QualifiedName> qualified_name_vector{};
185
186 // Fill up vector with qualified name from the map
187 // Insert all qualified name where path contains the group name at the
188 // first position
189 for (auto qualified_name : m_order_names) {
190 if (boost::starts_with(qualified_name.qualifiedName(), my_group)) {
191 qualified_name_vector.push_back(qualified_name);
192 }
193 } // Eof for
194
195 return (qualified_name_vector);
196}
197
198//-----------------------------------------------------------------------------
199// getDataset function
200//-----------------------------------------------------------------------------
201
203
204 auto it = m_name_file_map.find(qualified_name);
205 return (it != m_name_file_map.end()) ? m_parser->getDataset(it->second) : nullptr;
206}
207
209 auto it = m_name_file_map.find(qualified_name);
210 return (it != m_name_file_map.end()) ? m_parser->getParameter(it->second, key_word) : "";
211}
212
213} /* namespace XYDataset */
214} // end of namespace Euclid
T back(T... args)
static Logging getLogger(const std::string &name="")
void warn(const std::string &logMessage)
std::vector< QualifiedName > m_order_names
std::vector< QualifiedName > listContents(const std::string &group) override
List all files which belong to a group.
std::string getParameter(const QualifiedName &qualified_name, const std::string &key_word) override
FileSystemProvider(const std::string &root_path, std::unique_ptr< FileParser > parser)
constructor The FileSystemProvider handles files in a directory tree.
std::unique_ptr< FileParser > m_parser
std::unique_ptr< XYDataset > getDataset(const QualifiedName &qualified_name) override
Get a dataset corresponding to an unique qualified name.
std::map< QualifiedName, std::string > m_name_file_map
Represents a name qualified with a set of groups.
Definition: QualifiedName.h:66
This interface class provides the dataset following a qualified name object.
T emplace(T... args)
T empty(T... args)
T find_first_not_of(T... args)
T find(T... args)
ELEMENTS_API auto split(Args &&... args) -> decltype(splitPath(std::forward< Args >(args)...))
static std::vector< fs::path > getOrder(const fs::path &dir)
static Elements::Logging logger
static std::vector< fs::path > getRecursiveDirectoryContents(const fs::path &dir)
std::string checkEndSlashes(const std::string &input_str)
STL namespace.
T pop_back(T... args)
T push_back(T... args)
T length(T... args)
T substr(T... args)