OGS
TestDefinition.cpp
Go to the documentation of this file.
1
12#include "TestDefinition.h"
13
14#include <cmath>
15#include <cstdlib>
16#include <filesystem>
17#include <regex>
18#include <vector>
19
20#include "BaseLib/ConfigTree.h"
21#include "BaseLib/Error.h"
22#include "BaseLib/FileTools.h"
23#ifdef USE_PETSC
24#include <petsc.h>
25
26#include "MeshLib/IO/VtkIO/VtuInterface.h" // For petsc file name conversion.
27#endif
28
29namespace
30{
32bool isConvertibleToDouble(std::string const& s)
33{
34 std::size_t pos = 0;
35 double value;
36 try
37 {
38 value = std::stod(s, &pos);
39 }
40 catch (...)
41 {
42 ERR("The given string '{:s}' is not convertible to double.", s);
43 return false;
44 }
45 if (pos != s.size())
46 {
47 ERR("Only {:d} characters were used for double conversion of string "
48 "'{:s}'",
49 pos, s);
50 return false;
51 }
52
53 if (std::isnan(value))
54 {
55 ERR("The given string '{:s}' results in a NaN value.", s);
56 return false;
57 }
58 return true;
59}
60
62std::string safeString(std::string const& s)
63{
64 std::stringstream ss;
65 ss << std::quoted(s);
66 return ss.str();
67}
68
71std::string findVtkdiff()
72{
73 // Try to read the VTKDIFF_EXE environment variable.
74 if (const char* vtkdiff_exe_environment_variable =
75 std::getenv("VTKDIFF_EXE"))
76 {
77 std::string const vtkdiff_exe{vtkdiff_exe_environment_variable};
78 DBUG("VTKDIFF_EXE set to {:s}.", vtkdiff_exe);
79
80 //
81 // Sanity checks.
82 //
83 { // The base name is 'vtkdiff'
84 auto const& base_name =
86 if (base_name != "vtkdiff")
87 {
89 "The VTKDIFF_EXE environment variable does not point to "
90 "'vtkdiff'. VTKDIFF_EXE='{:s}'",
91 vtkdiff_exe);
92 }
93 }
94 { // vtkdiff must exist.
95 if (!BaseLib::IsFileExisting(vtkdiff_exe))
96 {
98 "The VTKDIFF_EXE points to a non-existing file. "
99 "VTKDIFF_EXE='{:s}'",
100 vtkdiff_exe);
101 }
102 }
103
104 //
105 // Test the actual call.
106 //
107 int const return_value =
108 // TODO (naumov) replace system call with output consuming call
109 // (fork + execl seems to be more safe), and extract the vtkdiff
110 // call to common function. Also properly escape all strings in
111 // command lines.
112 // Reference for POSIX and Windows:
113 // https://wiki.sei.cmu.edu/confluence/pages/viewpage.action?pageId=87152177
114 // Take care when using fork, which might copy resources.
115 std::system((vtkdiff_exe + " --version").c_str());
116 if (return_value == 0)
117 {
118 return vtkdiff_exe;
119 }
120 WARN(
121 "Calling {:s} from the VTKDIFF_EXE environment variable didn't "
122 "work as expected. Return value was {:d}.",
123 vtkdiff_exe, return_value);
124 }
125
126 std::string const vtkdiff_exe{"vtkdiff"};
127 std::vector<std::string> const paths = {"", "bin"};
128 auto const path =
129 find_if(begin(paths), end(paths),
130 [&vtkdiff_exe](std::string const& path)
131 {
132 int const return_value =
133 // TODO (naumov) replace system call with output
134 // consuming call as in an above todo comment.
135 std::system((BaseLib::joinPaths(path, vtkdiff_exe) +
136 " --version")
137 .c_str());
138 return return_value == 0;
139 });
140 if (path == end(paths))
141 {
142 OGS_FATAL("vtkdiff not found.");
143 }
144 return BaseLib::joinPaths(*path, vtkdiff_exe);
145}
146
147} // namespace
148
149namespace ApplicationsLib
150{
152 std::string const& reference_path,
153 std::string const& output_directory)
154{
155 if (reference_path.empty())
156 {
157 OGS_FATAL(
158 "Reference path containing expected result files can not be "
159 "empty.");
160 }
161
162 std::string const vtkdiff = findVtkdiff();
163
164 // Construct command lines for each entry.
166 auto const& vtkdiff_configs = config_tree.getConfigSubtreeList("vtkdiff");
167 _command_lines.reserve(vtkdiff_configs.size());
168 for (auto const& vtkdiff_config : vtkdiff_configs)
169 {
170 std::string const& field_name =
172 vtkdiff_config.getConfigParameter<std::string>("field");
173 DBUG("vtkdiff will compare field '{:s}'.", field_name);
174
175 std::vector<std::string> filenames;
176 if (auto const regex_string =
178 vtkdiff_config.getConfigParameterOptional<std::string>("regex"))
179 {
180 // TODO: insert rank into regex for mpi case
181 DBUG("vtkdiff regex is '{}'.", *regex_string);
182 auto const regex = std::regex(*regex_string);
183 for (auto const& p : std::filesystem::directory_iterator(
184 std::filesystem::path(reference_path)))
185 {
186 auto const filename = p.path().filename().string();
187 if (std::regex_match(filename, regex))
188 {
189 DBUG(" -> matched '{}'", filename);
190 filenames.push_back(filename);
191 }
192 }
193 }
194 else
195 {
196 std::string filename =
198 vtkdiff_config.getConfigParameter<std::string>("file");
199#ifdef USE_PETSC
200 int mpi_size;
201 MPI_Comm_size(PETSC_COMM_WORLD, &mpi_size);
202 if (mpi_size > 1)
203 {
204 int rank;
205 MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
206 filename =
208 filename) +
209 "_" + std::to_string(rank) + ".vtu";
210 }
211#endif // OGS_USE_PETSC
212 filenames.push_back(filename);
213 }
214
215 if (empty(filenames))
216 {
217 OGS_FATAL(
218 "No files from test definitions were added for tests but {} "
219 "{:s} specified.",
220 std::size(vtkdiff_configs),
221 (std::size(vtkdiff_configs) == 1 ? "test was" : "tests were"));
222 }
223
224 auto const absolute_tolerance =
226 vtkdiff_config.getConfigParameter<std::string>("absolute_tolerance",
227 "");
228 if (!absolute_tolerance.empty() &&
229 !isConvertibleToDouble(absolute_tolerance))
230 {
231 OGS_FATAL(
232 "The absolute tolerance value '{:s}' is not convertible to "
233 "double.",
234 absolute_tolerance);
235 }
236 std::string const absolute_tolerance_parameter =
237 "--abs " + absolute_tolerance;
238 auto const relative_tolerance =
240 vtkdiff_config.getConfigParameter<std::string>("relative_tolerance",
241 "");
242 if (!relative_tolerance.empty() &&
243 !isConvertibleToDouble(relative_tolerance))
244 {
245 OGS_FATAL(
246 "The relative tolerance value '{:s}' is not convertible to "
247 "double.",
248 relative_tolerance);
249 }
250 std::string const relative_tolerance_parameter =
251 "--rel " + relative_tolerance;
252
253 for (auto const& filename : filenames)
254 {
255 std::string output_filename =
256 BaseLib::joinPaths(output_directory, filename);
257 _output_files.push_back(output_filename);
258 std::string reference_filename =
259 BaseLib::joinPaths(reference_path, filename);
260#if _WIN32
261 // vtk does not handle Windows long paths:
262 // https://gitlab.kitware.com/vtk/vtk/-/blob/master/Utilities/KWSys/vtksys/SystemTools.cxx#L1519-1521
263 // workaround is to make path absolute and prefix with a special
264 // marker and put everything in quotes, see
265 // https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=powershell
266 auto const& long_path_indicator = R"(\\?\)";
267
268 reference_filename =
269 std::filesystem::absolute(reference_filename).string();
270 reference_filename = std::format("\"{}{}\"", long_path_indicator,
271 reference_filename);
272 output_filename =
273 std::filesystem::absolute(output_filename).string();
274 output_filename =
275 std::format("\"{}{}\"", long_path_indicator, output_filename);
276
277#else
278 output_filename = safeString(output_filename);
279 reference_filename = safeString(reference_filename);
280#endif
281 //
282 // Construct command line.
283 //
284 std::string command_line =
285 vtkdiff + " -a " + safeString(field_name) + " -b " +
286 safeString(field_name) + " " + reference_filename + " " +
287 output_filename + " " + absolute_tolerance_parameter + " " +
288 relative_tolerance_parameter;
289 INFO("Will run '{:s}'", command_line);
290 _command_lines.emplace_back(std::move(command_line));
291 }
292 }
293}
294
296{
297 std::vector<int> return_values;
298 transform(begin(_command_lines), end(_command_lines),
299 back_inserter(return_values),
300 [](std::string const& command_line)
301 {
302 INFO("---------- vtkdiff begin ----------");
303 int const return_value = std::system(command_line.c_str());
304 if (return_value != 0)
305 {
306 WARN("Return value {:d} was returned by '{:s}'.",
307 return_value, command_line);
308 }
309 INFO("---------- vtkdiff end ----------\n");
310 return return_value;
311 });
312 return !return_values.empty() &&
313 all_of(begin(return_values), end(return_values),
314 [](int const& return_value) { return return_value == 0; });
315}
316
317std::vector<std::string> const& TestDefinition::getOutputFiles() const
318{
319 return _output_files;
320}
321
323{
324 return size(_command_lines);
325}
326} // namespace ApplicationsLib
#define OGS_FATAL(...)
Definition Error.h:26
Filename manipulation routines.
void INFO(fmt::format_string< Args... > fmt, Args &&... args)
Definition Logging.h:35
void DBUG(fmt::format_string< Args... > fmt, Args &&... args)
Definition Logging.h:30
void ERR(fmt::format_string< Args... > fmt, Args &&... args)
Definition Logging.h:45
void WARN(fmt::format_string< Args... > fmt, Args &&... args)
Definition Logging.h:40
Implementation of the VtuInterface class.
OGS_EXPORT_SYMBOL bool runTests() const
std::vector< std::string > _output_files
std::vector< std::string > _command_lines
TestDefinition(BaseLib::ConfigTree const &config_tree, std::string const &reference_path, std::string const &output_directory)
std::vector< std::string > const & getOutputFiles() const
Range< SubtreeIterator > getConfigSubtreeList(std::string const &root) const
bool IsFileExisting(const std::string &strFilename)
Returns true if given file exists.
Definition FileTools.cpp:50
std::string extractBaseNameWithoutExtension(std::string const &pathname)
std::string joinPaths(std::string const &pathA, std::string const &pathB)
std::string getVtuFileNameForPetscOutputWithoutExtension(std::string const &file_name)
bool isConvertibleToDouble(std::string const &s)
Test if the given string is convertible to a valid double value, not a NaN.
std::string safeString(std::string const &s)
Wraps a string into double ticks.