OGS
TestDefinition.cpp
Go to the documentation of this file.
1 
12 #include "TestDefinition.h"
13 
14 #include <cmath>
15 #include <cstdlib>
16 #include <filesystem>
17 #include <regex>
18 #include <vector>
19 
20 #include "BaseLib/ConfigTree.h"
21 #include "BaseLib/Error.h"
22 #include "BaseLib/FileTools.h"
23 #ifdef USE_PETSC
24 #include <petsc.h>
25 
26 #include "MeshLib/IO/VtkIO/VtuInterface.h" // For petsc file name conversion.
27 #endif
28 
29 namespace
30 {
32 bool isConvertibleToDouble(std::string const& s)
33 {
34  std::size_t pos = 0;
35  double value;
36  try
37  {
38  value = std::stod(s, &pos);
39  }
40  catch (...)
41  {
42  OGS_FATAL("The given string '{:s}' is not convertible to double.", s);
43  }
44  if (pos != s.size())
45  {
46  OGS_FATAL(
47  "Only {:d} characters were used for double conversion of string "
48  "'{:s}'",
49  pos, s);
50  }
51 
52  if (std::isnan(value))
53  {
54  OGS_FATAL("The given string '{:s}' results in a NaN value.", s);
55  }
56  return true;
57 }
58 
60 std::string safeString(std::string const& s)
61 {
62  std::stringstream ss;
63  ss << std::quoted(s);
64  return ss.str();
65 }
66 
69 std::string findVtkdiff()
70 {
71  // Try to read the VTKDIFF_EXE environment variable.
72  if (const char* vtkdiff_exe_environment_variable =
73  std::getenv("VTKDIFF_EXE"))
74  {
75  std::string const vtkdiff_exe{vtkdiff_exe_environment_variable};
76  DBUG("VTKDIFF_EXE set to {:s}.", vtkdiff_exe);
77 
78  //
79  // Sanity checks.
80  //
81  { // The base name is 'vtkdiff'
82  auto const& base_name =
84  if (base_name != "vtkdiff")
85  {
86  OGS_FATAL(
87  "The VTKDIFF_EXE environment variable does not point to "
88  "'vtkdiff'. VTKDIFF_EXE='{:s}'",
89  vtkdiff_exe);
90  }
91  }
92  { // vtkdiff must exist.
93  if (!BaseLib::IsFileExisting(vtkdiff_exe))
94  {
95  OGS_FATAL(
96  "The VTKDIFF_EXE points to a non-existing file. "
97  "VTKDIFF_EXE='{:s}'",
98  vtkdiff_exe);
99  }
100  }
101 
102  //
103  // Test the actual call.
104  //
105  int const return_value =
106  // TODO (naumov) replace system call with output consuming call
107  // (fork + execl seems to be more safe), and extract the vtkdiff
108  // call to common function. Also properly escape all strings in
109  // command lines.
110  // Reference for POSIX and Windows:
111  // https://wiki.sei.cmu.edu/confluence/pages/viewpage.action?pageId=87152177
112  // Take care when using fork, which might copy resources.
113  std::system((vtkdiff_exe + " --version").c_str());
114  if (return_value == 0)
115  {
116  return vtkdiff_exe;
117  }
118  WARN(
119  "Calling {:s} from the VTKDIFF_EXE environment variable didn't "
120  "work as expected. Return value was {:d}.",
121  vtkdiff_exe, return_value);
122  }
123 
124  std::string const vtkdiff_exe{"vtkdiff"};
125  std::vector<std::string> const paths = {"", "bin"};
126  auto const path =
127  find_if(begin(paths), end(paths),
128  [&vtkdiff_exe](std::string const& path)
129  {
130  int const return_value =
131  // TODO (naumov) replace system call with output
132  // consuming call as in an above todo comment.
133  std::system((BaseLib::joinPaths(path, vtkdiff_exe) +
134  " --version")
135  .c_str());
136  return return_value == 0;
137  });
138  if (path == end(paths))
139  {
140  OGS_FATAL("vtkdiff not found.");
141  }
142  return BaseLib::joinPaths(*path, vtkdiff_exe);
143 }
144 
145 } // namespace
146 
147 namespace ApplicationsLib
148 {
150  std::string const& reference_path,
151  std::string const& output_directory)
152 {
153  if (reference_path.empty())
154  {
155  OGS_FATAL(
156  "Reference path containing expected result files can not be "
157  "empty.");
158  }
159 
160  std::string const vtkdiff = findVtkdiff();
161 
162  // Construct command lines for each entry.
164  auto const& vtkdiff_configs = config_tree.getConfigSubtreeList("vtkdiff");
165  _command_lines.reserve(vtkdiff_configs.size());
166  for (auto const& vtkdiff_config : vtkdiff_configs)
167  {
168  std::string const& field_name =
170  vtkdiff_config.getConfigParameter<std::string>("field");
171  DBUG("vtkdiff will compare field '{:s}'.", field_name);
172 
173  std::vector<std::string> filenames;
174  if (auto const regex_string =
176  vtkdiff_config.getConfigParameterOptional<std::string>("regex"))
177  {
178  // TODO: insert rank into regex for mpi case
179  DBUG("vtkdiff regex is '{}'.", *regex_string);
180  auto const regex = std::regex(*regex_string);
181  for (auto const& p : std::filesystem::directory_iterator(
182  std::filesystem::path(reference_path)))
183  {
184  auto const filename = p.path().filename().string();
185  if (std::regex_match(filename, regex))
186  {
187  DBUG(" -> matched '{}'", filename);
188  filenames.push_back(filename);
189  }
190  }
191  }
192  else
193  {
194  std::string filename =
196  vtkdiff_config.getConfigParameter<std::string>("file");
197 #ifdef USE_PETSC
198  int mpi_size;
199  MPI_Comm_size(PETSC_COMM_WORLD, &mpi_size);
200  if (mpi_size > 1)
201  {
202  int rank;
203  MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
204  filename =
206  filename) +
207  "_" + std::to_string(rank) + ".vtu";
208  }
209 #endif // OGS_USE_PETSC
210  filenames.push_back(filename);
211  }
212 
213  if (empty(filenames))
214  {
215  OGS_FATAL(
216  "No files from test definitions were added for tests but {} "
217  "{:s} specified.",
218  std::size(vtkdiff_configs),
219  (std::size(vtkdiff_configs) == 1 ? "test was" : "tests were"));
220  }
221 
222  auto const absolute_tolerance =
224  vtkdiff_config.getConfigParameter<std::string>("absolute_tolerance",
225  "");
226  if (!absolute_tolerance.empty() &&
227  !isConvertibleToDouble(absolute_tolerance))
228  {
229  OGS_FATAL(
230  "The absolute tolerance value '{:s}' is not convertible to "
231  "double.",
232  absolute_tolerance);
233  }
234  std::string const absolute_tolerance_parameter =
235  "--abs " + absolute_tolerance;
236  auto const relative_tolerance =
238  vtkdiff_config.getConfigParameter<std::string>("relative_tolerance",
239  "");
240  if (!relative_tolerance.empty() &&
241  !isConvertibleToDouble(relative_tolerance))
242  {
243  OGS_FATAL(
244  "The relative tolerance value '{:s}' is not convertible to "
245  "double.",
246  relative_tolerance);
247  }
248  std::string const relative_tolerance_parameter =
249  "--rel " + relative_tolerance;
250 
251  for (auto const& filename : filenames)
252  {
253  std::string const& output_filename =
254  BaseLib::joinPaths(output_directory, filename);
255  _output_files.push_back(output_filename);
256  std::string const& reference_filename =
257  BaseLib::joinPaths(reference_path, filename);
258 
259  //
260  // Construct command line.
261  //
262  std::string command_line =
263  vtkdiff + " -a " + safeString(field_name) + " -b " +
264  safeString(field_name) + " " + safeString(reference_filename) +
265  " " + safeString(output_filename) + " " +
266  absolute_tolerance_parameter + " " +
267  relative_tolerance_parameter;
268  INFO("Will run '{:s}'", command_line);
269  _command_lines.emplace_back(std::move(command_line));
270  }
271  }
272 }
273 
275 {
276  std::vector<int> return_values;
277  transform(begin(_command_lines), end(_command_lines),
278  back_inserter(return_values),
279  [](std::string const& command_line)
280  {
281  INFO("---------- vtkdiff begin ----------");
282  int const return_value = std::system(command_line.c_str());
283  if (return_value != 0)
284  {
285  WARN("Return value {:d} was returned by '{:s}'.",
286  return_value, command_line);
287  }
288  INFO("---------- vtkdiff end ----------\n");
289  return return_value;
290  });
291  return !return_values.empty() &&
292  all_of(begin(return_values), end(return_values),
293  [](int const& return_value) { return return_value == 0; });
294 }
295 
296 std::vector<std::string> const& TestDefinition::getOutputFiles() const
297 {
298  return _output_files;
299 }
300 
301 std::size_t TestDefinition::numberOfTests() const
302 {
303  return size(_command_lines);
304 }
305 } // namespace ApplicationsLib
#define OGS_FATAL(...)
Definition: Error.h:26
Filename manipulation routines.
void INFO(fmt::format_string< Args... > fmt, Args &&... args)
Definition: Logging.h:34
void DBUG(fmt::format_string< Args... > fmt, Args &&... args)
Definition: Logging.h:29
void WARN(fmt::format_string< Args... > fmt, Args &&... args)
Definition: Logging.h:39
Implementation of the VtuInterface class.
std::vector< std::string > _output_files
std::vector< std::string > _command_lines
TestDefinition(BaseLib::ConfigTree const &config_tree, std::string const &reference_path, std::string const &output_directory)
std::vector< std::string > const & getOutputFiles() const
Range< SubtreeIterator > getConfigSubtreeList(std::string const &root) const
Definition: ConfigTree.cpp:173
bool IsFileExisting(const std::string &strFilename)
Returns true if given file exists.
Definition: FileTools.cpp:43
constexpr bool all_of(List const &values)
Checks if all of the elements in the given list are true.
Definition: Algorithm.h:341
std::string extractBaseNameWithoutExtension(std::string const &pathname)
Definition: FileTools.cpp:181
std::string joinPaths(std::string const &pathA, std::string const &pathB)
Definition: FileTools.cpp:213
static const double s
std::string getVtuFileNameForPetscOutputWithoutExtension(std::string const &file_name)
bool isConvertibleToDouble(std::string const &s)
Test if the given string is convertible to a valid double value, not a NaN.
std::string safeString(std::string const &s)
Wraps a string into double ticks.