diff options
| -rw-r--r-- | geom_matching/wasserstein/CMakeLists.txt | 3 | ||||
| -rw-r--r-- | geom_matching/wasserstein/example/wasserstein_dist_finitize_dipha.cpp | 138 | ||||
| -rw-r--r-- | geom_matching/wasserstein/include/diagram_reader.h | 20 | ||||
| -rw-r--r-- | geom_matching/wasserstein/mpi/CMakeLists.txt | 36 | ||||
| -rw-r--r-- | geom_matching/wasserstein/mpi/debug.hpp | 8 | ||||
| -rw-r--r-- | geom_matching/wasserstein/mpi/main.cpp | 399 |
6 files changed, 604 insertions, 0 deletions
diff --git a/geom_matching/wasserstein/CMakeLists.txt b/geom_matching/wasserstein/CMakeLists.txt index dea4550..53da46b 100644 --- a/geom_matching/wasserstein/CMakeLists.txt +++ b/geom_matching/wasserstein/CMakeLists.txt @@ -52,6 +52,9 @@ target_link_libraries(wasserstein_dist PUBLIC ${libraries}) add_executable(wasserstein_dist_dipha ${CMAKE_CURRENT_SOURCE_DIR}/example/wasserstein_dist_dipha.cpp ${WS_HEADERS} include/hera_infinity.h) target_link_libraries(wasserstein_dist_dipha PUBLIC ${libraries}) +add_executable(wasserstein_dist_finitize_dipha ${CMAKE_CURRENT_SOURCE_DIR}/example/wasserstein_dist_finitize_dipha.cpp ${WS_HEADERS}) +target_link_libraries(wasserstein_dist_finitize_dipha PUBLIC ${libraries}) + # pure geometric version, arbitrary dimension add_executable(wasserstein_dist_point_cloud ${CMAKE_CURRENT_SOURCE_DIR}/example/wasserstein_dist_point_cloud.cpp ${WS_HEADERS} include/hera_infinity.h) target_link_libraries(wasserstein_dist_point_cloud PUBLIC ${libraries}) diff --git a/geom_matching/wasserstein/example/wasserstein_dist_finitize_dipha.cpp b/geom_matching/wasserstein/example/wasserstein_dist_finitize_dipha.cpp new file mode 100644 index 0000000..5269a36 --- /dev/null +++ b/geom_matching/wasserstein/example/wasserstein_dist_finitize_dipha.cpp @@ -0,0 +1,138 @@ +/* + +Copyright (c) 2015, M. Kerber, D. Morozov, A. Nigmetov +Copyright (c) 2018, G. Spreemann +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +You are under no obligation whatsoever to provide any bug fixes, patches, or +upgrades to the features, functionality or performance of the source code +(Enhancements) to anyone; however, if you choose to make your Enhancements +available either publicly, or directly to copyright holder, +without imposing a separate written license agreement for such Enhancements, +then you hereby grant the following license: a non-exclusive, royalty-free +perpetual license to install, use, modify, prepare derivative works, incorporate +into other computer software, distribute, and sublicense such enhancements or +derivative works thereof, in binary and source code form. + + */ + +#include <iostream> +#include <locale> +#include <iomanip> +#include <vector> + +//#define LOG_AUCTION + +//#include "auction_runner_fr.h" +//#include "auction_runner_fr.hpp" + +#include "wasserstein.h" + +// any container of pairs of doubles can be used, +// we use vector in this example. + +int main(int argc, char* argv[]) +{ + using PairVector = std::vector<std::pair<double, double>>; + PairVector diagramA, diagramB; + + hera::AuctionParams<double> params; + + if (argc < 4 ) { + std::cerr << "Usage: " << argv[0] << " file1 file2 ph_dim finitization [wasserstein_degree] [relative_error] [internal norm] [initial epsilon] [epsilon_factor] [max_bids_per_round] [gamma_threshold][log_filename_prefix]. By default power is 1.0, relative error is 0.01, internal norm is l_infinity, initall epsilon is chosen automatically, epsilon factor is 5.0, Jacobi variant is used (max bids per round is maximal), gamma_threshold = 0.0." << std::endl; + return 1; + } + + unsigned int dim = std::stoul(argv[3]); + + if (!hera::read_diagram_dipha<double, PairVector>(argv[1], dim, diagramA)) { + std::exit(1); + } + + if (!hera::read_diagram_dipha<double, PairVector>(argv[2], dim, diagramB)) { + std::exit(1); + } + + double finitization = std::atof(argv[4]); + + hera::finitize(finitization, diagramA); + hera::finitize(finitization, diagramB); + + params.wasserstein_power = (6 <= argc) ? atof(argv[5]) : 1.0; + if (params.wasserstein_power < 1.0) { + std::cerr << "The fourth argument (wasserstein_degree) was \"" << argv[5] << "\", must be a number >= 1.0. Cannot proceed. " << std::endl; + std::exit(1); + } + + if (params.wasserstein_power == 1.0) { + hera::remove_duplicates<double>(diagramA, diagramB); + } + + //default relative error: 1% + params.delta = (7 <= argc) ? atof(argv[6]) : 0.01; + if ( params.delta <= 0.0) { + std::cerr << "The 5th argument (relative error) was \"" << argv[6] << "\", must be a number > 0.0. Cannot proceed. " << std::endl; + std::exit(1); + } + + // default for internal metric is l_infinity + params.internal_p = ( 8 <= argc ) ? atof(argv[7]) : hera::get_infinity<double>(); + if (std::isinf(params.internal_p)) { + params.internal_p = hera::get_infinity<double>(); + } + + + if (not hera::is_p_valid_norm<double>(params.internal_p)) { + std::cerr << "The 7th argument (internal norm) was \"" << argv[7] << "\", must be a number >= 1.0 or inf. Cannot proceed. " << std::endl; + std::exit(1); + } + + // if you want to specify initial value for epsilon and the factor + // for epsilon-scaling + params.initial_epsilon= ( 9 <= argc ) ? atof(argv[8]) : 0.0 ; + + if (params.initial_epsilon < 0.0) { + std::cerr << "The 8th argument (initial epsilon) was \"" << argv[8] << "\", must be a non-negative number. Cannot proceed." << std::endl; + std::exit(1); + } + + params.epsilon_common_ratio = ( 10 <= argc ) ? atof(argv[9]) : 0.0 ; + if (params.epsilon_common_ratio <= 1.0 and params.epsilon_common_ratio != 0.0) { + std::cerr << "The 9th argument (epsilon factor) was \"" << argv[9] << "\", must be a number greater than 1. Cannot proceed." << std::endl; + std::exit(1); + } + + + params.max_bids_per_round = ( 11 <= argc ) ? atoi(argv[10]) : 0; + if (params.max_bids_per_round == 0) + params.max_bids_per_round = std::numeric_limits<size_t>::max(); + + + params.gamma_threshold = (12 <= argc) ? atof(argv[11]) : 0.0; + + std::string log_filename_prefix = ( 13 <= argc ) ? argv[12] : ""; + + params.max_num_phases = 800; + +#ifdef LOG_AUCTION + spdlog::set_level(spdlog::level::info); +#endif + + double res = hera::wasserstein_dist(diagramA, diagramB, params, log_filename_prefix); + + std::cout << std::setprecision(15) << res << std::endl; + + return 0; + +} diff --git a/geom_matching/wasserstein/include/diagram_reader.h b/geom_matching/wasserstein/include/diagram_reader.h index b52fcbd..4b24f78 100644 --- a/geom_matching/wasserstein/include/diagram_reader.h +++ b/geom_matching/wasserstein/include/diagram_reader.h @@ -324,6 +324,26 @@ inline void remove_duplicates(ContType& dgm_A, ContType& dgm_B) } } +template<class RealType = double> +int finitize(RealType finitization, std::vector<std::pair<RealType, RealType> >& diagram) +{ + if (finitization == std::numeric_limits<double>::infinity()) + return 0; + + int altered = 0; + + for (auto it = diagram.begin(); it != diagram.end(); ++it) + { + if (it->second > finitization) + { + it->second = finitization; + ++altered; + } + } + + return altered; +} + #ifdef WASSERSTEIN_PURE_GEOM diff --git a/geom_matching/wasserstein/mpi/CMakeLists.txt b/geom_matching/wasserstein/mpi/CMakeLists.txt new file mode 100644 index 0000000..c307a24 --- /dev/null +++ b/geom_matching/wasserstein/mpi/CMakeLists.txt @@ -0,0 +1,36 @@ +cmake_minimum_required(VERSION 2.8.8) + +project(hera_mpi) + +option(SILENCE "Turn off some warnings." OFF) + +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + add_definitions(-DDEBUG) + remove_definitions(-DNDEBUG) +endif() + +if(SILENCE) + message(WARNING "Silence flag currently ignored.") + add_definitions(-DSILENCE) +endif(SILENCE) + +include(TestBigEndian) +test_big_endian(BIG_ENDIAN) +if(BIG_ENDIAN) + add_definitions(-DBIGENDIAN) +endif() + +find_package(MPI REQUIRED) +include_directories(${MPI_INCLUDE_PATH}) +link_directories(${MPI_LIBRARIES}) + +find_package(Boost REQUIRED) +include_directories(${Boost_INCLUDE_DIRS}) + +include_directories("../include") + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${MPI_COMPILE_FLAGS} -Wall -pedantic -Wextra -std=c++14") +set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${MPI_LINK_FLAGS}") + +add_executable(hera_mpi main.cpp) +target_link_libraries(hera_mpi ${MPI_LIBRARIES} ${Boost_LIBRARIES}) diff --git a/geom_matching/wasserstein/mpi/debug.hpp b/geom_matching/wasserstein/mpi/debug.hpp new file mode 100644 index 0000000..a74a1d8 --- /dev/null +++ b/geom_matching/wasserstein/mpi/debug.hpp @@ -0,0 +1,8 @@ +#pragma once + +#ifdef DEBUG +#include <iostream> +#define IFDEBUG(x) do { x } while (false) +#else +#define IFDEBUG(x) +#endif diff --git a/geom_matching/wasserstein/mpi/main.cpp b/geom_matching/wasserstein/mpi/main.cpp new file mode 100644 index 0000000..c3b4e09 --- /dev/null +++ b/geom_matching/wasserstein/mpi/main.cpp @@ -0,0 +1,399 @@ +#include <iostream> +#include <fstream> +#include <string> +#include <vector> +#include <cassert> +#include <mpi.h> +#include <limits> +#include <algorithm> +#include <cstdlib> +#include "debug.hpp" +#include "wasserstein.h" + +void print_help(const std::string & invocation) +{ + std::cout << "Usage: " << invocation << " arguments" << std::endl; + std::cout << "Arguments:" << std::endl; + std::cout << " --in-list, -i file" << std::endl; + std::cout << " Mandatory. File containing a list of persistence diagram files to process, one file per line." << std::endl; + std::cout << " --in-type, -t type" << std::endl; + std::cout << " Optional (defaults to dipha). Input file format, dipha|txt." << std::endl; + std::cout << " --dimension, -d dim" << std::endl; + std::cout << " Mandatory if the input is a DIPHA persistence diagram file. Ignored otherwise. Integer." << std::endl; + std::cout << " --outer-norm, -p p" << std::endl; + std::cout << " Mandatory. Floating point. Outer norm. In the interval [1, ∞)." << std::endl; + std::cout << " --inner-norm, -q q" << std::endl; + std::cout << " Mandatory. Floating point. Inner norm. In the interval [1, ∞]. Use inf for infinity." << std::endl; + std::cout << " --error, -e e" << std::endl; + std::cout << " Mandatory. Relative error. Positive floating point." << std::endl; + std::cout << " --finitize, -f f" << std::endl; + std::cout << " Optional. Make infinite intervals die at f." << std::endl; + std::cout << " --out, -o file" << std::endl; + std::cout << " Mandatory. Output file name. Plain text." << std::endl; + std::cout << " --chunk, -c c" << std::endl; + std::cout << " Optional. Size of work chunk to send off to each computational node. Too small a value yields a lot of overhead, too large a value can cause an unbalanced load. Increase if there are many small computations. Default: 100." << std::endl; + std::cout << " --help, -h" << std::endl; + std::cout << " Print this help text." << std::endl; +} + +void arg_fail(const std::string & invocation, int rank, const std::string & message) +{ + if (rank == 0) + { + std::cout << message << std::endl; + print_help(invocation); + } + MPI_Finalize(); + exit(1); +} + +void fail(int rank, const std::string & message) +{ + if (rank == 0) + { + std::cout << message << std::endl; + } + MPI_Finalize(); + exit(1); +} + +inline int unroll(int n, int i, int j) +{ + IFDEBUG( + assert(i < n); + assert(j < n); + assert(n > 0); + assert(i < j); + assert(i >= 0); + assert(j >= 0); + ); + return i*n + j - (i*(i+1))/2 - i - 1; +} + +enum Message_tag { tag_result, tag_work }; +enum File_type { file_type_dipha, file_type_txt }; + +int main(int argc, char ** argv) +{ + MPI_Init(NULL, NULL); + + std::string invocation(argv[0]); + + int world_size; + MPI_Comm_size(MPI_COMM_WORLD, &world_size); + + int world_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); + + if (world_size < 2) + { + arg_fail(invocation, world_rank, "Currently there is no support for running with just one process. Please run at least 2 MPI jobs."); + } + + char processor_name_tmp[MPI_MAX_PROCESSOR_NAME]; + int name_len; + MPI_Get_processor_name(processor_name_tmp, &name_len); + std::string processor_name(processor_name_tmp); + + std::string list_file_name; + std::string out_file_name; + int dim = -1; + int in_type = file_type_dipha; + double finitization = std::numeric_limits<double>::infinity(); + int chunk_size = 100; + hera::AuctionParams<double> params; + params.wasserstein_power = std::numeric_limits<double>::quiet_NaN(); + params.delta = std::numeric_limits<double>::quiet_NaN(); + params.internal_p = std::numeric_limits<double>::quiet_NaN(); + params.initial_epsilon = 0.0; // Default value taken from upstream example code. + params.epsilon_common_ratio = 0.0; // Default value taken from upstream example code. + params.max_bids_per_round = std::numeric_limits<size_t>::max(); // Default value taken from upstream example code. + params.gamma_threshold = 0.0; // Default value taken from upstream example code. + params.max_num_phases = 800; // Default value taken from upstream example code. + + for (int i = 1; i < argc; ++i) + { + std::string arg(argv[i]); + if (arg == std::string("--in-list") || arg == std::string("-i")) + { + if (i < argc - 1) + list_file_name = std::string(argv[++i]); + else + arg_fail(invocation, world_rank, "Missing argument for --in-list."); + } + else if (arg == std::string("--in-type") || arg == std::string("-t")) + { + if (i < argc - 1) + { + std::string argnext(argv[++i]); + if (argnext == std::string("txt")) + in_type = file_type_txt; + else if (argnext == std::string("dipha")) + in_type = file_type_dipha; + else + arg_fail(invocation, world_rank, "Invalid argument for --in-type."); + } + else + arg_fail(invocation, world_rank, "Missing argument for --in-type."); + } + else if (arg == std::string("--dimension") || arg == std::string("-d")) + { + if (i < argc - 1) + dim = std::atoi(argv[++i]); + else + arg_fail(invocation, world_rank, "Missing argument for --dimension."); + } + else if (arg == std::string("--outer-norm") || arg == std::string("-p")) + { + if (i < argc - 1) + params.wasserstein_power = std::atof(argv[++i]); + else + arg_fail(invocation, world_rank, "Missing argument for --outer-norm."); + } + else if (arg == std::string("--inner-norm") || arg == std::string("-q")) + { + if (i < argc - 1) + { + std::string argnext(argv[++i]); + if (argnext == "inf") + params.internal_p = -1; + else + params.internal_p = std::stod(argnext); + } + else + arg_fail(invocation, world_rank, "Missing argument for --inner-norm."); + } + else if (arg == std::string("--error") || arg == std::string("-e")) + { + if (i < argc - 1) + params.delta = std::atof(argv[++i]); + else + arg_fail(invocation, world_rank, "Missing argument for --error."); + } + else if (arg == std::string("--finitize") || arg == std::string("-f")) + { + if (i < argc - 1) + finitization = std::atof(argv[++i]); + else + arg_fail(invocation, world_rank, "Missing argument for --finitize."); + } + else if (arg == std::string("--out") || arg == std::string("-o")) + { + if (i < argc - 1) + out_file_name = std::string(argv[++i]); + else + arg_fail(invocation, world_rank, "Missing argument for --out."); + } + else if (arg == std::string("--chunk") || arg == std::string("-c")) + { + if (i < argc - 1) + chunk_size = std::atoi(argv[++i]); + else + arg_fail(invocation, world_rank, "Missing argument for --chunk."); + } + else if (arg == std::string("--help") || arg == std::string("-h")) + { + if (world_rank == 0) + print_help(invocation); + MPI_Finalize(); + exit(0); + } + else + { + arg_fail(invocation, world_rank, "Incorrect argument."); + } + + } + + + // Argument validation. + if (list_file_name == std::string("")) + arg_fail(invocation, world_rank, "Need input file list."); + if (dim < 0) + arg_fail(invocation, world_rank, "Dimension must be non-negative."); + if (params.wasserstein_power < 1 || !std::isfinite(params.wasserstein_power)) + arg_fail(invocation, world_rank, "Outer norm power must be in the interval [1, ∞)."); + if (params.internal_p < 1 || std::isnan(params.internal_p)) + arg_fail(invocation, world_rank, "Inner norm power must be in ther interval [1, ∞]."); + if (params.delta <= 0 || !std::isfinite(params.delta)) + arg_fail(invocation, world_rank, "Error must be positive."); + if (out_file_name == std::string("")) + arg_fail(invocation, world_rank, "Need output file."); + if (chunk_size <= 0) + arg_fail(invocation, world_rank, "Chunk size must be positive."); + + std::vector<std::string> files; + + std::ifstream list_file(list_file_name, std::ios::in); + std::string line; + while (std::getline(list_file, line)) + { + files.push_back(line); + } + list_file.close(); + + int n = files.size(); + + std::vector<std::pair<int, int>> idxs((n*(n-1))/2); + int k = 0; + for (int i = 0; i < n; ++i) + { + for (int j = i+1; j < n; ++j) + { + idxs[k++] = std::make_pair(i, j); + } + } + + std::vector<double> results; + if (world_rank == 0) + { + results.resize(idxs.size(), std::numeric_limits<double>::quiet_NaN()); + double unused_buf = std::numeric_limits<double>::quiet_NaN(); + std::vector<MPI_Request> result_reqs(world_size); // Element zero not used, index by actual rank. + int done = 0; + int next_chunk = 0; + std::vector<std::pair<int, int>> assigned(world_size, std::make_pair(-2, -2)); + + std::cout << "Total things to compute: " << idxs.size() << std::endl; + + for (int r = 1; r < world_size; ++r) + { + MPI_Irecv(&unused_buf, 0, MPI_DOUBLE, r, tag_result, MPI_COMM_WORLD, &(result_reqs[r])); + } + + while ((size_t)done < idxs.size()) + { + int respondent_index = -1; + MPI_Status status; + MPI_Waitany(world_size - 1, &(result_reqs[1]), &respondent_index, &status); + int r = 1 + respondent_index; + + std::cout << "Heard back from rank " << r << "." << std::endl; + int recv_count = -1; + MPI_Get_count(&status, MPI_DOUBLE, &recv_count); + std::cout << "Received " << recv_count << " elements from rank " << r << "." << std::endl; + + assert(recv_count == assigned[r].second - assigned[r].first); + done += recv_count; + + int work[2] = {-1, -1}; + if ((size_t)next_chunk*chunk_size < idxs.size()) + { + work[0] = next_chunk*chunk_size; + work[1] = std::min((int)idxs.size(), (next_chunk + 1)*chunk_size); + MPI_Irecv(&(results[work[0]]), work[1] - work[0], MPI_DOUBLE, r, tag_result, MPI_COMM_WORLD, &(result_reqs[r])); + std::cout << "Rank will get new work." << std::endl; + } + else + std::cout << "Rank will terminate." << std::endl; + + assigned[r] = std::make_pair(work[0], work[1]); + ++next_chunk; + MPI_Send(work, 2, MPI_INT, r, tag_work, MPI_COMM_WORLD); + std::cout << 100*(double)done/(double)idxs.size() << "% complete." << std::endl; + std::cout << "------------------" << std::endl; + } + } + else // Slaves + { + results.resize(chunk_size, std::numeric_limits<double>::quiet_NaN()); + int work[2] = {-2, -2}; + + while (work[0] != -1) + { + int i_prev = -1; + int j_prev = -1; + int i = -1; + int j = -1; + + std::vector<std::pair<double, double> > pd_1; + std::vector<std::pair<double, double> > pd_2; + + for (int k = work[0]; k < work[1]; ++k) + { + i = idxs[k].first; + j = idxs[k].second; + + bool load_success = false; + + if (i != i_prev) + { + pd_1.clear(); + if (in_type == file_type_dipha) + { + load_success = hera::read_diagram_dipha<double, std::vector<std::pair<double, double> > >(files[i], dim, pd_1); + } + else if (in_type == file_type_txt) + { + load_success = hera::read_diagram_point_set<double, std::vector<std::pair<double, double> > >(files[i], pd_1); + } + else + { + fail(world_rank, "Boo"); + } + + if (!load_success) + fail(world_rank, std::string("Failed to load file ") + files[i] + std::string(".")); + + hera::finitize(finitization, pd_1); + } + + if (j != j_prev) + { + pd_2.clear(); + if (in_type == file_type_dipha) + { + load_success = hera::read_diagram_dipha<double, std::vector<std::pair<double, double> > >(files[j], dim, pd_2); + } + else if (in_type == file_type_txt) + { + load_success = hera::read_diagram_point_set<double, std::vector<std::pair<double, double> > >(files[j], pd_2); + } + else + { + fail(world_rank, "Boo"); + } + + if (!load_success) + fail(world_rank, std::string("Failed to load file ") + files[i] + std::string(".")); + + hera::finitize(finitization, pd_2); + } + + std::string fixme(""); + results[k - work[0]] = hera::wasserstein_dist(pd_1, pd_2, params, fixme); + + i_prev = i; + j_prev = j; + } + + MPI_Send(results.data(), work[1] - work[0], MPI_DOUBLE, 0, tag_result, MPI_COMM_WORLD); + MPI_Recv(work, 2, MPI_INT, 0, tag_work, MPI_COMM_WORLD, MPI_STATUS_IGNORE); + } + + } + + + if (world_rank == 0) + { + std::cout << "Writing out." << std::endl; + std::ofstream out_file(out_file_name, std::ios::out); + for (int i = 0; i < n; ++i) + { + for (int j = 0; j < i; ++j) + { + out_file << std::scientific << results[unroll(n, j, i)] << " "; + } + out_file << "0 "; + for (int j = i + 1; j < n; ++j) + { + out_file << std::scientific << results[unroll(n, i, j)] << " "; + } + out_file << std::endl; + } + out_file.close(); + } + + + MPI_Finalize(); +} |