#ifndef BENCHMARK_RUNNER_H #define BENCHMARK_RUNNER_H #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace std; class benchmark_runner { private: string csv_path_; string csv_name_; chrono::steady_clock::time_point last_start_time_; vector times_; bool memory_stats_enabled_{false}; const string MEMORY_PRE_RUN = "memory_pages_pre_run"; const string MEMORY_POST_RUN = "memory_pages_post_run"; unsigned long memory_pre_run_; unsigned long memory_post_run_; map> custom_stats_; void print_statistics() { long time_sum = std::accumulate(times_.begin(), times_.end(), 0l); cout << "Average Runtime (us): " << (time_sum / times_.size()) << endl; for (auto &iter : custom_stats_) { long custom_stat_sum = std::accumulate(iter.second.begin(), iter.second.end(), 0l); cout << "Average " << iter.first << ": " << (custom_stat_sum / times_.size()) << endl; } } inline bool file_exists(const std::string &name) { ifstream f(name); return f.good(); } public: benchmark_runner(string csv_path, string csv_name) : csv_path_{std::move(csv_path)}, csv_name_{std::move(csv_name)}, times_{} { string command = "mkdir -p " + csv_path_; int res = system(command.c_str()); if (res) { cout << "Error while creating directory!" << endl; exit(1); } } /** * Queries the pages used by the current process. * Splits between private and shared pages. * * @return Tuple(private_pages, shared_pages) used by the process. */ static std::pair query_process_memory_pages() { pid_t my_pid = getpid(); std::ostringstream proc_stats_path_builder; proc_stats_path_builder << "/proc/" << my_pid << "/smaps_rollup"; std::string proc_stats_path = proc_stats_path_builder.str(); std::ifstream proc_stats_file{proc_stats_path}; std::string line; unsigned long total_shared = 0; unsigned long total_private = 0; while (std::getline(proc_stats_file, line)) { std::stringstream line_input(line); std::string type; unsigned long size; line_input >> type >> size; if (type.find("Shared") != std::string::npos) { total_shared += size; } if (type.find("Private") != std::string::npos) { total_private += size; } } return std::make_pair(total_private, total_shared); } void add_custom_stats_field(const string name) { custom_stats_.insert({name, {}}); } void enable_memory_stats() { memory_stats_enabled_ = true; add_custom_stats_field(MEMORY_PRE_RUN); add_custom_stats_field(MEMORY_POST_RUN); } void pre_allocate_stats(size_t num = 100000) { times_.reserve(num); memset(times_.data(), 'a', num * sizeof(long)); for (auto &iter : custom_stats_) { iter.second.reserve(num); memset(iter.second.data(), 'a', num * sizeof(long)); } } static void read_args(int argc, char **argv, int &num_threads, string &path) { if (argc < 3) { cout << "Must Specifiy concurrency and output directory! (usage: `benchmark `)" << endl; exit(1); } string tmp = argv[1]; path = tmp; num_threads = atoi(argv[2]); } void start_iteration() { if (memory_stats_enabled_) { auto memory_stats = query_process_memory_pages(); memory_pre_run_ = memory_stats.first; } last_start_time_ = chrono::steady_clock::now(); } void end_iteration() { size_t iteration_index = times_.size(); auto end_time = chrono::steady_clock::now(); long time = chrono::duration_cast(end_time - last_start_time_).count(); times_.emplace_back(time); for (auto &iter : custom_stats_) { iter.second.emplace_back(0); } if (memory_stats_enabled_) { auto memory_stats = query_process_memory_pages(); memory_post_run_ = memory_stats.first; custom_stats_[MEMORY_PRE_RUN][iteration_index] = memory_pre_run_; custom_stats_[MEMORY_POST_RUN][iteration_index] = memory_post_run_; } } void store_custom_stat(const string &name, long value) { auto &stat_vector = custom_stats_[name]; stat_vector[stat_vector.size() - 1] = value; } void run_iterations(int count, const function measure, int warmup_count, const function prepare = []() {}, const function finish = []() {}) { for (int i = 0; i < warmup_count; i++) { prepare(); measure(); } for (int i = 0; i < count; i++) { using namespace std::literals; this_thread::sleep_for(100us); prepare(); start_iteration(); measure(); end_iteration(); finish(); } } void commit_results(bool print_stats) { if (print_stats) { print_statistics(); } string full_filename = csv_path_ + csv_name_; bool write_header = !file_exists(full_filename); { // Scope for output file ofstream o(full_filename, std::fstream::out | std::fstream::app); if (write_header) { o << "runtime_us"; for (auto &iter : custom_stats_) { o << ";" << iter.first; } o << endl; } // TODO: make this more efficient for (size_t i = 0; i < times_.size(); i++) { o << times_[i]; for (auto &iter : custom_stats_) { o << ";" << iter.second[i]; } o << endl; } } // End Scope for output file times_.clear(); for (auto &iter : custom_stats_) { iter.second.clear(); } } }; #endif //BENCHMARK_RUNNER_H