From b385b3b1c8897ac82f9f2a011956b47e7d18cbe0 Mon Sep 17 00:00:00 2001 From: ZY4N Date: Sun, 22 Dec 2024 21:52:05 +0100 Subject: [PATCH] further fixes --- .../prefetch_lookups/pose_prefetch_lookup.hpp | 2 - source/assets/data_loaders/kitti_loader.cpp | 7 +- .../assets/data_loaders/kitti_pose_loader.cpp | 75 ++---- source/assets/data_loaders/obj_loader.cpp | 227 +++++++++++++++--- .../prefetch_lookups/pose_prefetch_lookup.cpp | 18 +- 5 files changed, 214 insertions(+), 115 deletions(-) diff --git a/include/assets/prefetch_lookups/pose_prefetch_lookup.hpp b/include/assets/prefetch_lookups/pose_prefetch_lookup.hpp index 5204764..b80cfc0 100644 --- a/include/assets/prefetch_lookups/pose_prefetch_lookup.hpp +++ b/include/assets/prefetch_lookups/pose_prefetch_lookup.hpp @@ -24,7 +24,6 @@ public: void emplace_hint_dir( directory_iterator directory_it, - const std::filesystem::path& directory, index_type index, dynamic_pose_store::id_type id ); @@ -45,7 +44,6 @@ public: index_type index ); -protected: directory_iterator emplace_dir( directory_iterator directory_it, const std::filesystem::path& directory diff --git a/source/assets/data_loaders/kitti_loader.cpp b/source/assets/data_loaders/kitti_loader.cpp index 71b73a9..c7bc3d5 100644 --- a/source/assets/data_loaders/kitti_loader.cpp +++ b/source/assets/data_loaders/kitti_loader.cpp @@ -115,6 +115,7 @@ std::error_code kitti_loader::load( continue; } + // TODO find out if the path splitting is consistent const auto base_directory = parent_directory(file.substr(0, sep_index)); if (not base_directory) [[unlikely]] { @@ -162,14 +163,12 @@ std::error_code kitti_loader::load( { const auto file_path = std::string_view{ file.path().c_str() }; - const auto extension_begin = file_path.rfind('.'); - - if (extension_begin == std::string_view::npos or file_path.substr(extension_begin) != ".bin") + if (not file_path.ends_with(".bin")) { continue; } - auto filename_begin = file_path.rfind(fs::path::preferred_separator, extension_begin); + auto filename_begin = file_path.rfind(fs::path::preferred_separator); filename_begin = filename_begin == std::string_view::npos ? 0 : filename_begin + 1; const auto filename_only = file_path.substr(filename_begin); diff --git a/source/assets/data_loaders/kitti_pose_loader.cpp b/source/assets/data_loaders/kitti_pose_loader.cpp index d256b8a..6fe48e5 100644 --- a/source/assets/data_loaders/kitti_pose_loader.cpp +++ b/source/assets/data_loaders/kitti_pose_loader.cpp @@ -42,9 +42,9 @@ std::error_code kitti_pose_loader::load( auto path_buffer = fs::path{}; auto in = std::ifstream{}; // TODO disable exceptions (for other loaders as well) - auto pose_buffer = dynamic_pose_buffer{}; - pose_buffer = glm::identity(); + // Needed to initialize 4th row and col + buffer = glm::identity(); auto processed_filenames = ztu::string_list{}; @@ -62,13 +62,12 @@ std::error_code kitti_pose_loader::load( ztu::logger::error("Kitti pose file does not exist: %", path_buffer); return; } - processed_filenames.push_back(path_buffer.c_str()); }; for (const auto directory : paths.directories) { path_buffer.assign(directory.begin(), directory.end()); - path_buffer /= "pose.txt"; + path_buffer /= pose_filename; preprocess_file(); } @@ -79,6 +78,16 @@ std::error_code kitti_pose_loader::load( for (const auto filename : processed_filenames) { + // TODO if (not) removing the path separator creates issues. + const auto directory = filename.substr(0, filename.length() - pose_filename.length()); + + auto [ dir_it, dir_match ] = id_lookup.poses.find_directory(directory); + + if (not dir_match) [[unlikely]] + { + dir_it = id_lookup.poses.emplace_dir(dir_it, directory); + } + in.open(filename.data()); // Safe because string list adds null terminator if (not in.is_open()) { @@ -90,7 +99,7 @@ std::error_code kitti_pose_loader::load( for (auto i = pose_prefetch_lookup::index_type{}; in.peek() != std::ifstream::traits_type::eof(); ++i) { - if (const auto error = parse_pose(in, pose_buffer)) + if (const auto error = parse_pose(in, buffer)) { ztu::logger::error( "Error occurred while parsing kitti pose % in file %: [%] %", @@ -102,62 +111,10 @@ std::error_code kitti_pose_loader::load( continue; } - const auto id = store.poses.add(pose_buffer); - - // TODO if (not) removing the path separator creates issues. - const auto directory = filename.substr(0, filename.length() - pose_filename.length()); - - id_lookup.poses.emplace(directory, i, id); + const auto id = store.poses.add(buffer); + id_lookup.poses.emplace_hint_dir(dir_it, i, id); } in.close(); } } - -void kitti_pose_loader::load( - const ztu::string_list& directories, - dynamic_pose_store& store, - pose_prefetch_lookup& id_lookup -) { - - auto filename_buffer = std::filesystem::path{}; - auto in = std::ifstream{}; // TODO disable exceptions (for other loaders as well) - auto pose_buffer = dynamic_pose_buffer{}; - - pose_buffer = glm::identity(); - - for (const auto directory : directories) - { - filename_buffer = directory; - filename_buffer /= "pose.txt"; - - in.open(filename_buffer); - if (not in.is_open()) - { - ztu::logger::error("Cannot open kitti pose file %", filename_buffer); - continue; - } - - in >> std::skipws; - - for (auto i = pose_prefetch_lookup::index_type{}; in.peek() != std::ifstream::traits_type::eof(); ++i) - { - if (const auto error = parse_pose(in, pose_buffer)) - { - ztu::logger::error( - "Error occurred while parsing kitti pose % in file %: [%] %", - i, - filename_buffer, - error.category().name(), - error.message() - ); - continue; - } - - const auto id = store.add(pose_buffer); - id_lookup.emplace(directory, i, id); - } - - in.close(); - } -} \ No newline at end of file diff --git a/source/assets/data_loaders/obj_loader.cpp b/source/assets/data_loaders/obj_loader.cpp index b15b2cd..df1a579 100755 --- a/source/assets/data_loaders/obj_loader.cpp +++ b/source/assets/data_loaders/obj_loader.cpp @@ -61,9 +61,10 @@ inline std::error_code make_error_code(codes e) { } // namespace mesh_loader_error -using vertex_type = std::array; +using vertex_type = std::array; -struct indexed_vertex_type { +struct indexed_vertex_type +{ vertex_type vertex; ztu::u32 buffer_index; @@ -102,6 +103,121 @@ std::errc parse_numeric_vector(std::string_view param, std::array& val return {}; }; + +void find_materials( + std::span buffer, + std::ifstream& in, + ztu::string_list& material_filenames +) { + static constexpr auto keyword = std::string_view{ "\nmtllib " }; + + const auto buffer_view = std::string_view(buffer); + + // Add linebreak to simplify line begin search + buffer.front() = '\n'; + auto leftover = std::size_t{ 1 }; + + do + { + // Keep some old characters to continue matching interrupted sequence + std::copy(buffer.end() - leftover, buffer.end(), buffer.begin()); + + in.read(buffer.data() + leftover, buffer.size() - leftover); + + const auto str = buffer_view.substr(0, leftover + in.gcount()); + + leftover = keyword.size(); + + auto pos = std::string_view::size_type{}; + while ((pos = str.find(keyword, pos)) != std::string_view::npos) + { + const auto filename_begin = pos + keyword.size(); + const auto filename_end = str.find('\n', filename_begin); + + if (filename_end != std::string_view::npos) + { + const auto length = filename_end - filename_begin; + const auto material_filename = str.substr(filename_begin, length); + + // TODO get base dir from param figure out how to avoid heap + if (material_filename.is_relative()) + { + material_filename = directory / material_filename; + } + + material_filenames.push_back(material_filename); + + pos = filename_end; + } + else // string exceeds the buffer + { + if (pos == 0) [[unlikely]] + { + std::cout << "Ignoring string match, as it exceeds buffer size." << std::endl; + leftover = 0; + } + else + { + leftover = str.size() - pos; + } + break; + } + } + } while (not in.eof()); +} + + +std::error_code obj_loader::prefetch( + const file_dir_list& paths, + prefetch_queue& queue +) { + namespace fs = std::filesystem; + using obj_loader_error::codes; + using obj_loader_error::make_error_code; + + auto buffer = std::vector(8 * 1024, '\0'); + + auto in = std::ifstream{}; + + const auto parse_file = [&](const char* filename) + { + in.open(filename); + if (not in.is_open()) { + ztu::logger::error("Could not open .obj file '%'", filename); + return; + } + + find_materials(buffer, in, queue.mtl_queue.files); + + in.close(); + }; + + for (const auto file : paths.files) + { + // `file` is null-terminates by list. + parse_file(file.data()); + } + + for (const auto directory : paths.directories) + { + for (const auto& file : fs::directory_iterator{ directory }) { + + const auto& file_path = std::string_view{ file.path().c_str() }; + + // TODO remove heap allocation + if (not file_path.ends_with(".obj")) + { + continue; + } + + // Null terminated by fs::path. + parse_file(file_path.data()); + } + } + + return {}; +} + std::error_code obj_loader::load_directory( dynamic_data_loader_ctx& ctx, dynamic_mesh_store& store, @@ -144,15 +260,67 @@ std::error_code obj_loader::load_directory( return {}; } +[[nodiscard]] static std::error_code load( + components::mesh_vertex::flags enabled_components, + dynamic_mesh_buffer& buffer, + const file_dir_list& paths, + prefetch_lookup& id_lookup, + dynamic_data_store& store, + bool pedantic = false +) { + + + auto in = std::ifstream{}; + std::set vertex_ids; + + const auto parse_file = [&](const char* filename) + { + in.open(filename); + if (not in.is_open()) { + ztu::logger::error("Could not open .obj file '%'", filename); + return; + } + + + vertex_ids.clear(); + + in.close(); + }; + + for (const auto file : paths.files) + { + // `file` is null-terminates by list. + parse_file(file.data()); + } + + for (const auto directory : paths.directories) + { + for (const auto& file : fs::directory_iterator{ directory }) { + + const auto& file_path = std::string_view{ file.path().c_str() }; + + // TODO remove heap allocation + if (not file_path.ends_with(".obj")) + { + continue; + } + + // Null terminated by fs::path. + parse_file(file_path.data()); + } + } + +} // TODO refactor so there is a function like parse_normals etc. std::error_code obj_loader::load( - dynamic_data_loader_ctx& ctx, - dynamic_mesh_store& store, components::mesh_vertex::flags enabled_components, - const std::filesystem::path& filename, - const bool pedantic + dynamic_mesh_buffer& buffer, + const char* filename, + prefetch_lookup& id_lookup, + dynamic_data_store& store, + bool pedantic ) { using obj_loader_error::codes; using obj_loader_error::make_error_code; @@ -173,33 +341,26 @@ std::error_code obj_loader::load( // and only push unique combinations to the buffers. std::set vertex_ids; - auto mesh = dynamic_mesh_data{}; - // Buffers - auto position_buffer = mesh.positions(); - auto normal_buffer = mesh.normals(); - auto tex_coord_buffer = mesh.tex_coords(); + // TODO find out if this is still relevant + auto position_buffer = buffer.positions(); + auto normal_buffer = buffer.normals(); + auto tex_coord_buffer = buffer.tex_coords(); std::unordered_map material_name_lookup; - - constexpr auto mtl_loader_id = *ctx.material_loader.find_loader_static("mtl"); - mtl_loader& material_loader = ctx.material_loader.get_loader(); - - material_loader.clear_name_lookup(); - std::string material_name; const auto push_mesh = [&](const bool clear_buffers = false) { - if (not mesh.positions().empty()) + if (not buffer.positions().empty()) { // Copy buffers instead of moving to keep capacity for further parsing // and have the final buffers be shrunk to size. if (not material_name.empty()) { if (const auto id = material_loader.find_id(material_name)) { - mesh.material_id() = *id; + buffer.material_id() = *id; } else { @@ -214,24 +375,24 @@ std::error_code obj_loader::load( ztu::logger::debug("Parsed % normals.", mesh.normals().size()); ztu::logger::debug("Parsed % tex_coords.", mesh.tex_coords().size()); - if (not mesh.positions().empty()) + if (not buffer.positions().empty()) { - mesh.components() |= components::mesh_vertex::flags::position; + buffer.components() |= components::mesh_vertex::flags::position; } - if (not mesh.normals().empty()) + if (not buffer.normals().empty()) { - mesh.components() |= components::mesh_vertex::flags::normal; + buffer.components() |= components::mesh_vertex::flags::normal; } - if (not mesh.tex_coords().empty()) + if (not buffer.tex_coords().empty()) { - mesh.components() |= components::mesh_vertex::flags::tex_coord; + buffer.components() |= components::mesh_vertex::flags::tex_coord; } - ztu::logger::debug("Pushing obj mesh with % triangles.", mesh.triangles().size()); + ztu::logger::debug("Pushing obj mesh with % triangles.", buffer.triangles().size()); - store.add(std::move(mesh)); + store.meshes.add(buffer); } if (clear_buffers) @@ -241,7 +402,7 @@ std::error_code obj_loader::load( tex_coord_buffer.clear(); } - mesh = dynamic_mesh_data{}; + buffer.clear(); vertex_ids.clear(); material_name.clear(); @@ -251,7 +412,7 @@ std::error_code obj_loader::load( auto indexed_vid = indexed_vertex_type{ .vertex = vertex, - .buffer_index = static_cast(mesh.positions().size()) + .buffer_index = static_cast(buffer.positions().size()) }; // Search through sorted lookup to check if index combination is unique @@ -263,17 +424,17 @@ std::error_code obj_loader::load( if (position_index < position_buffer.size()) { - mesh.positions().emplace_back(position_buffer[position_index]); + buffer.positions().emplace_back(position_buffer[position_index]); } if (normal_index < normal_buffer.size()) { - mesh.normals().emplace_back(normal_buffer[normal_index]); + buffer.normals().emplace_back(normal_buffer[normal_index]); } if (tex_coord_index < tex_coord_buffer.size()) { - mesh.tex_coords().emplace_back(tex_coord_buffer[tex_coord_index]); + buffer.tex_coords().emplace_back(tex_coord_buffer[tex_coord_index]); } } @@ -437,8 +598,6 @@ std::error_code obj_loader::load( }) ); - material_loader.clear_name_lookup(); - if (ec != codes::ok) { return make_error_code(ec); diff --git a/source/assets/prefetch_lookups/pose_prefetch_lookup.cpp b/source/assets/prefetch_lookups/pose_prefetch_lookup.cpp index df551c6..6e98a9e 100644 --- a/source/assets/prefetch_lookups/pose_prefetch_lookup.cpp +++ b/source/assets/prefetch_lookups/pose_prefetch_lookup.cpp @@ -70,19 +70,8 @@ void pose_prefetch_lookup::emplace( directory_it = emplace_dir(directory_it, directory); } - const auto [ index_it, match ] = find_index( + emplace_hint_dir( directory_it, - index - ); - - if (not match) - { - - } - - emplace_hint_dir_index( - directory_it, - index_it, index, id ); @@ -91,13 +80,10 @@ void pose_prefetch_lookup::emplace( void pose_prefetch_lookup::emplace_hint_dir( directory_iterator directory_it, - const std::filesystem::path& directory, const index_type index, const dynamic_pose_store::id_type id ) { - directory_it = emplace_dir(directory_it, directory); - const auto [ index_it, match ] = find_index( directory_it, index @@ -105,7 +91,7 @@ void pose_prefetch_lookup::emplace_hint_dir( if (match) [[unlikely]] { - const auto dir_index = dir_it->second; + const auto dir_index = directory_it.first->second; const auto id_index = index_it - m_directory_indices.begin() - dir_index; m_pose_ids[id_index] = match; // TODO I guess I should warn?!? }