From 447146b7f5e572b17a418bca52b9a70ff5694a98 Mon Sep 17 00:00:00 2001 From: ZY4N Date: Thu, 26 Dec 2024 20:20:02 +0100 Subject: [PATCH] worked on obj and mtl prefetching nad parsing --- include/assets/data_loaders/mtl_loader.hpp | 40 +- include/assets/data_loaders/obj_loader.hpp | 56 ++- .../dynamic_material_buffer.hpp | 34 +- .../dynamic_mesh_buffer.hpp | 32 +- .../dynamic_point_cloud_buffer.hpp | 20 +- .../dynamic_shader_buffer.hpp | 6 + .../dynamic_texture_buffer.hpp | 44 +- .../{ => generic}/dynamic_vertex_buffer.hpp | 15 +- source/assets/data_loaders/kitti_loader.cpp | 2 +- source/assets/data_loaders/mtl_loader.cpp | 260 +++++++++-- source/assets/data_loaders/obj_loader.cpp | 440 ++++++++---------- .../dynamic_texture_loader.cpp | 2 - .../dynamic_material_buffer.ipp | 11 + .../dynamic_mesh_buffer.ipp | 7 + .../dynamic_point_cloud_buffer.ipp | 5 + .../dynamic_texture_buffer.ipp | 8 + 16 files changed, 606 insertions(+), 376 deletions(-) rename include/assets/dynamic_read_buffers/{ => generic}/dynamic_vertex_buffer.hpp (56%) diff --git a/include/assets/data_loaders/mtl_loader.hpp b/include/assets/data_loaders/mtl_loader.hpp index dd2df5f..db8d53a 100644 --- a/include/assets/data_loaders/mtl_loader.hpp +++ b/include/assets/data_loaders/mtl_loader.hpp @@ -12,30 +12,25 @@ #include "util/string_lookup.hpp" #include "util/result.hpp" -namespace mtl_loader_error { - +namespace mtl_loader_error +{ enum class codes { ok = 0, - mtl_cannot_open_file, - mtl_cannot_open_texture, - mtl_malformed_ambient_color, - mtl_malformed_diffuse_color, - mtl_malformed_specular_color, - mtl_malformed_specular_exponent, - mtl_malformed_dissolve, - mlt_unknown_line_begin + cannot_open_file, + cannot_open_texture, + malformed_ambient_color, + malformed_diffuse_color, + malformed_specular_color, + malformed_specular_exponent, + malformed_dissolve, + unknown_line_begin }; - } // namespace mtl_loader_error -class mtl_loader { -public: +struct mtl_loader +{ static constexpr auto name = std::string_view("mtl"); - std::optional find_id(std::string_view name); - - void clear_name_lookup(); - [[nodiscard]] static std::error_code prefetch( const file_dir_list& paths, prefetch_queue& queue @@ -50,7 +45,12 @@ public: bool pedantic = false ); - -private: - ztu::string_lookup m_id_lookup; +protected: + static void find_textures( + std::span buffer, + std::filesystem::path& path_buffer, + const std::filesystem::path& base_directory, + std::ifstream& in, + ztu::string_list& texture_filenames + ); }; diff --git a/include/assets/data_loaders/obj_loader.hpp b/include/assets/data_loaders/obj_loader.hpp index 7bd95e6..a0aedd4 100755 --- a/include/assets/data_loaders/obj_loader.hpp +++ b/include/assets/data_loaders/obj_loader.hpp @@ -6,18 +6,21 @@ #include "assets/dynamic_data_loaders/dynamic_material_loader.hpp" #include "assets/dynamic_data_stores/dynamic_mesh_store.hpp" #include "assets/prefetch_lookup.hpp" +#include namespace obj_loader_error { enum class codes { ok = 0, - obj_cannot_open_file, - obj_malformed_vertex, - obj_malformed_texture_coordinate, - obj_malformed_normal, - obj_malformed_face, - obj_face_index_out_of_range, - obj_unknown_line_begin + cannot_open_file, + malformed_vertex, + malformed_texture_coordinate, + malformed_normal, + malformed_face, + face_index_out_of_range, + unknown_line_begin, + use_material_without_material_library, + unknown_material_name }; } // namespace obj_loader_error @@ -32,11 +35,48 @@ struct obj_loader { ); [[nodiscard]] static std::error_code load( - components::mesh_vertex::flags enabled_components, dynamic_mesh_buffer& buffer, const file_dir_list& paths, prefetch_lookup& id_lookup, dynamic_data_store& store, bool pedantic = false ); + +protected: + using index_type = dynamic_mesh_buffer::index_type; + using vertex_type = std::array; + + struct indexed_vertex_type + { + vertex_type vertex; + index_type buffer_index; + + friend auto operator<=>(const indexed_vertex_type& a, const indexed_vertex_type& b) { + return a.vertex <=> b.vertex; + } + + bool operator==(const indexed_vertex_type& other) const noexcept { + return other.vertex == vertex; + } + }; + + static void find_materials( + std::span buffer, + std::filesystem::path& path_buffer, + const std::filesystem::path& base_directory, + std::ifstream& in, + ztu::string_list& material_filenames + ); + + [[nodiscard]] static std::error_code obj_loader::parse_file( + dynamic_mesh_buffer& read_buffer, + dynamic_mesh_buffer& mesh_buffer, + std::filesystem::path& path_buffer, + const std::filesystem::path& base_directory, + std::set& vertex_ids, + std::ifstream& in, + prefetch_lookup& id_lookup, + dynamic_data_store& store, + bool pedantic + ); }; diff --git a/include/assets/dynamic_read_buffers/dynamic_material_buffer.hpp b/include/assets/dynamic_read_buffers/dynamic_material_buffer.hpp index d52e5b6..03dc4c1 100755 --- a/include/assets/dynamic_read_buffers/dynamic_material_buffer.hpp +++ b/include/assets/dynamic_read_buffers/dynamic_material_buffer.hpp @@ -11,23 +11,25 @@ struct dynamic_material_buffer { components::material::surface_properties& initialized_surface_properties(); - [[nodiscard]] std::optional& surface_properties(); - [[nodiscard]] std::optional& transparency(); - [[nodiscard]] std::optional& ambient_color_texture_id(); - [[nodiscard]] std::optional& diffuse_color_texture_id(); - [[nodiscard]] std::optional& specular_color_texture_id(); - [[nodiscard]] std::optional& shininess_texture_id(); - [[nodiscard]] std::optional& alpha_texture_id(); - [[nodiscard]] std::optional& bump_texture_id(); + [[nodiscard]] inline std::optional& surface_properties(); + [[nodiscard]] inline std::optional& transparency(); + [[nodiscard]] inline std::optional& ambient_color_texture_id(); + [[nodiscard]] inline std::optional& diffuse_color_texture_id(); + [[nodiscard]] inline std::optional& specular_color_texture_id(); + [[nodiscard]] inline std::optional& shininess_texture_id(); + [[nodiscard]] inline std::optional& alpha_texture_id(); + [[nodiscard]] inline std::optional& bump_texture_id(); - [[nodiscard]] const std::optional& surface_properties() const; - [[nodiscard]] const std::optional& transparency() const; - [[nodiscard]] const std::optional& ambient_color_texture_id() const; - [[nodiscard]] const std::optional& diffuse_color_texture_id() const; - [[nodiscard]] const std::optional& specular_color_texture_id() const; - [[nodiscard]] const std::optional& shininess_texture_id() const; - [[nodiscard]] const std::optional& alpha_texture_id() const; - [[nodiscard]] const std::optional& bump_texture_id() const; + [[nodiscard]] inline const std::optional& surface_properties() const; + [[nodiscard]] inline const std::optional& transparency() const; + [[nodiscard]] inline const std::optional& ambient_color_texture_id() const; + [[nodiscard]] inline const std::optional& diffuse_color_texture_id() const; + [[nodiscard]] inline const std::optional& specular_color_texture_id() const; + [[nodiscard]] inline const std::optional& shininess_texture_id() const; + [[nodiscard]] inline const std::optional& alpha_texture_id() const; + [[nodiscard]] inline const std::optional& bump_texture_id() const; + + inline void clear(); std::tuple< std::optional, diff --git a/include/assets/dynamic_read_buffers/dynamic_mesh_buffer.hpp b/include/assets/dynamic_read_buffers/dynamic_mesh_buffer.hpp index b8491aa..370f9cd 100644 --- a/include/assets/dynamic_read_buffers/dynamic_mesh_buffer.hpp +++ b/include/assets/dynamic_read_buffers/dynamic_mesh_buffer.hpp @@ -5,7 +5,7 @@ #include "util/uix.hpp" #include "assets/components/mesh_vertex_components.hpp" -#include "assets/dynamic_read_buffers/dynamic_vertex_buffer.hpp" +#include "generic/dynamic_vertex_buffer.hpp" #include "assets/dynamic_data_stores/dynamic_material_store.hpp" class dynamic_mesh_buffer : public dynamic_vertex_buffer< @@ -20,21 +20,23 @@ public: using index_type = ztu::u32; using triangle_type = std::array; - [[nodiscard]] std::vector& positions(); - [[nodiscard]] std::vector& normals(); - [[nodiscard]] std::vector& tex_coords(); - [[nodiscard]] std::vector& colors(); - [[nodiscard]] std::vector& reflectances(); - [[nodiscard]] std::vector& triangles(); - [[nodiscard]] auto& material_id(); + [[nodiscard]] inline std::vector& positions(); + [[nodiscard]] inline std::vector& normals(); + [[nodiscard]] inline std::vector& tex_coords(); + [[nodiscard]] inline std::vector& colors(); + [[nodiscard]] inline std::vector& reflectances(); + [[nodiscard]] inline std::vector& triangles(); + [[nodiscard]] inline auto& material_id(); - [[nodiscard]] const std::vector& positions() const; - [[nodiscard]] const std::vector& normals() const; - [[nodiscard]] const std::vector& tex_coords() const; - [[nodiscard]] const std::vector& colors() const; - [[nodiscard]] const std::vector& reflectances() const; - [[nodiscard]] const std::vector& triangles() const; - [[nodiscard]] const auto& material_id() const; + [[nodiscard]] inline const std::vector& positions() const; + [[nodiscard]] inline const std::vector& normals() const; + [[nodiscard]] inline const std::vector& tex_coords() const; + [[nodiscard]] inline const std::vector& colors() const; + [[nodiscard]] inline const std::vector& reflectances() const; + [[nodiscard]] inline const std::vector& triangles() const; + [[nodiscard]] inline const auto& material_id() const; + + inline void clear(); private: std::vector m_triangles{}; diff --git a/include/assets/dynamic_read_buffers/dynamic_point_cloud_buffer.hpp b/include/assets/dynamic_read_buffers/dynamic_point_cloud_buffer.hpp index 3ff27b5..142bc72 100644 --- a/include/assets/dynamic_read_buffers/dynamic_point_cloud_buffer.hpp +++ b/include/assets/dynamic_read_buffers/dynamic_point_cloud_buffer.hpp @@ -4,7 +4,7 @@ #include #include -#include "assets/dynamic_read_buffers/dynamic_vertex_buffer.hpp" +#include "generic/dynamic_vertex_buffer.hpp" class dynamic_point_cloud_buffer : public dynamic_vertex_buffer< components::point_cloud_vertex::flags, @@ -14,15 +14,17 @@ class dynamic_point_cloud_buffer : public dynamic_vertex_buffer< components::point_cloud_vertex::reflectance > { public: - [[nodiscard]] std::vector& positions(); - [[nodiscard]] std::vector& normals(); - [[nodiscard]] std::vector& colors(); - [[nodiscard]] std::vector& reflectances(); + [[nodiscard]] inline std::vector& positions(); + [[nodiscard]] inline std::vector& normals(); + [[nodiscard]] inline std::vector& colors(); + [[nodiscard]] inline std::vector& reflectances(); - [[nodiscard]] const std::vector& positions() const; - [[nodiscard]] const std::vector& normals() const; - [[nodiscard]] const std::vector& colors() const; - [[nodiscard]] const std::vector& reflectances() const; + [[nodiscard]] inline const std::vector& positions() const; + [[nodiscard]] inline const std::vector& normals() const; + [[nodiscard]] inline const std::vector& colors() const; + [[nodiscard]] inline const std::vector& reflectances() const; + + inline void clear(); }; #define INCLUDE_DYNAMIC_TEXTURE_DATA_IMPLEMENTATION diff --git a/include/assets/dynamic_read_buffers/dynamic_shader_buffer.hpp b/include/assets/dynamic_read_buffers/dynamic_shader_buffer.hpp index 5fb00d6..9a3f46a 100644 --- a/include/assets/dynamic_read_buffers/dynamic_shader_buffer.hpp +++ b/include/assets/dynamic_read_buffers/dynamic_shader_buffer.hpp @@ -7,4 +7,10 @@ struct dynamic_shader_buffer { std::vector source{}; GLenum type{ GL_INVALID_ENUM }; + + inline void clear() + { + source.clear(); + type = GL_INVALID_ENUM; + } }; diff --git a/include/assets/dynamic_read_buffers/dynamic_texture_buffer.hpp b/include/assets/dynamic_read_buffers/dynamic_texture_buffer.hpp index 267b97f..4437397 100755 --- a/include/assets/dynamic_read_buffers/dynamic_texture_buffer.hpp +++ b/include/assets/dynamic_read_buffers/dynamic_texture_buffer.hpp @@ -21,50 +21,52 @@ public: dynamic_texture_buffer() = default; - dynamic_texture_buffer( + inline dynamic_texture_buffer( std::unique_ptr&& data, dim_type width, dim_type height, components::texture::flags components - ); + );; - dynamic_texture_buffer(const dynamic_texture_buffer&); + inline dynamic_texture_buffer(const dynamic_texture_buffer&); - dynamic_texture_buffer(dynamic_texture_buffer&&) noexcept; + inline dynamic_texture_buffer(dynamic_texture_buffer&&) noexcept; - [[nodiscard]] dynamic_texture_buffer& operator=(const dynamic_texture_buffer&); + [[nodiscard]] inline dynamic_texture_buffer& operator=(const dynamic_texture_buffer&); - [[nodiscard]] dynamic_texture_buffer& operator=(dynamic_texture_buffer&&) noexcept; + [[nodiscard]] inline dynamic_texture_buffer& operator=(dynamic_texture_buffer&&) noexcept; - [[nodiscard]] components::texture::flags components() const; + [[nodiscard]] inline components::texture::flags components() const; - [[nodiscard]] dim_type width() const; + [[nodiscard]] inline dim_type width() const; - [[nodiscard]] dim_type height() const; + [[nodiscard]] inline dim_type height() const; - [[nodiscard]] std::pair dimensions() const; + [[nodiscard]] inline std::pair dimensions() const; - [[nodiscard]] size_type pixel_count() const; + [[nodiscard]] inline size_type pixel_count() const; - [[nodiscard]] size_type component_count() const; + [[nodiscard]] inline size_type component_count() const; - [[nodiscard]] size_type size() const; + [[nodiscard]] inline size_type size() const; - [[nodiscard]] const_iterator begin() const; + [[nodiscard]] inline const_iterator begin() const; - [[nodiscard]] iterator begin(); + [[nodiscard]] inline iterator begin(); - [[nodiscard]] const_iterator end() const; + [[nodiscard]] inline const_iterator end() const; - [[nodiscard]] iterator end(); + [[nodiscard]] inline iterator end(); - [[nodiscard]] const_iterator cbegin() const; + [[nodiscard]] inline const_iterator cbegin() const; - [[nodiscard]] const_iterator cend() const; + [[nodiscard]] inline const_iterator cend() const; - [[nodiscard]] const_pointer data() const; + [[nodiscard]] inline const_pointer data() const; - [[nodiscard]] pointer data(); + [[nodiscard]] inline pointer data(); + + void inline clear(); private: std::unique_ptr m_data{ nullptr }; diff --git a/include/assets/dynamic_read_buffers/dynamic_vertex_buffer.hpp b/include/assets/dynamic_read_buffers/generic/dynamic_vertex_buffer.hpp similarity index 56% rename from include/assets/dynamic_read_buffers/dynamic_vertex_buffer.hpp rename to include/assets/dynamic_read_buffers/generic/dynamic_vertex_buffer.hpp index 31dd353..c18d138 100644 --- a/include/assets/dynamic_read_buffers/dynamic_vertex_buffer.hpp +++ b/include/assets/dynamic_read_buffers/generic/dynamic_vertex_buffer.hpp @@ -8,8 +8,15 @@ template struct dynamic_vertex_buffer { std::tuple...> vertices{}; -}; -#define INCLUDE_DYNAMIC_MODEL_DATA_IMPLEMENTATION -#include "assets/dynamic_read_buffers/dynamic_texture_buffer.ipp" -#undef INCLUDE_DYNAMIC_MODEL_DATA_IMPLEMENTATION +protected: + inline void clear_vertices() + { + std::apply( + [](auto&... vertex_opt) { + (vertex_opt.clear(), ...); + }, + vertices + ); + } +}; diff --git a/source/assets/data_loaders/kitti_loader.cpp b/source/assets/data_loaders/kitti_loader.cpp index c7bc3d5..2a7d683 100644 --- a/source/assets/data_loaders/kitti_loader.cpp +++ b/source/assets/data_loaders/kitti_loader.cpp @@ -208,7 +208,7 @@ std::error_code kitti_loader::load( } - for (const auto [ filename, pose_it ] : std::ranges::views::zip_view(processed_filenames, pose_its)) + for (const auto [ filename, pose_it ] : std::ranges::views::zip(processed_filenames, pose_its)) { buffer.clear(); diff --git a/source/assets/data_loaders/mtl_loader.cpp b/source/assets/data_loaders/mtl_loader.cpp index 040d411..5a2b209 100644 --- a/source/assets/data_loaders/mtl_loader.cpp +++ b/source/assets/data_loaders/mtl_loader.cpp @@ -10,31 +10,35 @@ #include "assets/dynamic_data_loaders/dynamic_texture_loader.hpp" -namespace mtl_loader_error { - -struct category : std::error_category { - [[nodiscard]] const char* name() const noexcept override { - return "connector"; +namespace mtl_loader_error +{ +struct category : std::error_category +{ + [[nodiscard]] const char* name() const noexcept override + { + return "mtl_loader"; } - [[nodiscard]] std::string message(int ev) const override { - switch (static_cast(ev)) { + [[nodiscard]] std::string message(int ev) const override + { + switch (static_cast(ev)) + { using enum codes; - case mtl_cannot_open_file: + case cannot_open_file: return "Cannot open mtl file."; - case mtl_cannot_open_texture: + case cannot_open_texture: return "Cannot open texture file."; - case mtl_malformed_ambient_color: + case malformed_ambient_color: return "File contains malformed 'Ka' statement."; - case mtl_malformed_diffuse_color: + case malformed_diffuse_color: return "File contains malformed 'Kd' statement."; - case mtl_malformed_specular_color: + case malformed_specular_color: return "File contains malformed 'Ks' statement."; - case mtl_malformed_specular_exponent: + case malformed_specular_exponent: return "File contains malformed 'Ns' statement."; - case mtl_malformed_dissolve: + case malformed_dissolve: return "File contains malformed 'd' statement."; - case mlt_unknown_line_begin: + case unknown_line_begin: return "Unknown mtl line begin"; default: using namespace std::string_literals; @@ -45,22 +49,24 @@ struct category : std::error_category { } // namespace mesh_loader_error -inline std::error_category& connector_error_category() { +inline std::error_category& connector_error_category() +{ static mtl_loader_error::category category; return category; } -namespace mtl_loader_error { - -inline std::error_code make_error_code(codes e) { +namespace mtl_loader_error +{ +inline std::error_code make_error_code(codes e) +{ return { static_cast(e), connector_error_category() }; } - } // namespace mtl_loader_error template -std::errc parse_numeric_vector(std::string_view param, std::array& values) { +std::errc parse_numeric_vector(std::string_view param, std::array& values) +{ auto it = param.begin(), end = param.end(); for (auto& value : values) @@ -83,20 +89,202 @@ std::errc parse_numeric_vector(std::string_view param, std::array& val return {}; }; -std::optional mtl_loader::find_id(std::string_view name) -{ - const auto it = m_id_lookup.find(name); - if (it == m_id_lookup.end()) +void mtl_loader::find_textures( + std::span buffer, + std::filesystem::path& path_buffer, + const std::filesystem::path& base_directory, + std::ifstream& in, + ztu::string_list& texture_filenames +) { + using namespace std::string_view_literals; + + // TODO 'bump' is missing!!! + + static constexpr auto keyword = "\nmap_"sv; + + using long_postfix_type = std::array; + + static constexpr auto postfix_length = std::tuple_size_v; + + static constexpr auto make_postfix = [](const std::string_view str) static constexpr { - return it->second; - } + auto postfix = long_postfix_type{}; + assert(str.length() >= postfix_length); + std::copy_n(str.begin(), postfix.size(), postfix.begin()); + return postfix; + }; - return std::nullopt; + static constexpr auto postfixes = std::array{ + make_postfix("d "), + make_postfix("Ka"), + make_postfix("Kd"), + make_postfix("Ks"), + make_postfix("Ns") + }; + + const auto buffer_view = std::string_view(buffer); + + // Add linebreak to simplify line begin search. + buffer.front() = '\n'; + auto leftover = std::size_t{ 1 }; + + enum class match { + exact, + overflowed, + none + }; + + const auto check_match = [](std::string_view& potential_match) static -> match + { + std::cout << '\'' << potential_match.substr(0, std::min(40ul, potential_match.size())) << '\'' << std::endl; + + if (potential_match.length() < postfix_length) + { + return match::overflowed; + } + + const auto postfix = make_postfix(potential_match); + + // Optimized for SIMD. + if (not std::ranges::contains(postfixes, postfix)) + { + return match::none; + } + + const auto long_match = postfix.back() != ' '; + + if (long_match and ( + potential_match.length() < postfix_length + sizeof(' ') or + potential_match[postfix_length] != ' ' + )) { + return match::overflowed; + } + + const auto actual_postfix_length = std::size_t{ 1 } + static_cast(long_match); + + const auto filename_begin = actual_postfix_length + sizeof(' '); + const auto filename_end = potential_match.find('\n', filename_begin); + + if (filename_end == std::string_view::npos) + { + return match::overflowed; + } + + const auto length = filename_end - filename_begin; + potential_match = potential_match.substr(filename_begin, length); + + return match::exact; + }; + + do + { + // Keep some old characters to continue matching interrupted sequence. + std::copy(buffer.end() - leftover, buffer.end(), buffer.begin()); + + in.read(buffer.data() + leftover, buffer.size() - leftover); + + const auto str = buffer_view.substr(0, leftover + in.gcount()); + + auto pos = std::string_view::size_type{}; + while ((pos = str.find(keyword, pos)) != std::string_view::npos) + { + const auto keyword_end = pos + keyword.size(); + auto potential_match = str.substr(keyword_end); + + const auto match_type = check_match(potential_match); + + if (match_type == match::exact) + { + path_buffer.assign(potential_match); + + if (path_buffer.is_relative()) + { + path_buffer = base_directory; + path_buffer /= potential_match; + } + + texture_filenames.push_back(path_buffer.c_str()); + + pos += potential_match.size(); + + leftover = 0; + } + else if (match_type == match::overflowed) + { + if (pos == 0) [[unlikely]] + { + ztu::logger::error("Ignoring string match, as it exceeds buffer size of % characters.", buffer.size()); + leftover = 0; + } + else + { + leftover = str.size() - pos; + } + break; + } + else + { + leftover = keyword.size(); + } + } + } while (not in.eof()); } -void mtl_loader::clear_name_lookup() { - m_id_lookup.clear(); +std::error_code mtl_loader::prefetch( + const file_dir_list& paths, + prefetch_queue& queue +) { + namespace fs = std::filesystem; + using mtl_loader_error::codes; + using mtl_loader_error::make_error_code; + + auto buffer = std::vector(8 * 1024, '\0'); + + auto in = std::ifstream{}; + + auto path_buffer = fs::path{}; + auto filename_buffer = fs::path{}; + + const auto process_file = [&]() + { + in.open(filename_buffer); + if (not in.is_open()) { + ztu::logger::error("Could not open .mtl file '%'", filename_buffer); + return; + } + + filename_buffer.remove_filename(); + + find_textures(buffer, path_buffer, filename_buffer, in, queue.texture.files); + + in.close(); + }; + + + for (const auto file : paths.files) + { + filename_buffer.assign(file); + process_file(); + } + + for (const auto directory : paths.directories) + { + for (const auto& file : fs::directory_iterator{ directory }) { + + filename_buffer.assign(file.path()); + + // Avoid heap allocation of .extension() + if (not std::string_view(filename_buffer.c_str()).ends_with(".obj")) + { + continue; + } + + process_file(); + } + } + + return {}; } std::error_code mtl_loader::load_directory( @@ -164,7 +352,7 @@ std::error_code mtl_loader::load( auto in = std::ifstream{ filename }; if (not in.is_open()) { - return make_error_code(codes::mtl_cannot_open_file); + return make_error_code(codes::cannot_open_file); } namespace fs = std::filesystem; @@ -252,7 +440,7 @@ std::error_code mtl_loader::load( auto& properties = material.initialized_surface_properties(); if (parse_numeric_vector(param, properties.ambient_filter) != std::errc{}) [[unlikely]] { - return codes::mtl_malformed_ambient_color; + return codes::malformed_ambient_color; } material.components() |= flags::surface_properties; @@ -266,7 +454,7 @@ std::error_code mtl_loader::load( auto& properties = material.initialized_surface_properties(); if (parse_numeric_vector(param, properties.diffuse_filter) != std::errc{}) [[unlikely]] { - return codes::mtl_malformed_diffuse_color; + return codes::malformed_diffuse_color; } material.components() |= flags::surface_properties; @@ -280,7 +468,7 @@ std::error_code mtl_loader::load( auto& properties = material.initialized_surface_properties(); if (parse_numeric_vector(param, properties.specular_filter) != std::errc{}) [[unlikely]] { - return codes::mtl_malformed_specular_color; + return codes::malformed_specular_color; } material.components() |= flags::surface_properties; @@ -295,7 +483,7 @@ std::error_code mtl_loader::load( std::array shininess{}; if (parse_numeric_vector(param, shininess) != std::errc{}) [[unlikely]] { - return codes::mtl_malformed_specular_exponent; + return codes::malformed_specular_exponent; } properties.shininess = shininess.front(); @@ -310,7 +498,7 @@ std::error_code mtl_loader::load( std::array transparency{}; if (parse_numeric_vector(param, transparency) != std::errc{}) [[unlikely]] { - return codes::mtl_malformed_dissolve; + return codes::malformed_dissolve; } material.transparency().emplace(transparency.front()); diff --git a/source/assets/data_loaders/obj_loader.cpp b/source/assets/data_loaders/obj_loader.cpp index df1a579..f015586 100755 --- a/source/assets/data_loaders/obj_loader.cpp +++ b/source/assets/data_loaders/obj_loader.cpp @@ -14,30 +14,37 @@ #include "util/line_parser.hpp" -namespace obj_loader_error { - -struct category : std::error_category { +namespace obj_loader_error +{ +struct category : std::error_category +{ [[nodiscard]] const char* name() const noexcept override { - return "connector"; + return "obj_loader"; } - [[nodiscard]] std::string message(int ev) const override { - switch (static_cast(ev)) { + [[nodiscard]] std::string message(int ev) const override + { + switch (static_cast(ev)) + { using enum codes; - case obj_cannot_open_file: + case cannot_open_file: return "Cannot open given obj file."; - case obj_malformed_vertex: + case malformed_vertex: return "File contains malformed 'v' statement."; - case obj_malformed_texture_coordinate: + case malformed_texture_coordinate: return "File contains malformed 'vt' statement."; - case obj_malformed_normal: + case malformed_normal: return "File contains malformed 'vn' statement."; - case obj_malformed_face: + case malformed_face: return "File contains malformed 'f' statement."; - case obj_face_index_out_of_range: + case face_index_out_of_range: return "Face index out of range."; - case obj_unknown_line_begin: + case unknown_line_begin: return "Unknown obj line begin."; + case use_material_without_material_library: + return "'usemtl' statement before material library loaded."; + case unknown_material_name: + return "No matching material name found in material library."; default: using namespace std::string_literals; return "unrecognized error ("s + std::to_string(ev) + ")"; @@ -47,41 +54,25 @@ struct category : std::error_category { } // namespace mesh_loader_error -inline std::error_category& connector_error_category() { +inline std::error_category& connector_error_category() +{ static obj_loader_error::category category; return category; } -namespace obj_loader_error { - -inline std::error_code make_error_code(codes e) { +namespace obj_loader_error +{ +inline std::error_code make_error_code(codes e) +{ return { static_cast(e), connector_error_category() }; } - } // namespace mesh_loader_error - -using vertex_type = std::array; - -struct indexed_vertex_type -{ - vertex_type vertex; - ztu::u32 buffer_index; - - friend auto operator<=>(const indexed_vertex_type& a, const indexed_vertex_type& b) { - return a.vertex <=> b.vertex; - } - - bool operator==(const indexed_vertex_type& other) const noexcept { - return other.vertex == vertex; - } -}; - - -// TODO add compile time selection and unrolling template -std::errc parse_numeric_vector(std::string_view param, std::array& values) { - auto it = param.begin(), end = param.end(); +std::errc parse_numeric_vector(std::string_view param, std::array& values) +{ + auto it = param.begin(); + const auto end = param.end(); for (auto& value : values) { @@ -97,15 +88,17 @@ std::errc parse_numeric_vector(std::string_view param, std::array& val return ec; } - it = ptr + 1; // skip space in between components + it = ptr + 1; // Skip space in between components. } return {}; }; -void find_materials( +void obj_loader::find_materials( std::span buffer, + std::filesystem::path& path_buffer, + const std::filesystem::path& base_directory, std::ifstream& in, ztu::string_list& material_filenames ) { @@ -113,13 +106,13 @@ void find_materials( const auto buffer_view = std::string_view(buffer); - // Add linebreak to simplify line begin search + // Add linebreak to simplify line begin search. buffer.front() = '\n'; auto leftover = std::size_t{ 1 }; do { - // Keep some old characters to continue matching interrupted sequence + // Keep some old characters to continue matching interrupted sequence. std::copy(buffer.end() - leftover, buffer.end(), buffer.begin()); in.read(buffer.data() + leftover, buffer.size() - leftover); @@ -137,23 +130,25 @@ void find_materials( if (filename_end != std::string_view::npos) { const auto length = filename_end - filename_begin; - const auto material_filename = str.substr(filename_begin, length); + const auto filename = str.substr(filename_begin, length); - // TODO get base dir from param figure out how to avoid heap - if (material_filename.is_relative()) + path_buffer.assign(filename); + + if (path_buffer.is_relative()) { - material_filename = directory / material_filename; + path_buffer = base_directory; + path_buffer /= filename; } - material_filenames.push_back(material_filename); + material_filenames.push_back(path_buffer.c_str()); pos = filename_end; } - else // string exceeds the buffer + else // String match exceeds buffer. { if (pos == 0) [[unlikely]] { - std::cout << "Ignoring string match, as it exceeds buffer size." << std::endl; + ztu::logger::error("Ignoring string match, as it exceeds buffer size of % characters.", buffer.size()); leftover = 0; } else @@ -179,145 +174,139 @@ std::error_code obj_loader::prefetch( auto in = std::ifstream{}; - const auto parse_file = [&](const char* filename) + auto path_buffer = fs::path{}; + auto filename_buffer = fs::path{}; + + const auto process_file = [&]() { - in.open(filename); + in.open(filename_buffer); if (not in.is_open()) { - ztu::logger::error("Could not open .obj file '%'", filename); + ztu::logger::error("Could not open .obj file '%'", filename_buffer); return; } - find_materials(buffer, in, queue.mtl_queue.files); + filename_buffer.remove_filename(); + + find_materials(buffer, path_buffer, filename_buffer, in, queue.mtl_queue.files); in.close(); }; + for (const auto file : paths.files) { - // `file` is null-terminates by list. - parse_file(file.data()); + filename_buffer.assign(file); + process_file(); } for (const auto directory : paths.directories) { for (const auto& file : fs::directory_iterator{ directory }) { - const auto& file_path = std::string_view{ file.path().c_str() }; + filename_buffer.assign(file.path()); - // TODO remove heap allocation - if (not file_path.ends_with(".obj")) + // Avoid heap allocation of .extension() + if (not std::string_view(filename_buffer.c_str()).ends_with(".obj")) { continue; } - // Null terminated by fs::path. - parse_file(file_path.data()); + process_file(); } } return {}; } -std::error_code obj_loader::load_directory( - dynamic_data_loader_ctx& ctx, - dynamic_mesh_store& store, - components::mesh_vertex::flags enabled_components, - const std::filesystem::path& path, - const bool pedantic -) { - namespace fs = std::filesystem; - - if (not fs::exists(path)) - { - return make_error_code(std::errc::no_such_file_or_directory); - } - - for (const auto& file : fs::directory_iterator{ path }) - { - const auto& file_path = file.path(); - - if (file_path.extension() != ".obj") - { - continue; - } - - if (const auto e = load( - ctx, - store, - enabled_components, - path, - pedantic - )) { - ztu::logger::error( - "Error while loading obj file '%': [%] %", - file_path, - e.category().name(), - e.message() - ); - } - } - - return {}; -} - -[[nodiscard]] static std::error_code load( - components::mesh_vertex::flags enabled_components, +std::error_code obj_loader::load( dynamic_mesh_buffer& buffer, const file_dir_list& paths, prefetch_lookup& id_lookup, dynamic_data_store& store, - bool pedantic = false + bool pedantic ) { + namespace fs = std::filesystem; + auto position_buffer = buffer.positions(); + auto normal_buffer = buffer.normals(); + auto tex_coord_buffer = buffer.tex_coords(); + auto read_buffer = dynamic_mesh_buffer{}; + auto path_buffer = fs::path{}; + auto vertex_ids = std::set{}; auto in = std::ifstream{}; - std::set vertex_ids; - const auto parse_file = [&](const char* filename) + auto filename_buffer = fs::path{}; + + const auto process_file = [&]() { - in.open(filename); + in.open(filename_buffer); if (not in.is_open()) { - ztu::logger::error("Could not open .obj file '%'", filename); + ztu::logger::error("Could not open .obj file '%'", filename_buffer); return; } + filename_buffer.remove_filename(); - vertex_ids.clear(); + // parse file + const auto error = parse_file( + read_buffer, + buffer, + path_buffer, + filename_buffer, + vertex_ids, + in, + id_lookup, + store, + pedantic + ); + + if (error) + { + ztu::logger::error( + "Error occurred while parsing .obj file: [%] %", + error.category().name(), + error.message() + ); + } in.close(); }; + for (const auto file : paths.files) { - // `file` is null-terminates by list. - parse_file(file.data()); + filename_buffer.assign(file); + process_file(); } for (const auto directory : paths.directories) { for (const auto& file : fs::directory_iterator{ directory }) { - const auto& file_path = std::string_view{ file.path().c_str() }; + filename_buffer.assign(file.path()); - // TODO remove heap allocation - if (not file_path.ends_with(".obj")) + // Avoid heap allocation of .extension() + if (not std::string_view(filename_buffer.c_str()).ends_with(".obj")) { continue; } - // Null terminated by fs::path. - parse_file(file_path.data()); + process_file(); } } + return {}; } -// TODO refactor so there is a function like parse_normals etc. -std::error_code obj_loader::load( - components::mesh_vertex::flags enabled_components, - dynamic_mesh_buffer& buffer, - const char* filename, +std::error_code obj_loader::parse_file( + dynamic_mesh_buffer& read_buffer, + dynamic_mesh_buffer& mesh_buffer, + std::filesystem::path& path_buffer, + const std::filesystem::path& base_directory, + std::set& vertex_ids, + std::ifstream& in, prefetch_lookup& id_lookup, dynamic_data_store& store, bool pedantic @@ -325,94 +314,47 @@ std::error_code obj_loader::load( using obj_loader_error::codes; using obj_loader_error::make_error_code; - auto in = std::ifstream{ filename }; - if (not in.is_open()) { - return make_error_code(codes::obj_cannot_open_file); - } + read_buffer.clear(); + mesh_buffer.clear(); + vertex_ids.clear(); - namespace fs = std::filesystem; - const auto directory = fs::path(filename).parent_path(); + // Buffers for storing the vertex component definitions. + auto& position_buffer = read_buffer.positions(); + auto& normal_buffer = read_buffer.normals(); + auto& tex_coord_buffer = read_buffer.tex_coords(); - // Each vertex of a face can represent a unique combination of vertex-/texture-/normal-coordinates. - // But some combinations may occur more than once, for example on every corner of a cube 3 triangles will - // reference the exact same corner vertex. - // To get the best rendering performance and lowest final memory footprint these duplicates - // need to be removed. So this std::set lookup is used to identify the aforementioned duplicates - // and only push unique combinations to the buffers. - std::set vertex_ids; + auto& positions = mesh_buffer.positions(); + auto& normals = mesh_buffer.normals(); + auto& tex_coords = mesh_buffer.tex_coords(); + auto& triangles = mesh_buffer.triangles(); - // Buffers - // TODO find out if this is still relevant - auto position_buffer = buffer.positions(); - auto normal_buffer = buffer.normals(); - auto tex_coord_buffer = buffer.tex_coords(); - - std::unordered_map material_name_lookup; - - std::string material_name; - - const auto push_mesh = [&](const bool clear_buffers = false) { - - if (not buffer.positions().empty()) + const auto push_mesh = [&](const bool clear_read_buffer = false) + { + if (not triangles.empty()) { - // Copy buffers instead of moving to keep capacity for further parsing - // and have the final buffers be shrunk to size. - if (not material_name.empty()) { - if (const auto id = material_loader.find_id(material_name)) - { - buffer.material_id() = *id; - } - else - { - ztu::logger::warn( - "Could not find material '%'.", - material_name - ); - } - } + ztu::logger::debug("Parsed % positions.", positions.size()); + ztu::logger::debug("Parsed % normals.", normals.size()); + ztu::logger::debug("Parsed % tex_coords.", tex_coords.size()); + ztu::logger::debug("Parsed % triangles.", triangles.size()); - ztu::logger::debug("Parsed % positions.", mesh.positions().size()); - ztu::logger::debug("Parsed % normals.", mesh.normals().size()); - ztu::logger::debug("Parsed % tex_coords.", mesh.tex_coords().size()); - - if (not buffer.positions().empty()) - { - buffer.components() |= components::mesh_vertex::flags::position; - } - - if (not buffer.normals().empty()) - { - buffer.components() |= components::mesh_vertex::flags::normal; - } - - if (not buffer.tex_coords().empty()) - { - buffer.components() |= components::mesh_vertex::flags::tex_coord; - } - - ztu::logger::debug("Pushing obj mesh with % triangles.", buffer.triangles().size()); - - store.meshes.add(buffer); + // Copy buffer into store and keep capacity. + store.meshes.add(mesh_buffer); } - if (clear_buffers) + if (clear_read_buffer) { - position_buffer.clear(); - normal_buffer.clear(); - tex_coord_buffer.clear(); + read_buffer.clear(); } - buffer.clear(); - + mesh_buffer.clear(); vertex_ids.clear(); - material_name.clear(); }; - const auto find_or_push_vertex = [&](const vertex_type& vertex) -> ztu::u32 { - + const auto find_or_push_vertex = [&](const vertex_type& vertex) -> index_type + { auto indexed_vid = indexed_vertex_type{ .vertex = vertex, - .buffer_index = static_cast(buffer.positions().size()) + .buffer_index = static_cast(positions.size()) }; // Search through sorted lookup to check if index combination is unique @@ -422,92 +364,86 @@ std::error_code obj_loader::load( { const auto& [ position_index, tex_coord_index, normal_index ] = vertex; + // If index is out of range, push default constructed value. + // Not ideal, but better than out of range indices. + + auto& position = positions.emplace_back(); if (position_index < position_buffer.size()) { - buffer.positions().emplace_back(position_buffer[position_index]); + position = position_buffer[position_index]; } + auto& normal = normals.emplace_back(); if (normal_index < normal_buffer.size()) { - buffer.normals().emplace_back(normal_buffer[normal_index]); + normal = normal_buffer[normal_index]; } + auto& tex_coord = tex_coords.emplace_back(); if (tex_coord_index < tex_coord_buffer.size()) { - buffer.tex_coords().emplace_back(tex_coord_buffer[tex_coord_index]); + tex_coord = tex_coord_buffer[tex_coord_index]; } } return id_it->buffer_index; }; - using flags = components::mesh_vertex::flags; - - const auto component_disabled = [&](const flags component) { - return (enabled_components & component) == flags::none; - }; - - const auto positions_disabled = component_disabled(flags::position); - const auto normals_disabled = component_disabled(flags::normal); - const auto tex_coords_disabled = component_disabled(flags::tex_coord); + auto curr_material_library_it = dynamic_material_library_store::iterator_type{}; + auto has_material_library = false; const auto ec = ztu::parse_lines( in, pedantic, - ztu::make_line_parser("v ", ztu::is_repeating, [&](const auto& param) + make_line_parser("v ", ztu::is_repeating, [&](const auto& param) { - if (positions_disabled) return codes::ok; - components::mesh_vertex::position position; if (parse_numeric_vector(param, position) != std::errc{}) [[unlikely]] { - return codes::obj_malformed_vertex; + return codes::malformed_vertex; } position_buffer.push_back(position); return codes::ok; }), - ztu::make_line_parser("vt ", ztu::is_repeating, [&](const auto& param) { - if (tex_coords_disabled) return codes::ok; - + make_line_parser("vt ", ztu::is_repeating, [&](const auto& param) + { components::mesh_vertex::tex_coord coord; if (parse_numeric_vector(param, coord) != std::errc{}) [[unlikely]] { - return codes::obj_malformed_texture_coordinate; + return codes::malformed_texture_coordinate; } tex_coord_buffer.push_back(coord); return codes::ok; }), - ztu::make_line_parser("vn ", ztu::is_repeating, [&](const auto& param) + make_line_parser("vn ", ztu::is_repeating, [&](const auto& param) { - if (normals_disabled) return codes::ok; - components::mesh_vertex::normal normal; if (parse_numeric_vector(param, normal) != std::errc{}) [[unlikely]] { - return codes::obj_malformed_normal; + return codes::malformed_normal; } normal_buffer.push_back(normal); return codes::ok; }), - ztu::make_line_parser("o ", ztu::is_not_repeating, [&](const auto&) + make_line_parser("o ", ztu::is_not_repeating, [&](const auto&) { push_mesh(); // Name is currently ignored return codes::ok; }), - ztu::make_line_parser("f ", ztu::is_repeating, [&](const auto& param) + make_line_parser("f ", ztu::is_repeating, [&](const auto& param) { const auto begin = param.begin().base(); const auto end = param.end().base(); auto vertex = vertex_type{}; - ztu::u32 first_index{}, prev_index{}; + index_type first_index{}, prev_index{}; auto vertex_count = std::size_t{}; @@ -525,11 +461,11 @@ std::error_code obj_loader::load( const auto [ptr, ec] = std::from_chars(it, end, component_index); if (ec != std::errc()) [[unlikely]] { - // Discard whole face if one index is malformed - return codes::obj_malformed_face; + // Discard whole face if one index is malformed. + return codes::malformed_face; } - --component_index; // Indices start at one + --component_index; // Convert to zero based index. it = ptr; if (it == end or *it != '/') @@ -544,14 +480,14 @@ std::error_code obj_loader::load( if (it != end and *it != ' ') [[unlikely]] { - return codes::obj_malformed_face; + return codes::malformed_face; } const auto curr_index = find_or_push_vertex(vertex); if (vertex_count >= 3) { - auto& triangle = mesh.triangles().emplace_back(); + auto& triangle = triangles.emplace_back(); triangle[0] = first_index; triangle[1] = prev_index; triangle[2] = curr_index; @@ -566,34 +502,50 @@ std::error_code obj_loader::load( return codes::ok; }), - ztu::make_line_parser("usemtl ", ztu::is_not_repeating, [&](const auto& param) + make_line_parser("usemtl ", ztu::is_not_repeating, [&](const auto& param) { push_mesh(false); - material_name = param; + if (not has_material_library) [[unlikely]] + { + return codes::use_material_without_material_library; + } + + const auto material_id_it = curr_material_library_it->find(param); + + if (material_id_it == curr_material_library_it->end()) [[unlikely]] + { + return codes::unknown_material_name; + } + + mesh_buffer.material_id() = material_id_it->second; return codes::ok; }), - ztu::make_line_parser("mtllib ", ztu::is_not_repeating, [&](const auto& param) + make_line_parser("mtllib ", ztu::is_not_repeating, [&](const auto& param) { - auto material_filename = fs::path(param); - if (material_filename.is_relative()) + path_buffer.assign(param); + + if (path_buffer.is_relative()) { - material_filename = directory / material_filename; + path_buffer = base_directory; + path_buffer /= param; // TODO Doesn't thi allocate an extra path?!? } - if (const auto error = ctx.material_loader.read( - ctx, - mtl_loader_id, - material_filename, - pedantic - )) { + const auto material_library_id_it = id_lookup.material_libraries.find(path_buffer); + + if (material_library_id_it != id_lookup.material_libraries.end()) [[likely]] + { + const auto material_library_id = material_library_id_it->second; + std::tie(curr_material_library_it, has_material_library) = store.material_libraries.find(material_library_id); + } + else [[unlikely]] + { ztu::logger::warn( - "Error occurred while loading mtl files '%': [%] %", - material_filename, - error.category().name(), - error.message() + "Could not find a matching material library with path '%'. Proceeding with default material.", + param ); + has_material_library = false; } }) ); diff --git a/source/assets/dynamic_data_loaders/dynamic_texture_loader.cpp b/source/assets/dynamic_data_loaders/dynamic_texture_loader.cpp index a001239..b2c3e34 100644 --- a/source/assets/dynamic_data_loaders/dynamic_texture_loader.cpp +++ b/source/assets/dynamic_data_loaders/dynamic_texture_loader.cpp @@ -7,8 +7,6 @@ #pragma warning(push, 0) #endif -#define STB_IMAGE_IMPLEMENTATION -#define STB_IMAGE_STATIC #include "stb_image.h" #if defined(__GNUC__) || defined(__GNUG__) diff --git a/source/assets/dynamic_read_buffers/dynamic_material_buffer.ipp b/source/assets/dynamic_read_buffers/dynamic_material_buffer.ipp index 8363407..0eb616c 100644 --- a/source/assets/dynamic_read_buffers/dynamic_material_buffer.ipp +++ b/source/assets/dynamic_read_buffers/dynamic_material_buffer.ipp @@ -86,3 +86,14 @@ inline components::material::surface_properties& dynamic_material_buffer::initia } return *surface_properties_opt; } + +inline void dynamic_material_buffer::clear() +{ + std::apply( + [](auto&... data_opt) { + (data_opt.reset(), ...); + }, + data + ); +} + diff --git a/source/assets/dynamic_read_buffers/dynamic_mesh_buffer.ipp b/source/assets/dynamic_read_buffers/dynamic_mesh_buffer.ipp index 7b1178e..b33b59a 100644 --- a/source/assets/dynamic_read_buffers/dynamic_mesh_buffer.ipp +++ b/source/assets/dynamic_read_buffers/dynamic_mesh_buffer.ipp @@ -71,3 +71,10 @@ inline const auto& dynamic_mesh_buffer::material_id() const { return m_material_id; } + +inline void dynamic_mesh_buffer::clear() +{ + clear_vertices(); + m_triangles.clear(); + m_material_id = {}; +} diff --git a/source/assets/dynamic_read_buffers/dynamic_point_cloud_buffer.ipp b/source/assets/dynamic_read_buffers/dynamic_point_cloud_buffer.ipp index 685aba8..80169a2 100644 --- a/source/assets/dynamic_read_buffers/dynamic_point_cloud_buffer.ipp +++ b/source/assets/dynamic_read_buffers/dynamic_point_cloud_buffer.ipp @@ -42,3 +42,8 @@ inline const std::vector& dynamic_p { return std::get(vertices); } + +inline void dynamic_point_cloud_buffer::clear() +{ + clear_vertices(); +} \ No newline at end of file diff --git a/source/assets/dynamic_read_buffers/dynamic_texture_buffer.ipp b/source/assets/dynamic_read_buffers/dynamic_texture_buffer.ipp index 855117d..205ffab 100644 --- a/source/assets/dynamic_read_buffers/dynamic_texture_buffer.ipp +++ b/source/assets/dynamic_read_buffers/dynamic_texture_buffer.ipp @@ -149,3 +149,11 @@ inline dynamic_texture_buffer::const_iterator dynamic_texture_buffer::cend() con { return const_cast(begin()); } + +inline void dynamic_texture_buffer::clear() +{ + m_data.reset(); + m_width = 0; + m_height = 0; + m_components = {}; +} \ No newline at end of file