From bc065bc657e96fe20e57ed812651d08473fb6224 Mon Sep 17 00:00:00 2001 From: zy4n Date: Mon, 31 Mar 2025 21:41:24 +0200 Subject: [PATCH] Started obj port and gave up. --- CMakeLists.txt | 2 - include/assets/file_parsers/glsl_parser.hpp | 37 +- include/assets/file_parsers/kitti_parser.hpp | 2 - .../assets/file_parsers/kitti_pose_parser.hpp | 6 +- include/assets/file_parsers/mtl_parser.hpp | 4 +- include/assets/file_parsers/obj_loader.hpp | 73 ++- include/assets/prefetch_lookup.hpp | 25 - .../material_library_prefetch_lookup.hpp | 12 - .../material_prefetch_lookup.hpp | 12 - .../prefetch_lookups/mesh_prefetch_lookup.hpp | 12 - .../point_cloud_prefetch_lookup.hpp | 12 - .../prefetch_lookups/pose_prefetch_lookup.hpp | 62 -- .../shader_prefetch_lookup.hpp | 12 - .../texture_prefetch_lookup.hpp | 12 - include/assets/prefetch_queue.hpp | 34 - source/assets/file_parsers/glsl_parser.cpp | 40 +- source/assets/file_parsers/kitti_parser.cpp | 27 +- .../assets/file_parsers/kitti_pose_parser.cpp | 14 +- source/assets/file_parsers/mtl_parser.cpp | 10 +- source/assets/file_parsers/obj_loader.cpp | 607 ++++++++++-------- 20 files changed, 422 insertions(+), 593 deletions(-) delete mode 100644 include/assets/prefetch_lookup.hpp delete mode 100644 include/assets/prefetch_lookups/material_library_prefetch_lookup.hpp delete mode 100644 include/assets/prefetch_lookups/material_prefetch_lookup.hpp delete mode 100644 include/assets/prefetch_lookups/mesh_prefetch_lookup.hpp delete mode 100644 include/assets/prefetch_lookups/point_cloud_prefetch_lookup.hpp delete mode 100644 include/assets/prefetch_lookups/pose_prefetch_lookup.hpp delete mode 100644 include/assets/prefetch_lookups/shader_prefetch_lookup.hpp delete mode 100644 include/assets/prefetch_lookups/texture_prefetch_lookup.hpp delete mode 100644 include/assets/prefetch_queue.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 955e1a8..17add23 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -154,7 +154,6 @@ add_executable(z3d main.cpp source/assets/file_parsers/threedtk_pose_loader.cpp include/assets/data/pose_data.hpp source/assets/file_parsers/kitti_pose_parser.cpp - include/assets/prefetch_queue.hpp include/util/string_list.hpp include/assets/data_stores/material_library_store.hpp include/assets/data/material_library_data.hpp @@ -164,7 +163,6 @@ add_executable(z3d main.cpp include/assets/data_stores/shader_source_store.hpp include/assets/file_parsers/generic/generic_3dtk_loader.hpp source/assets/file_parsers/generic/generic_3dtk_loader.ipp - include/assets/prefetch_lookup.hpp source/assets/prefetch_lookups/pose_prefetch_lookup.cpp include/assets/data_stores.hpp include/opengl/metadata/shader_source_metadata.hpp diff --git a/include/assets/file_parsers/glsl_parser.hpp b/include/assets/file_parsers/glsl_parser.hpp index 0e2c9bf..e59257b 100644 --- a/include/assets/file_parsers/glsl_parser.hpp +++ b/include/assets/file_parsers/glsl_parser.hpp @@ -30,10 +30,7 @@ protected: class parser_context { public: - parser_context( - store_type& m_store, - std::mutex& m_store_mutex - ); + parser_context(store_type& m_store); void operator()(lookup_type::const_pointer entry) noexcept; @@ -48,7 +45,6 @@ protected: private: store_type* m_store; - std::mutex* m_store_mutex; shader_source_data m_buffer{}; std::vector m_value_buffer{}; std::vector m_declaration_value_count_buffer{}; @@ -60,50 +56,29 @@ protected: std::vector& source ); - static void tokenize_declarations( - std::string_view source, - std::vector& value_buffer, - std::vector& declaration_value_count_buffer, - std::array& declaration_type_index_buffer - ); - - static [[nodiscard]] bool parse_metadata_from_tokens( - const std::vector& value_buffer, - const std::vector& declaration_value_count_buffer, - const std::array& declaration_type_index_buffer, - shader_source_data& buffer - ); - - static void remove_metadata_declarations( - const std::vector& value_buffer, - const std::vector& declaration_value_count_buffer, - const std::array& declaration_type_index_buffer, - std::vector& source - ); - [[nodiscard]] static bool parse_geometry_declaration( std::span values, - model_geometry::types& geometry_type + shader_source_data::metadata& meta ); [[nodiscard]] static bool parse_stage_declaration( std::span values, - shader_components::stage& stage + shader_source_data::metadata& meta ); [[nodiscard]] static bool parse_components_declaration( std::span values, - shader_source_data& buffer + shader_source_data::metadata& meta ); [[nodiscard]] static bool parse_static_enable_declaration( std::span values, - shader_source_data& buffer + shader_source_data::metadata& meta ); [[nodiscard]] static bool parse_dynamic_enable_declaration( std::span values, - shader_source_data& buffer + shader_source_data::metadata& meta ); [[nodiscard]] static bool parse_component_tokens( diff --git a/include/assets/file_parsers/kitti_parser.hpp b/include/assets/file_parsers/kitti_parser.hpp index 24c0b64..c00cc59 100644 --- a/include/assets/file_parsers/kitti_parser.hpp +++ b/include/assets/file_parsers/kitti_parser.hpp @@ -42,7 +42,6 @@ protected: const pose_list_id_lookup& pose_list_lookup, const pose_list_store& pose_list_store, store_type& m_store, - std::mutex& m_store_mutex ); void operator()(lookup_type::const_pointer entry) noexcept; @@ -60,7 +59,6 @@ protected: pose_list_id_lookup const* m_pose_list_lookup; pose_list_store const* m_pose_list_store; store_type* m_store; - std::mutex* m_store_mutex; data_type m_buffer{}; std::filesystem::path m_last_pose_path{}; pose_list_view m_last_pose_list{}; diff --git a/include/assets/file_parsers/kitti_pose_parser.hpp b/include/assets/file_parsers/kitti_pose_parser.hpp index 7640f46..5f6bcaf 100644 --- a/include/assets/file_parsers/kitti_pose_parser.hpp +++ b/include/assets/file_parsers/kitti_pose_parser.hpp @@ -33,10 +33,7 @@ private: class parser_context { public: - parser_context( - store_type& m_store, - std::mutex& m_store_mutex - ); + parser_context(store_type& m_store); void operator()(lookup_type::const_pointer entry) noexcept; @@ -45,7 +42,6 @@ private: private: store_type* m_store; - std::mutex* m_store_mutex; data_type m_buffer{}; }; diff --git a/include/assets/file_parsers/mtl_parser.hpp b/include/assets/file_parsers/mtl_parser.hpp index 0c50ffb..9350060 100644 --- a/include/assets/file_parsers/mtl_parser.hpp +++ b/include/assets/file_parsers/mtl_parser.hpp @@ -57,8 +57,7 @@ protected: const texture_id_lookup& texture_id_lookup, material_id_lookup& material_id_lookup, material_store& material_store, - store_type& store, - std::mutex& store_mutex + store_type& store ); void operator()(lookup_type::const_pointer entry) noexcept; @@ -77,7 +76,6 @@ protected: material_id_lookup* m_material_id_lookup; material_store* m_material_store; store_type* m_store; - std::mutex* m_store_mutex; data_type m_buffer{}; }; diff --git a/include/assets/file_parsers/obj_loader.hpp b/include/assets/file_parsers/obj_loader.hpp index 8d9596e..3d18dc8 100755 --- a/include/assets/file_parsers/obj_loader.hpp +++ b/include/assets/file_parsers/obj_loader.hpp @@ -3,13 +3,13 @@ #include #include #include -#include "../data_loaders" -#include "../data_stores" -#include "assets/prefetch_lookup.hpp" +#include "assets/data/mesh_data.hpp" +#include "assets/data_stores.hpp" +#include "assets/path_id_lookups.hpp" #include -namespace obj_loader_error { - +namespace assets::obj_loader_error +{ enum class codes { ok = 0, cannot_open_file, @@ -22,34 +22,34 @@ enum class codes { use_material_without_material_library, unknown_material_name }; - } // namespace obj_loader_error -struct obj_loader { +namespace assets +{ +struct obj_loader +{ static constexpr auto name = std::string_view("obj"); + using data_type = mesh_data; + using store_type = mesh_store; + using lookup_type = mesh_id_lookup; - [[nodiscard]] static std::error_code prefetch( - const file_dir_list& paths, - prefetch_queue& queue + // TODO port this mess to the new interface + [[nodiscard]] std::error_code prefetch( + path_id_lookups& lookups ); - [[nodiscard]] static std::error_code load( - dynamic_mesh_buffer& buffer, - const file_dir_list& paths, - prefetch_lookup& id_lookup, - dynamic_shader_source_store& store, + [[nodiscard]] std::error_code load( + path_id_lookups& lookups, + data_stores& stores, bool pedantic = false ); protected: - using index_type = dynamic_mesh_buffer::index_type; - using vertex_type = std::array; - struct indexed_vertex_type { - vertex_type vertex; - index_type buffer_index; + z3d::index_triangle vertex; + z3d::vertex_index buffer_index; friend auto operator<=>(const indexed_vertex_type& a, const indexed_vertex_type& b) { return a.vertex <=> b.vertex; @@ -60,6 +60,38 @@ protected: } }; + class parser_context + { + public: + parser_context( + const texture_id_lookup& texture_id_lookup, + material_id_lookup& material_id_lookup, + material_store& material_store, + store_type& store + ); + + void operator()(lookup_type::const_pointer entry) noexcept; + + protected: + void reset(); + + [[nodiscard]] std::optional fetch_texture_id( + const std::filesystem::path& mtl_dir, + std::string_view filename, + std::string_view texture_type_name + ); + + private: + const path_id_lookups* m_id_lookups; + data_stores* m_stores; + data_type m_buffer{}; + data_type m_read_buffer{}; + std::set vertex_ids{}; + }; + + + + static void find_materials( std::span buffer, std::filesystem::path& path_buffer, @@ -80,3 +112,4 @@ protected: bool pedantic ); }; +} diff --git a/include/assets/prefetch_lookup.hpp b/include/assets/prefetch_lookup.hpp deleted file mode 100644 index 0bb48cf..0000000 --- a/include/assets/prefetch_lookup.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include "prefetch_lookups/material_library_prefetch_lookup.hpp" -#include "prefetch_lookups/material_prefetch_lookup.hpp" -#include "prefetch_lookups/mesh_prefetch_lookup.hpp" -#include "prefetch_lookups/point_cloud_prefetch_lookup.hpp" -#include "prefetch_lookups/pose_prefetch_lookup.hpp" -#include "prefetch_lookups/shader_prefetch_lookup.hpp" -#include "prefetch_lookups/texture_prefetch_lookup.hpp" - -namespace assets -{ - -struct prefetch_lookup -{ - texture_prefetch_lookup textures; - material_library_prefetch_lookup material_libraries; - material_prefetch_lookup materials; - mesh_prefetch_lookup meshes; - pose_prefetch_lookup poses; - point_cloud_prefetch_lookup point_clouds; - shader_source_prefetch_lookup shader_sources; -}; - -} diff --git a/include/assets/prefetch_lookups/material_library_prefetch_lookup.hpp b/include/assets/prefetch_lookups/material_library_prefetch_lookup.hpp deleted file mode 100644 index 0bcfc6a..0000000 --- a/include/assets/prefetch_lookups/material_library_prefetch_lookup.hpp +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include -#include -#include "assets/data_stores/material_library_store.hpp" - -namespace assets -{ - -using material_library_prefetch_lookup = std::unordered_map; - -} diff --git a/include/assets/prefetch_lookups/material_prefetch_lookup.hpp b/include/assets/prefetch_lookups/material_prefetch_lookup.hpp deleted file mode 100644 index d9f4833..0000000 --- a/include/assets/prefetch_lookups/material_prefetch_lookup.hpp +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include -#include -#include "assets/data_stores/material_store.hpp" - -namespace assets -{ - -using material_prefetch_lookup = std::unordered_map; - -} diff --git a/include/assets/prefetch_lookups/mesh_prefetch_lookup.hpp b/include/assets/prefetch_lookups/mesh_prefetch_lookup.hpp deleted file mode 100644 index 41e0b55..0000000 --- a/include/assets/prefetch_lookups/mesh_prefetch_lookup.hpp +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include -#include -#include "assets/data_stores/mesh_store.hpp" - -namespace assets -{ - -using mesh_prefetch_lookup = std::unordered_map; - -} diff --git a/include/assets/prefetch_lookups/point_cloud_prefetch_lookup.hpp b/include/assets/prefetch_lookups/point_cloud_prefetch_lookup.hpp deleted file mode 100644 index b6f4936..0000000 --- a/include/assets/prefetch_lookups/point_cloud_prefetch_lookup.hpp +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include -#include -#include "assets/data_stores/point_cloud_store.hpp" - -namespace assets -{ - -using point_cloud_prefetch_lookup = std::unordered_map; - -} diff --git a/include/assets/prefetch_lookups/pose_prefetch_lookup.hpp b/include/assets/prefetch_lookups/pose_prefetch_lookup.hpp deleted file mode 100644 index 3b41d11..0000000 --- a/include/assets/prefetch_lookups/pose_prefetch_lookup.hpp +++ /dev/null @@ -1,62 +0,0 @@ -#pragma once - -#include -#include - -#include "util/uix.hpp" -#include "assets/data_stores/pose_store.hpp" - -namespace assets -{ - -class pose_prefetch_lookup -{ -public: - using index_type = ztu::u32; - using directory_lookup = std::unordered_map; - using directory_iterator = std::pair; - using index_lookup = std::vector; - using index_iterator = index_lookup::iterator; - using lookup_type = std::vector; - using iterator = lookup_type::iterator; - - void emplace( - const std::filesystem::path& directory, - index_type index, - pose_store::id_type id - ); - - void emplace_hint_dir( - directory_iterator directory_it, - index_type index, - pose_store::id_type id - ); - - void emplace_hint_dir_index( - directory_iterator directory_it, - index_iterator index_it, - index_type index, - pose_store::id_type id - ); - - std::pair find_directory( - const std::filesystem::path& directory - ); - - std::pair find_index( - directory_iterator directory_it, - index_type index - ); - - directory_iterator emplace_dir( - directory_iterator directory_it, - const std::filesystem::path& directory - ); - -private: - directory_lookup m_directory_lookup; - index_lookup m_directory_indices; // count before indices, indices sorted per dir - lookup_type m_pose_ids; // offset by 1 -}; - -} diff --git a/include/assets/prefetch_lookups/shader_prefetch_lookup.hpp b/include/assets/prefetch_lookups/shader_prefetch_lookup.hpp deleted file mode 100644 index 8b04f81..0000000 --- a/include/assets/prefetch_lookups/shader_prefetch_lookup.hpp +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include -#include -#include "assets/data_stores/shader_source_store.hpp" - -namespace assets -{ - -using shader_source_prefetch_lookup = std::unordered_map; - -} diff --git a/include/assets/prefetch_lookups/texture_prefetch_lookup.hpp b/include/assets/prefetch_lookups/texture_prefetch_lookup.hpp deleted file mode 100644 index b65837c..0000000 --- a/include/assets/prefetch_lookups/texture_prefetch_lookup.hpp +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include -#include -#include "assets/data_stores/texture_store.hpp" - -namespace assets -{ - -using texture_prefetch_lookup = std::unordered_map; - -} diff --git a/include/assets/prefetch_queue.hpp b/include/assets/prefetch_queue.hpp deleted file mode 100644 index b1e6f9e..0000000 --- a/include/assets/prefetch_queue.hpp +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include "util/string_list.hpp" - -namespace assets -{ - -struct file_dir_list -{ - ztu::string_list files; - ztu::string_list directories; -}; - - -struct prefetch_queue -{ - file_dir_list obj_queue; - file_dir_list stl_queue; - - file_dir_list mtl_queue; - - file_dir_list uosr_queue; - file_dir_list uos_rgb_queue; - file_dir_list uos_normal_queue; - file_dir_list uos_queue; - file_dir_list kitti_queue; - - file_dir_list threedtk_pose_queue; - file_dir_list kitti_pose_queue; - - file_dir_list glsl_queue; -}; - -} diff --git a/source/assets/file_parsers/glsl_parser.cpp b/source/assets/file_parsers/glsl_parser.cpp index 33056ef..64e0050 100644 --- a/source/assets/file_parsers/glsl_parser.cpp +++ b/source/assets/file_parsers/glsl_parser.cpp @@ -9,6 +9,8 @@ #include #include +#include "util/string_lookup.hpp" + namespace assets::language { enum class declaration_type : std::size_t @@ -88,13 +90,11 @@ std::error_code assets::glsl_parser::load( m_path_buffer.clear(); lookups.shader_sources.by_extension(".glsl", m_path_buffer); - auto store_mutex = std::mutex{}; - std::for_each( std::execution::parallel_unsequenced_policy{}, m_path_buffer.begin(), m_path_buffer.end(), - parser_context{ store, store_mutex } + parser_context{ store } ); return {}; @@ -102,9 +102,8 @@ std::error_code assets::glsl_parser::load( assets::glsl_parser::parser_context::parser_context( - store_type& m_store, - std::mutex& m_store_mutex -) : m_store{ &m_store }, m_store_mutex{ &m_store_mutex } + store_type& m_store +) : m_store{ &m_store } { m_buffer.source.reserve(4096); m_value_buffer.reserve(64); @@ -142,16 +141,13 @@ void assets::glsl_parser::parser_context::operator()(lookup_type::const_pointer remove_metadata_declarations(); - { - auto lock = std::lock_guard{ *m_store_mutex }; - m_store->insert(id, m_buffer); - } + m_store->insert(id, m_buffer); } void assets::glsl_parser::parser_context::tokenize_declarations() { - auto source_rest = std::string_view( m_buffer.source.data(), m_buffer.source.size() ); + auto source_rest = std::string_view{ m_buffer.source.data(), m_buffer.source.size() }; auto offset = std::string_view::size_type{}; while ((offset = source_rest.find(language::declaration_prefix)) != std::string_view::npos) @@ -237,7 +233,7 @@ bool assets::glsl_parser::parser_context::parse_metadata_from_tokens() ); const auto value_token_count = m_declaration_value_count_buffer[index]; - if (not parser(std::span(m_value_buffer).subspan(value_token_offset, value_token_count), m_buffer)) + if (not parser(std::span(m_value_buffer).subspan(value_token_offset, value_token_count), m_buffer.meta)) { return false; } @@ -353,7 +349,7 @@ std::error_code assets::glsl_parser::read_file( bool assets::glsl_parser::parse_geometry_declaration( const std::span values, - model_geometry::types& geometry_type + shader_source_data::metadata& meta ) { if (values.size() != 1) @@ -366,7 +362,7 @@ bool assets::glsl_parser::parse_geometry_declaration( if (const auto it = language::geometry_lookup.find(value); it != language::geometry_lookup.end()) { - geometry_type = it->second; + meta.geometry_type = it->second; } else { @@ -379,7 +375,7 @@ bool assets::glsl_parser::parse_geometry_declaration( bool assets::glsl_parser::parse_stage_declaration( const std::span values, - shader_components::stage& stage + shader_source_data::metadata& meta ) { if (values.size() != 1) @@ -392,7 +388,7 @@ bool assets::glsl_parser::parse_stage_declaration( if (const auto it = language::stage_lookup.find(value); it != language::stage_lookup.end()) { - stage = it->second; + meta.stage = it->second; } else { @@ -405,23 +401,23 @@ bool assets::glsl_parser::parse_stage_declaration( bool assets::glsl_parser::parse_components_declaration( const std::span values, - shader_source_data& buffer + shader_source_data::metadata& meta ) { - return parse_component_tokens(values, buffer.geometry_type, buffer.components); + return parse_component_tokens(values, meta.geometry_type, meta.components); } bool assets::glsl_parser::parse_static_enable_declaration( const std::span values, - shader_source_data& buffer + shader_source_data::metadata& meta ) { - return parse_component_tokens(values, buffer.geometry_type, buffer.static_enable); + return parse_component_tokens(values, meta.geometry_type, meta.static_enable); } bool assets::glsl_parser::parse_dynamic_enable_declaration( const std::span values, - shader_source_data& buffer + shader_source_data::metadata& meta ) { - return parse_component_tokens(values, buffer.geometry_type, buffer.dynamic_enable); + return parse_component_tokens(values, meta.geometry_type, meta.dynamic_enable); } bool assets::glsl_parser::parse_component_tokens( diff --git a/source/assets/file_parsers/kitti_parser.cpp b/source/assets/file_parsers/kitti_parser.cpp index 6f6463d..d1e5fc6 100644 --- a/source/assets/file_parsers/kitti_parser.cpp +++ b/source/assets/file_parsers/kitti_parser.cpp @@ -16,19 +16,18 @@ assets::kitti_parser::parser_context::parser_context( const pose_list_id_lookup& pose_list_lookup, const pose_list_store& pose_list_store, - store_type& m_store, - std::mutex& m_store_mutex + store_type& m_store ) : m_pose_list_lookup{ &pose_list_lookup }, m_pose_list_store{ &pose_list_store }, - m_store{ &m_store }, - m_store_mutex{ &m_store_mutex } + m_store{ &m_store } { - m_buffer.positions().reserve(8192); - m_buffer.normals().reserve(8192); - m_buffer.colors().reserve(8192); + constexpr auto expected_vertex_count = 8192; + m_buffer.positions().reserve(expected_vertex_count); + m_buffer.normals().reserve(expected_vertex_count); + m_buffer.colors().reserve(expected_vertex_count); } -5 + void assets::kitti_parser::parser_context::reset() { m_buffer.clear(); @@ -73,7 +72,7 @@ void assets::kitti_parser::parser_context::operator()(lookup_type::const_pointer return; } - clear(); + reset(); if (const auto e = load_point_file(filename, m_buffer)) { @@ -83,10 +82,7 @@ void assets::kitti_parser::parser_context::operator()(lookup_type::const_pointer transform_point_cloud(m_buffer.positions(), pose); - { - auto lock = std::lock_guard{ *m_store_mutex }; - m_store->insert(id, m_buffer); - } + m_store->insert(id, m_buffer); } ztu::result assets::kitti_parser::parent_directory( @@ -147,8 +143,6 @@ std::error_code assets::kitti_parser::load( m_path_buffer.clear(); lookups.point_clouds.by_extension(".bin", m_path_buffer); - auto store_mutex = std::mutex{}; - std::for_each( std::execution::parallel_unsequenced_policy{}, m_path_buffer.begin(), @@ -156,8 +150,7 @@ std::error_code assets::kitti_parser::load( parser_context{ lookups.pose_lists, stores.pose_lists, - stores.point_clouds, - store_mutex + stores.point_clouds } ); diff --git a/source/assets/file_parsers/kitti_pose_parser.cpp b/source/assets/file_parsers/kitti_pose_parser.cpp index c855167..98aed56 100644 --- a/source/assets/file_parsers/kitti_pose_parser.cpp +++ b/source/assets/file_parsers/kitti_pose_parser.cpp @@ -7,9 +7,8 @@ assets::kitti_pose_parser::parser_context::parser_context( - store_type& m_store, - std::mutex& m_store_mutex -) : m_store{ &m_store }, m_store_mutex{ &m_store_mutex } + store_type& m_store +) : m_store{ &m_store } { m_buffer.reserve(128); } @@ -29,10 +28,7 @@ void assets::kitti_pose_parser::parser_context::operator()(lookup_type::const_po return; } - { - auto lock = std::lock_guard{ *m_store_mutex }; - m_store->add(id, m_buffer); - } + m_store->insert(id, m_buffer); } @@ -55,13 +51,11 @@ std::error_code assets::kitti_pose_parser::parse( m_path_buffer.clear(); lookups.pose_lists.by_extension(".glsl", m_path_buffer); - auto store_mutex = std::mutex{}; - std::for_each( std::execution::parallel_unsequenced_policy{}, m_path_buffer.begin(), m_path_buffer.end(), - parser_context{ store, store_mutex } + parser_context{ store } ); return {}; diff --git a/source/assets/file_parsers/mtl_parser.cpp b/source/assets/file_parsers/mtl_parser.cpp index 82d2f12..d85c6d3 100644 --- a/source/assets/file_parsers/mtl_parser.cpp +++ b/source/assets/file_parsers/mtl_parser.cpp @@ -168,14 +168,12 @@ assets::mtl_parser::parser_context::parser_context( const texture_id_lookup& texture_id_lookup, material_id_lookup& material_id_lookup, material_store& material_store, - store_type& store, - std::mutex& store_mutex + store_type& store ) : m_texture_id_lookup{ &texture_id_lookup }, m_material_id_lookup{ &material_id_lookup }, m_material_store{ &material_store }, - m_store{ &store }, - m_store_mutex{ &store_mutex } + m_store{ &store } { m_buffer.reserve(32); } @@ -215,7 +213,7 @@ void assets::mtl_parser::parser_context::operator()(lookup_type::const_pointer e const auto material_id = id_it->second; - m_material_store->emplace(id, material); + m_material_store->insert(id, material); m_buffer.emplace(name, material_id); } @@ -367,6 +365,8 @@ void assets::mtl_parser::parser_context::operator()(lookup_type::const_pointer e } push_material(); + + m_store->insert(id, m_buffer); } std::optional assets::mtl_parser::parser_context::fetch_texture_id( diff --git a/source/assets/file_parsers/obj_loader.cpp b/source/assets/file_parsers/obj_loader.cpp index b70abdd..8a9c0b9 100755 --- a/source/assets/file_parsers/obj_loader.cpp +++ b/source/assets/file_parsers/obj_loader.cpp @@ -5,16 +5,12 @@ #include #include "assets/components/mesh_vertex_components.hpp" -#include "assets/data_loaders/" #include "util/logger.hpp" #include "util/for_each.hpp" -#include "util/uix.hpp" -#include - #include "util/line_parser.hpp" -namespace obj_loader_error +namespace assets::obj_loader_error { struct category : std::error_category { @@ -54,48 +50,328 @@ struct category : std::error_category } // namespace mesh_loader_error -inline std::error_category& connector_error_category() +inline std::error_category& obj_loader_error_category() { - static obj_loader_error::category category; + static assets::obj_loader_error::category category; return category; } -namespace obj_loader_error +namespace assets::obj_loader_error { inline std::error_code make_error_code(codes e) { - return { static_cast(e), connector_error_category() }; + return { static_cast(e), obj_loader_error_category() }; } } // namespace mesh_loader_error -template -std::errc parse_numeric_vector(std::string_view param, std::array& values) + + +assets::obj_loader::parser_context::parser_context( + const texture_id_lookup& texture_id_lookup, + material_id_lookup& material_id_lookup, + material_store& material_store, + store_type& store +) : + m_texture_id_lookup{ &texture_id_lookup }, + m_material_id_lookup{ &material_id_lookup }, + m_material_store{ &material_store }, + m_store{ &store } { - auto it = param.begin(); - const auto end = param.end(); + constexpr auto expected_vertex_count = 8192; + m_buffer.positions().reserve(expected_vertex_count); + m_buffer.normals().reserve(expected_vertex_count); + m_buffer.colors().reserve(expected_vertex_count); + m_buffer.reflectances().reserve(expected_vertex_count); + m_buffer.tex_coords().reserve(expected_vertex_count); + m_buffer.triangles().reserve(2 * expected_vertex_count); - for (auto& value : values) + m_read_buffer.positions().reserve(expected_vertex_count); + m_read_buffer.normals().reserve(expected_vertex_count); + m_read_buffer.colors().reserve(expected_vertex_count); + m_read_buffer.reflectances().reserve(expected_vertex_count); + m_read_buffer.tex_coords().reserve(expected_vertex_count); + m_read_buffer.triangles().reserve(2 * expected_vertex_count); +} + +void assets::obj_loader::parser_context::reset() +{ + m_buffer.clear(); + m_read_buffer.clear(); + vertex_ids.clear(); +} + +void assets::obj_loader::parser_context::operator()(lookup_type::const_pointer entry) noexcept +{ + using obj_loader_error::codes; + using obj_loader_error::make_error_code; + namespace fs = std::filesystem; + + reset(); + + auto path_buffer = fs::path{}; + const auto base_dir = fs::canonical(fs::path(filename).parent_path()); + + // Buffers for storing the vertex component definitions. + auto& position_buffer = m_read_buffer.positions(); + auto& normal_buffer = m_read_buffer.normals(); + auto& tex_coord_buffer = m_read_buffer.tex_coords(); + + auto& positions = m_buffer.positions(); + auto& normals = m_buffer.normals(); + auto& tex_coords = m_buffer.tex_coords(); + auto& triangles = m_buffer.triangles(); + + const auto& [ filename, id ] = *entry; + + auto in = std::ifstream{ filename }; + if (not in.is_open()) { - if (it >= end) - { - return std::errc::invalid_argument; - } - - const auto [ptr, ec] = std::from_chars(it, end, value); - - if (ec != std::errc{}) - { - return ec; - } - - it = ptr + 1; // Skip space in between components. + ztu::logger::warn("Cannot open obj file %.", filename); + return; } - return {}; -}; + const auto push_mesh = [&](const bool clear_read_buffer = false) + { + if (not triangles.empty()) + { + ztu::logger::debug("Parsed % positions.", positions.size()); + ztu::logger::debug("Parsed % normals.", normals.size()); + ztu::logger::debug("Parsed % tex_coords.", tex_coords.size()); + ztu::logger::debug("Parsed % triangles.", triangles.size()); -void obj_loader::find_materials( + // Copy buffer into store and keep capacity. + m_store->insert(id, m_buffer); + } + + if (clear_read_buffer) + { + m_read_buffer.clear(); + } + + m_buffer.clear(); + vertex_ids.clear(); + }; + + const auto find_or_push_vertex = [&](const z3d::index_triangle& vertex) -> z3d::vertex_index + { + auto indexed_vid = indexed_vertex_type{ + .vertex = vertex, + .buffer_index = static_cast(positions.size()) + }; + + // Search through sorted lookup to check if index combination is unique + const auto [ id_it, unique ] = vertex_ids.insert(indexed_vid); + + if (unique) + { + const auto& [ position_index, tex_coord_index, normal_index ] = vertex; + + // If index is out of range, push default constructed value. + // Not ideal, but better than out of range indices. + + auto& position = positions.emplace_back(); + if (position_index < position_buffer.size()) + { + position = position_buffer[position_index]; + } + + auto& normal = normals.emplace_back(); + if (normal_index < normal_buffer.size()) + { + normal = normal_buffer[normal_index]; + } + + auto& tex_coord = tex_coords.emplace_back(); + if (tex_coord_index < tex_coord_buffer.size()) + { + tex_coord = tex_coord_buffer[tex_coord_index]; + } + } + + return id_it->buffer_index; + }; + + const material_library_data* curr_material_library{}; + + const auto ec = ztu::parse_lines( + in, + pedantic, + make_line_parser("v ", ztu::is_repeating, [&](const auto& param) + { + mesh_vertex_components::position position; + if (parse_numeric_vector(param, position) != std::errc{}) [[unlikely]] + { + return codes::malformed_vertex; + } + + position_buffer.push_back(position); + + return codes::ok; + }), + make_line_parser("vt ", ztu::is_repeating, [&](const auto& param) + { + mesh_vertex_components::tex_coord coord; + if (parse_numeric_vector(param, coord) != std::errc{}) [[unlikely]] + { + return codes::malformed_texture_coordinate; + } + + tex_coord_buffer.push_back(coord); + + return codes::ok; + }), + make_line_parser("vn ", ztu::is_repeating, [&](const auto& param) + { + mesh_vertex_components::normal normal; + if (parse_numeric_vector(param, normal) != std::errc{}) [[unlikely]] + { + return codes::malformed_normal; + } + + normal_buffer.push_back(normal); + + return codes::ok; + }), + make_line_parser("o ", ztu::is_not_repeating, [&](const auto&) + { + push_mesh(); // Name is currently ignored + return codes::ok; + }), + make_line_parser("f ", ztu::is_repeating, [&](const auto& param) + { + const auto begin = param.begin().base(); + const auto end = param.end().base(); + + auto vertex = z3d::index_triangle{}; + + z3d::vertex_index first_index{}, prev_index{}; + + auto vertex_count = std::size_t{}; + + for (auto it = begin; it <= end; ++it) + { + + for (auto& component_index : vertex) + { + if (it != end and *it == '/') + { + ++it; + continue; + } + + const auto [ptr, ec] = std::from_chars(it, end, component_index); + if (ec != std::errc()) [[unlikely]] + { + // Discard whole face if one index is malformed. + return codes::malformed_face; + } + + --component_index; // Convert to zero based index. + it = ptr; + + if (it == end or *it != '/') + { + break; + } + + ++it; + } + + ++vertex_count; + + if (it != end and *it != ' ') [[unlikely]] + { + return codes::malformed_face; + } + + const auto curr_index = find_or_push_vertex(vertex); + + if (vertex_count >= 3) + { + triangles.emplace_back() = { + first_index, + prev_index, + curr_index + }; + } + else if (vertex_count == 1) + { + first_index = curr_index; + } + + prev_index = curr_index; + } + + return codes::ok; + }), + make_line_parser("usemtl ", ztu::is_not_repeating, [&](const auto& param) + { + push_mesh(false); + + if (not curr_material_library) [[unlikely]] + { + return codes::use_material_without_material_library; + } + + const auto material_id_it = curr_material_library->find(param); + + if (material_id_it == curr_material_library->end()) [[unlikely]] + { + return codes::unknown_material_name; + } + + m_buffer.material() = material_id_it->second; + + return codes::ok; + }), + make_line_parser("mtllib ", ztu::is_not_repeating, [&](const auto& param) + { + path_buffer.assign(param); + + if (path_buffer.is_relative()) + { + path_buffer = base_dir; + path_buffer /= param; + } + + const auto material_library_id_it = m_id_lookups->material_libraries.find(path_buffer); + + if (material_library_id_it != m_id_lookups->material_libraries.end()) [[likely]] + { + const auto material_library_id = material_library_id_it->second; + + const auto [ it, found ] = m_stores->material_libraries.find(material_library_id); + if (found) + { + curr_material_library = &(it->second); + } + else + { + // TODO ALARM!!! + } + } + else [[unlikely]] + { + ztu::logger::warn( + "Could not find a matching material library with path '%'. Proceeding with default material.", + param + ); + curr_material_library = nullptr; + } + }) + ); + + if (ec != codes::ok) + { + const auto e = make_error_code(ec); + ztu::logger::error("Error while parsing obj file %: %", filename, e.message()); + } + + push_mesh(); +} + +void assets::obj_loader::find_materials( std::span buffer, std::filesystem::path& path_buffer, const std::filesystem::path& base_directory, @@ -162,13 +438,13 @@ void obj_loader::find_materials( } -std::error_code obj_loader::prefetch( +std::error_code assets::obj_loader::prefetch( const file_dir_list& paths, prefetch_queue& queue ) { namespace fs = std::filesystem; - using obj_loader_error::codes; - using obj_loader_error::make_error_code; + using assets::obj_loader_error::codes; + using assets::obj_loader_error::make_error_code; auto buffer = std::vector(8 * 1024, '\0'); @@ -218,7 +494,7 @@ std::error_code obj_loader::prefetch( return {}; } -std::error_code obj_loader::load( +std::error_code assets::obj_loader::load( dynamic_mesh_buffer& buffer, const file_dir_list& paths, prefetch_lookup& id_lookup, @@ -299,263 +575,28 @@ std::error_code obj_loader::load( return {}; } +template +std::errc parse_numeric_vector(std::string_view param, std::array& values) +{ + auto it = param.begin(); + const auto end = param.end(); -std::error_code obj_loader::parse_file( - dynamic_mesh_buffer& read_buffer, - dynamic_mesh_buffer& mesh_buffer, - std::filesystem::path& path_buffer, - const std::filesystem::path& base_directory, - std::set& vertex_ids, - std::ifstream& in, - prefetch_lookup& id_lookup, - dynamic_shader_source_store& store, - bool pedantic -) { - using obj_loader_error::codes; - using obj_loader_error::make_error_code; - - read_buffer.clear(); - mesh_buffer.clear(); - vertex_ids.clear(); - - // Buffers for storing the vertex component definitions. - auto& position_buffer = read_buffer.positions(); - auto& normal_buffer = read_buffer.normals(); - auto& tex_coord_buffer = read_buffer.tex_coords(); - - auto& positions = mesh_buffer.positions(); - auto& normals = mesh_buffer.normals(); - auto& tex_coords = mesh_buffer.tex_coords(); - auto& triangles = mesh_buffer.triangles(); - - const auto push_mesh = [&](const bool clear_read_buffer = false) + for (auto& value : values) { - if (not triangles.empty()) + if (it >= end) { - ztu::logger::debug("Parsed % positions.", positions.size()); - ztu::logger::debug("Parsed % normals.", normals.size()); - ztu::logger::debug("Parsed % tex_coords.", tex_coords.size()); - ztu::logger::debug("Parsed % triangles.", triangles.size()); - - // Copy buffer into store and keep capacity. - store.meshes.add(mesh_buffer); + return std::errc::invalid_argument; } - if (clear_read_buffer) + const auto [ptr, ec] = std::from_chars(it, end, value); + + if (ec != std::errc{}) { - read_buffer.clear(); + return ec; } - mesh_buffer.clear(); - vertex_ids.clear(); - }; - - const auto find_or_push_vertex = [&](const vertex_type& vertex) -> index_type - { - auto indexed_vid = indexed_vertex_type{ - .vertex = vertex, - .buffer_index = static_cast(positions.size()) - }; - - // Search through sorted lookup to check if index combination is unique - const auto [ id_it, unique ] = vertex_ids.insert(indexed_vid); - - if (unique) - { - const auto& [ position_index, tex_coord_index, normal_index ] = vertex; - - // If index is out of range, push default constructed value. - // Not ideal, but better than out of range indices. - - auto& position = positions.emplace_back(); - if (position_index < position_buffer.size()) - { - position = position_buffer[position_index]; - } - - auto& normal = normals.emplace_back(); - if (normal_index < normal_buffer.size()) - { - normal = normal_buffer[normal_index]; - } - - auto& tex_coord = tex_coords.emplace_back(); - if (tex_coord_index < tex_coord_buffer.size()) - { - tex_coord = tex_coord_buffer[tex_coord_index]; - } - } - - return id_it->buffer_index; - }; - - auto curr_material_library_it = dynamic_material_library_store::iterator_type{}; - auto has_material_library = false; - - const auto ec = ztu::parse_lines( - in, - pedantic, - make_line_parser("v ", ztu::is_repeating, [&](const auto& param) - { - mesh_vertex_components::position position; - if (parse_numeric_vector(param, position) != std::errc{}) [[unlikely]] - { - return codes::malformed_vertex; - } - - position_buffer.push_back(position); - - return codes::ok; - }), - make_line_parser("vt ", ztu::is_repeating, [&](const auto& param) - { - mesh_vertex_components::tex_coord coord; - if (parse_numeric_vector(param, coord) != std::errc{}) [[unlikely]] - { - return codes::malformed_texture_coordinate; - } - - tex_coord_buffer.push_back(coord); - - return codes::ok; - }), - make_line_parser("vn ", ztu::is_repeating, [&](const auto& param) - { - mesh_vertex_components::normal normal; - if (parse_numeric_vector(param, normal) != std::errc{}) [[unlikely]] - { - return codes::malformed_normal; - } - - normal_buffer.push_back(normal); - - return codes::ok; - }), - make_line_parser("o ", ztu::is_not_repeating, [&](const auto&) - { - push_mesh(); // Name is currently ignored - return codes::ok; - }), - make_line_parser("f ", ztu::is_repeating, [&](const auto& param) - { - const auto begin = param.begin().base(); - const auto end = param.end().base(); - - auto vertex = vertex_type{}; - - index_type first_index{}, prev_index{}; - - auto vertex_count = std::size_t{}; - - for (auto it = begin; it <= end; ++it) - { - - for (auto& component_index : vertex) - { - if (it != end and *it == '/') - { - ++it; - continue; - } - - const auto [ptr, ec] = std::from_chars(it, end, component_index); - if (ec != std::errc()) [[unlikely]] - { - // Discard whole face if one index is malformed. - return codes::malformed_face; - } - - --component_index; // Convert to zero based index. - it = ptr; - - if (it == end or *it != '/') - { - break; - } - - ++it; - } - - ++vertex_count; - - if (it != end and *it != ' ') [[unlikely]] - { - return codes::malformed_face; - } - - const auto curr_index = find_or_push_vertex(vertex); - - if (vertex_count >= 3) - { - auto& triangle = triangles.emplace_back(); - triangle[0] = first_index; - triangle[1] = prev_index; - triangle[2] = curr_index; - } - else if (vertex_count == 1) - { - first_index = curr_index; - } - - prev_index = curr_index; - } - - return codes::ok; - }), - make_line_parser("usemtl ", ztu::is_not_repeating, [&](const auto& param) - { - push_mesh(false); - - if (not has_material_library) [[unlikely]] - { - return codes::use_material_without_material_library; - } - - const auto material_id_it = curr_material_library_it->find(param); - - if (material_id_it == curr_material_library_it->end()) [[unlikely]] - { - return codes::unknown_material_name; - } - - mesh_buffer.material_id() = material_id_it->second; - - return codes::ok; - }), - make_line_parser("mtllib ", ztu::is_not_repeating, [&](const auto& param) - { - path_buffer.assign(param); - - if (path_buffer.is_relative()) - { - path_buffer = base_directory; - path_buffer /= param; // TODO Doesn't thi allocate an extra path?!? - } - - const auto material_library_id_it = id_lookup.material_libraries.find(path_buffer); - - if (material_library_id_it != id_lookup.material_libraries.end()) [[likely]] - { - const auto material_library_id = material_library_id_it->second; - std::tie(curr_material_library_it, has_material_library) = store.material_libraries.find(material_library_id); - } - else [[unlikely]] - { - ztu::logger::warn( - "Could not find a matching material library with path '%'. Proceeding with default material.", - param - ); - has_material_library = false; - } - }) - ); - - if (ec != codes::ok) - { - return make_error_code(ec); + it = ptr + 1; // Skip space in between components. } - push_mesh(); - return {}; -} +};