zy4n/Z3D
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finally refactored all stores!

Browse Source
This commit is contained in:
zy4n
2025-04-02 17:45:41 +02:00
parent 27977c1738
commit 835c645da4
11 changed files with 469 additions and 648 deletions
Download Patch File Download Diff File Expand all files Collapse all files

318
CMakeLists.txt
View File

@@ -15,12 +15,12 @@ add_executable(z3d main.cpp
include/assets/components/point_cloud_vertex_components.hpp
include/assets/components/shader_components.hpp
include/assets/components/texture_components.hpp
# include/assets/data_loaders/generic/base_dynamic_loader.hpp
# include/assets/data_loaders/material_library_loader.hpp
# include/assets/data_loaders/material_loader.hpp
# include/assets/data_loaders/mesh_loader.hpp
# include/assets/data_loaders/point_cloud_loader.hpp
# include/assets/data_loaders/texture_loader.hpp
include/assets/data_loaders/generic/base_dynamic_loader.hpp
include/assets/data_loaders/material_library_loader.hpp
include/assets/data_loaders/material_loader.hpp
include/assets/data_loaders/mesh_loader.hpp
include/assets/data_loaders/point_cloud_loader.hpp
include/assets/data_loaders/texture_loader.hpp
include/assets/data_stores.hpp
include/assets/data_stores/generic/generic_basic_store.hpp
include/assets/data_stores/generic/generic_material_store.hpp
@@ -72,86 +72,86 @@ add_executable(z3d main.cpp
include/assets/model_geometry.hpp
include/assets/path_id_lookups.hpp
include/config/primitives.hpp
# include/geometry/aabb.hpp
# include/geometry/normal_estimation.hpp
# include/opengl/data_managers/material_manager.hpp
# include/opengl/data_managers/mesh_index_buffer_manager.hpp
# include/opengl/data_managers/mesh_vertex_buffer_manager.hpp
# include/opengl/data_managers/point_cloud_vertex_buffer_manager.hpp
# include/opengl/data_managers/shader_manager.hpp
# include/opengl/data_managers/shader_program_manager.hpp
# include/opengl/data_managers/shader_source_manager.hpp
# include/opengl/data_managers/texture_manager.hpp
# include/opengl/data/material_data.hpp
# include/opengl/data/mesh_data.hpp
# include/opengl/data/point_cloud_data.hpp
# include/opengl/data/shader_data.hpp
# include/opengl/data/shader_program_data.hpp
# include/opengl/error.hpp
# include/opengl/error.hpp
# include/opengl/handles/alpha_handle.hpp
# include/opengl/handles/index_buffer_handle.hpp
# include/opengl/handles/material_handle.hpp
# include/opengl/handles/matrix_handles.hpp
# include/opengl/handles/mesh_handle.hpp
# include/opengl/handles/point_cloud_handle.hpp
# include/opengl/handles/shader_handle_set.hpp
# include/opengl/handles/shader_handle.hpp
# include/opengl/handles/shader_program_handle.hpp
# include/opengl/handles/surface_properties_handle.hpp
# include/opengl/handles/texture_handle.hpp
# include/opengl/handles/vertex_buffer_handle.hpp
# include/opengl/metadata/index_buffer_metadata.hpp
# include/opengl/metadata/material_metadata.hpp
# include/opengl/metadata/mesh_vertex_buffer_metadata.hpp
# include/opengl/metadata/point_cloud_vertex_buffer_metadata.hpp
# include/opengl/metadata/preprocessed_shader_source_metadata.hpp
# include/opengl/metadata/shader_metadata.hpp
# include/opengl/metadata/shader_program_metadata.hpp
# include/opengl/metadata/shader_set_metadata.hpp
# include/opengl/metadata/shader_source_metadata.hpp
# include/opengl/metadata/texture_metadata.hpp
# include/opengl/metadata/uniform_buffer_alignment_metadata.hpp
# include/opengl/resource_management/reference_counter.hpp
# include/opengl/resource_management/reference_counter.hpp
# include/opengl/resource_management/resource_handle.hpp
# include/opengl/resource_management/resource_manager.hpp
# include/opengl/shader_program_lookup.hpp
# include/opengl/shading/attributes/mesh_attributes.hpp
# include/opengl/shading/attributes/point_cloud_attributes.hpp
# include/opengl/shading/features/combined_features.hpp
# include/opengl/shading/features/generic_features.hpp
# include/opengl/shading/features/mesh_features.hpp
# include/opengl/shading/features/point_cloud_features.hpp
# include/opengl/shading/model_geometry.hpp
# include/opengl/shading/requirements/shader_program_requirements.hpp
# include/opengl/shading/requirements/shader_requirements.hpp
# include/opengl/shading/sampler_uniform.hpp
# include/opengl/shading/sampler_uniforms/mesh_samplers_uniforms.hpp
# include/opengl/shading/shader_metadata_language.hpp
# include/opengl/shading/shader_stage.hpp
# include/opengl/shading/uniform_block.hpp
# include/opengl/shading/uniform_blocks/mesh_uniform_blocks.hpp
# include/opengl/shading/uniform_blocks/point_cloud_uniform_blocks.hpp
# include/opengl/shading/uniform.hpp
# include/opengl/shading/uniforms/mesh_uniforms.hpp
# include/opengl/shading/uniforms/point_cloud_uniforms.hpp
# include/opengl/type_utils.hpp
# include/opengl/types.hpp
# include/rendering/batch_renderers/mesh_batch_renderer.hpp
# include/rendering/batch_renderers/point_cloud_batch_renderer.hpp
# include/rendering/batches/mesh_batch.hpp
# include/rendering/batches/point_cloud_batch.hpp
# include/rendering/modes/mesh_modes.hpp
# include/rendering/modes/point_cloud_modes.hpp
# include/rendering/requirements/mesh_requirements.hpp
# include/rendering/requirements/point_cloud_requirements.hpp
# include/rendering/shader_program_lookups/mesh_lookup.hpp
# include/rendering/shader_program_lookups/mesh_lookup.hpp
# include/rendering/shader_program_lookups/point_cloud_lookup.hpp
# include/scene/camera.hpp
# include/scene/flying_camera.hpp
# include/scene/lighting_setup.hpp
include/geometry/aabb.hpp
include/geometry/normal_estimation.hpp
include/opengl/data_managers/material_manager.hpp
include/opengl/data_managers/mesh_index_buffer_manager.hpp
include/opengl/data_managers/mesh_vertex_buffer_manager.hpp
include/opengl/data_managers/point_cloud_vertex_buffer_manager.hpp
include/opengl/data_managers/shader_manager.hpp
include/opengl/data_managers/shader_program_manager.hpp
include/opengl/data_managers/shader_source_manager.hpp
include/opengl/data_managers/texture_manager.hpp
include/opengl/data/material_data.hpp
include/opengl/data/mesh_data.hpp
include/opengl/data/point_cloud_data.hpp
include/opengl/data/shader_data.hpp
include/opengl/data/shader_program_data.hpp
include/opengl/error.hpp
include/opengl/error.hpp
include/opengl/handles/alpha_handle.hpp
include/opengl/handles/index_buffer_handle.hpp
include/opengl/handles/material_handle.hpp
include/opengl/handles/matrix_handles.hpp
include/opengl/handles/mesh_handle.hpp
include/opengl/handles/point_cloud_handle.hpp
include/opengl/handles/shader_handle_set.hpp
include/opengl/handles/shader_handle.hpp
include/opengl/handles/shader_program_handle.hpp
include/opengl/handles/surface_properties_handle.hpp
include/opengl/handles/texture_handle.hpp
include/opengl/handles/vertex_buffer_handle.hpp
include/opengl/metadata/index_buffer_metadata.hpp
include/opengl/metadata/material_metadata.hpp
include/opengl/metadata/mesh_vertex_buffer_metadata.hpp
include/opengl/metadata/point_cloud_vertex_buffer_metadata.hpp
include/opengl/metadata/preprocessed_shader_source_metadata.hpp
include/opengl/metadata/shader_metadata.hpp
include/opengl/metadata/shader_program_metadata.hpp
include/opengl/metadata/shader_set_metadata.hpp
include/opengl/metadata/shader_source_metadata.hpp
include/opengl/metadata/texture_metadata.hpp
include/opengl/metadata/uniform_buffer_alignment_metadata.hpp
include/opengl/resource_management/reference_counter.hpp
include/opengl/resource_management/reference_counter.hpp
include/opengl/resource_management/resource_handle.hpp
include/opengl/resource_management/resource_manager.hpp
include/opengl/shader_program_lookup.hpp
include/opengl/shading/attributes/mesh_attributes.hpp
include/opengl/shading/attributes/point_cloud_attributes.hpp
include/opengl/shading/features/combined_features.hpp
include/opengl/shading/features/generic_features.hpp
include/opengl/shading/features/mesh_features.hpp
include/opengl/shading/features/point_cloud_features.hpp
include/opengl/shading/model_geometry.hpp
include/opengl/shading/requirements/shader_program_requirements.hpp
include/opengl/shading/requirements/shader_requirements.hpp
include/opengl/shading/sampler_uniform.hpp
include/opengl/shading/sampler_uniforms/mesh_samplers_uniforms.hpp
include/opengl/shading/shader_metadata_language.hpp
include/opengl/shading/shader_stage.hpp
include/opengl/shading/uniform_block.hpp
include/opengl/shading/uniform_blocks/mesh_uniform_blocks.hpp
include/opengl/shading/uniform_blocks/point_cloud_uniform_blocks.hpp
include/opengl/shading/uniform.hpp
include/opengl/shading/uniforms/mesh_uniforms.hpp
include/opengl/shading/uniforms/point_cloud_uniforms.hpp
include/opengl/type_utils.hpp
include/opengl/types.hpp
include/rendering/batch_renderers/mesh_batch_renderer.hpp
include/rendering/batch_renderers/point_cloud_batch_renderer.hpp
include/rendering/batches/mesh_batch.hpp
include/rendering/batches/point_cloud_batch.hpp
include/rendering/modes/mesh_modes.hpp
include/rendering/modes/point_cloud_modes.hpp
include/rendering/requirements/mesh_requirements.hpp
include/rendering/requirements/point_cloud_requirements.hpp
include/rendering/shader_program_lookups/mesh_lookup.hpp
include/rendering/shader_program_lookups/mesh_lookup.hpp
include/rendering/shader_program_lookups/point_cloud_lookup.hpp
include/scene/camera.hpp
include/scene/flying_camera.hpp
include/scene/lighting_setup.hpp
include/util/arx.hpp
include/util/binary_ifstream.hpp
include/util/enum_bitfield_operators.hpp
@@ -173,16 +173,16 @@ add_executable(z3d main.cpp
include/util/uix.hpp
include/util/unroll_bool_template.hpp
include/util/vector_replace_range.hpp
# include/viewer/asset_loader.hpp
# include/viewer/asset_types.hpp
# include/viewer/dynamic_shader_program_loading.hpp
# include/viewer/instance.hpp
# include/viewer/settings.hpp
# source/assets/data_loaders/generic/base_dynamic_loader.ipp
# source/assets/data_loaders/mesh_loader.cpp
# source/assets/data_loaders/mesh_loader.cpp
# source/assets/data_loaders/point_cloud_loader.cpp
# source/assets/data_loaders/texture_loader.cpp
include/viewer/asset_loader.hpp
include/viewer/asset_types.hpp
include/viewer/dynamic_shader_program_loading.hpp
include/viewer/instance.hpp
include/viewer/settings.hpp
source/assets/data_loaders/generic/base_dynamic_loader.ipp
source/assets/data_loaders/mesh_loader.cpp
source/assets/data_loaders/mesh_loader.cpp
source/assets/data_loaders/point_cloud_loader.cpp
source/assets/data_loaders/texture_loader.cpp
source/assets/data_stores/generic/generic_basic_store.ipp
source/assets/data_stores/generic/generic_material_store.ipp
source/assets/data_stores/generic/generic_mesh_store.ipp
@@ -192,54 +192,92 @@ add_executable(z3d main.cpp
source/assets/data/mesh_data.ipp
source/assets/data/point_cloud_data.ipp
source/assets/data/texture_data.ipp
# source/assets/fallback_data/default_texture_data.cpp
# source/assets/fallback_data/fallback_material.cpp
# source/assets/file_parsers/generic/generic_3dtk_loader.ipp
# source/assets/file_parsers/glsl_parser.cpp
# source/assets/file_parsers/kitti_parser.cpp
# source/assets/file_parsers/kitti_pose_parser.cpp
source/assets/fallback_data/default_texture_data.cpp
source/assets/fallback_data/fallback_material.cpp
source/assets/file_parsers/generic/generic_3dtk_loader.ipp
source/assets/file_parsers/glsl_parser.cpp
source/assets/file_parsers/kitti_parser.cpp
source/assets/file_parsers/kitti_pose_parser.cpp
source/assets/file_parsers/mtl_parser.cpp
source/assets/file_parsers/obj_parser.cpp
include/util/array_hash.hpp
include/util/vector_hash.hpp
# source/assets/file_parsers/stl_loader.cpp
# source/assets/file_parsers/threedtk_pose_loader.cpp
# source/geometry/normal_estimation.cpp
# source/opengl/data_managers/mesh_index_buffer_manager.cpp
# source/opengl/data_managers/mesh_vertex_buffer_manager.cpp
# source/opengl/data_managers/point_cloud_vertex_buffer_manager.cpp
# source/opengl/data_managers/shader_manager.cpp
# source/opengl/data_managers/shader_program_manager.cpp
# source/opengl/data_managers/shader_source_manager.cpp
# source/opengl/data_managers/texture_manager.cpp
# source/opengl/data/material_data.ipp
# source/opengl/data/mesh_data.cpp
# source/opengl/data/point_cloud_data.cpp
# source/opengl/data/point_cloud_data.ipp
# source/opengl/data/shader_data.cpp
# source/opengl/data/shader_data.ipp
# source/opengl/data/shader_program_data.cpp
# source/opengl/data/texture_data.ipp
# source/opengl/handles/index_buffer_handle.ipp
# source/opengl/handles/mesh_handle.ipp
# source/opengl/handles/point_cloud_handle.ipp
# source/opengl/handles/shader_program_handle.cpp
# source/opengl/handles/shader_program_handle.ipp
# source/opengl/handles/texture_handle.ipp
# source/opengl/handles/vertex_buffer_handle.ipp
# source/opengl/resource_management/resource_handle.ipp
# source/opengl/resource_management/resource_manager.ipp
# source/opengl/shader_program_lookup.cpp
# source/rendering/batch_renderers/mesh_batch_renderer.cpp
# source/rendering/batch_renderers/point_cloud_batch_renderer.cpp
# source/rendering/batches/mesh_batch.ipp
# source/rendering/batches/point_cloud_batch.ipp
# source/rendering/shader_program_lookups/mesh_lookup.cpp
# source/rendering/shader_program_lookups/point_cloud_lookup.cpp
# source/scene/flying_camera.cpp
# source/viewer/asset_loader.cpp
# source/viewer/dynamic_shader_program_loading.cpp
# source/viewer/instance.cpp
source/assets/file_parsers/stl_loader.cpp
source/assets/file_parsers/threedtk_pose_loader.cpp
source/geometry/normal_estimation.cpp
source/opengl/data_managers/mesh_index_buffer_manager.cpp
source/opengl/data_managers/mesh_vertex_buffer_manager.cpp
source/opengl/data_managers/point_cloud_vertex_buffer_manager.cpp
source/opengl/data_managers/shader_manager.cpp
source/opengl/data_managers/shader_program_manager.cpp
source/opengl/data_managers/shader_source_manager.cpp
source/opengl/data_managers/texture_manager.cpp
source/opengl/data/material_data.ipp
source/opengl/data/mesh_data.cpp
source/opengl/data/point_cloud_data.cpp
source/opengl/data/point_cloud_data.ipp
source/opengl/data/shader_data.cpp
source/opengl/data/shader_data.ipp
source/opengl/data/shader_program_data.cpp
source/opengl/data/texture_data.ipp
source/opengl/handles/index_buffer_handle.ipp
source/opengl/handles/mesh_handle.ipp
source/opengl/handles/point_cloud_handle.ipp
source/opengl/handles/shader_program_handle.cpp
source/opengl/handles/shader_program_handle.ipp
source/opengl/handles/texture_handle.ipp
source/opengl/handles/vertex_buffer_handle.ipp
source/opengl/resource_management/resource_handle.ipp
source/opengl/resource_management/resource_manager.ipp
source/opengl/shader_program_lookup.cpp
source/rendering/batch_renderers/mesh_batch_renderer.cpp
source/rendering/batch_renderers/point_cloud_batch_renderer.cpp
source/rendering/batches/mesh_batch.ipp
source/rendering/batches/point_cloud_batch.ipp
source/rendering/shader_program_lookups/mesh_lookup.cpp
source/rendering/shader_program_lookups/point_cloud_lookup.cpp
source/scene/flying_camera.cpp
source/viewer/asset_loader.cpp
source/viewer/dynamic_shader_program_loading.cpp
source/viewer/instance.cpp
)
target_include_directories(z3d PRIVATE include)

40
include/assets/file_parsers/generic/generic_3dtk_loader.hpp
View File

@@ -13,7 +13,7 @@
#include "assets/data/point_cloud_data.hpp"
#include "assets/data_stores/point_cloud_store.hpp"
namespace assets
namespace assets::detail
{
template<bool Normal, bool Color, bool Reflectance>
@@ -47,41 +47,27 @@ protected:
protected:
void reset();
z3d::vertex_index find_or_push_vertex(const component_indices& vertex_comp_indices);
[[nodiscard]] std::optional<texture_id> fetch_texture_id(
const std::filesystem::path& mtl_dir,
std::string_view filename,
std::string_view texture_type_name
);
private:
path_id_lookups* m_id_lookups;
data_stores* m_stores;
data_type m_mesh{};
std::vector<mesh_vertex_components::position> m_position_buffer{};
std::vector<mesh_vertex_components::normal> m_normal_buffer{};
std::vector<mesh_vertex_components::tex_coord> m_tex_coord_buffer{};
std::unordered_map<component_indices, z3d::vertex_index> m_vertex_comp_indices_to_vertex_index{};
data_type m_point_cloud{};
std::filesystem::path m_last_pose_path{};
pose_list_view m_last_pose_list{};
};
[[nodiscard]] static ztu::result<pose_prefetch_lookup::index_type> parse_index(
[[nodiscard]] static ztu::result<std::size_t> parse_scan_index(
std::string_view filename
);
ztu::result<std::pair<ztu::u32, std::chars_format>> analyze_component_format(
std::string_view line
);
void transform_point_cloud(
std::span<point_cloud_vertex_components::position::value_type> points,
const glm::mat4& pose
);
private:
std::error_code read_point_file(
protected:
[[nodiscard]] static std::error_code parse_file(
const std::filesystem::path& filename,
dynamic_point_cloud_buffer& point_cloud
data_type& point_cloud
);
static void transform_point_cloud(
std::span<point_cloud_vertex_components::position> points,
const pose_data& pose
);
private:

13
include/assets/file_parsers/kitti_parser.hpp
View File

@@ -39,9 +39,8 @@ protected:
{
public:
parser_context(
const pose_list_id_lookup& pose_list_lookup,
const pose_list_store& pose_list_store,
store_type& m_store,
path_id_lookups& pose_list_lookup,
data_stores& stores
);
void operator()(lookup_type::const_pointer entry) noexcept;
@@ -56,9 +55,8 @@ protected:
void remove_metadata_declarations();
private:
pose_list_id_lookup const* m_pose_list_lookup;
pose_list_store const* m_pose_list_store;
store_type* m_store;
path_id_lookups* m_pose_list_lookup;
data_stores* m_stores;
data_type m_buffer{};
std::filesystem::path m_last_pose_path{};
pose_list_view m_last_pose_list{};
@@ -69,7 +67,7 @@ protected:
std::string_view filename
);
[[nodiscard]] static std::error_code load_point_file(
[[nodiscard]] static std::error_code parse_file(
const std::filesystem::path& filename,
data_type& point_cloud
);
@@ -93,4 +91,3 @@ private:
};
}

50
include/assets/file_parsers/threedtk_pose_loader.hpp
View File

@@ -2,11 +2,10 @@
#include <filesystem>
#include "assets/prefetch_lookup.hpp"
#include "assets/prefetch_queue.hpp"
#include "assets/data_stores.hpp"
#include "assets/path_id_lookups.hpp"
#include "assets/data/pose_data.hpp"
#include "util/result.hpp"
#include "assets/prefetch_lookups/pose_prefetch_lookup.hpp"
namespace assets
{
@@ -14,30 +13,45 @@ namespace assets
struct threedtk_pose_loader
{
static constexpr auto name = std::string_view("3dtk_pose");
using data_type = pose_list_data;
using store_type = pose_list_store;
using lookup_type = pose_list_id_lookup;
[[nodiscard]] static std::error_code prefetch(
const file_dir_list& paths,
prefetch_queue& queue
[[nodiscard]] std::error_code prefetch(
path_id_lookups& lookups
);
[[nodiscard]] static std::error_code load(
pose_data& buffer,
const file_dir_list& paths,
prefetch_lookup& id_lookup,
shader_source_store& store,
[[nodiscard]] std::error_code load(
path_id_lookups& lookups,
data_stores& stores,
bool pedantic = false
);
protected:
static std::error_code parse_transform_info(
std::ifstream& in,
std::string& line,
std::array<glm::vec3, 2>& transform_info
class parser_context
{
public:
parser_context(
path_id_lookups& pose_list_lookup,
data_stores& stores
);
void operator()(lookup_type::const_pointer entry) noexcept;
private:
path_id_lookups* m_id_lookups;
data_stores* m_stores;
data_type m_pose{};
std::string m_line_buffer{};
};
static ztu::result<pose_data> parse_file(
const std::filesystem::path& filename,
std::string& line_buffer
);
static ztu::result<pose_prefetch_lookup::index_type> parse_index(
std::string_view filename
);
private:
std::vector<lookup_type::const_pointer> m_path_buffer;
};
}

6
include/assets/file_parsers/uos_loader.hpp
View File

@@ -2,8 +2,12 @@
#include "generic/generic_3dtk_loader.hpp"
struct uos_loader : generic_3dtk_loader<false, false, false>
namespace assets
{
struct uos_loader : detail::generic_3dtk_loader<false, false, false>
{
static constexpr auto name = std::string_view("uos");
};
}

7
include/assets/file_parsers/uos_normal_loader.hpp
View File

@@ -2,7 +2,12 @@
#include "generic/generic_3dtk_loader.hpp"
struct uos_normal_loader : generic_3dtk_loader<true, false, false>
namespace assets
{
struct uos_normal_loader : detail::generic_3dtk_loader<true, false, false>
{
static constexpr auto name = std::string_view("uos_normal");
};
}

7
include/assets/file_parsers/uos_rgb_loader.hpp
View File

@@ -2,7 +2,12 @@
#include "generic/generic_3dtk_loader.hpp"
struct uos_rgb_loader : generic_3dtk_loader<false, true, false>
namespace assets
{
struct uos_rgb_loader : detail::generic_3dtk_loader<false, true, false>
{
static constexpr auto name = std::string_view("uos_rgb");
};
}

7
include/assets/file_parsers/uosr_loader.hpp
View File

@@ -2,7 +2,12 @@
#include "generic/generic_3dtk_loader.hpp"
struct uosr_loader : generic_3dtk_loader<false, false, true>
namespace assets
{
struct uosr_loader : detail::generic_3dtk_loader<false, false, true>
{
static constexpr auto name = std::string_view("uosr");
};
}

413
source/assets/file_parsers/generic/generic_3dtk_loader.ipp
View File

@@ -14,11 +14,11 @@
template<bool Normal, bool Color, bool Reflectance>
std::error_code assets::generic_3dtk_loader<Normal, Color, Reflectance>::prefetch(
std::error_code assets::detail::generic_3dtk_loader<Normal, Color, Reflectance>::prefetch(
path_id_lookups& lookups
) {
m_path_buffer.clear();
lookups.meshes.by_extension(".3d", m_path_buffer);
lookups.point_clouds.by_extension(".3d", m_path_buffer);
auto path_buffer = std::filesystem::path{};
@@ -33,7 +33,7 @@ std::error_code assets::generic_3dtk_loader<Normal, Color, Reflectance>::prefetc
}
template<bool Normal, bool Color, bool Reflectance>
std::error_code assets::generic_3dtk_loader<Normal, Color, Reflectance>::load(
std::error_code assets::detail::generic_3dtk_loader<Normal, Color, Reflectance>::load(
path_id_lookups& lookups,
data_stores& stores,
bool pedantic
@@ -55,7 +55,7 @@ std::error_code assets::generic_3dtk_loader<Normal, Color, Reflectance>::load(
}
template<bool Normal, bool Color, bool Reflectance>
assets::generic_3dtk_loader<Normal, Color, Reflectance>::parser_context::parser_context(
assets::detail::generic_3dtk_loader<Normal, Color, Reflectance>::parser_context::parser_context(
path_id_lookups& m_id_lookups,
data_stores& m_stores
) :
@@ -63,47 +63,28 @@ assets::generic_3dtk_loader<Normal, Color, Reflectance>::parser_context::parser_
m_stores{ &m_stores }
{
constexpr auto expected_vertex_count = 8192;
m_mesh.positions().reserve(expected_vertex_count);
m_mesh.normals().reserve(expected_vertex_count);
m_mesh.colors().reserve(expected_vertex_count);
m_mesh.reflectances().reserve(expected_vertex_count);
m_mesh.tex_coords().reserve(expected_vertex_count);
m_mesh.triangles().reserve(2 * expected_vertex_count);
m_position_buffer.reserve(expected_vertex_count);
m_normal_buffer.reserve(expected_vertex_count);
m_tex_coord_buffer.reserve(expected_vertex_count);
m_point_cloud.positions().reserve(expected_vertex_count);
m_point_cloud.normals().reserve(expected_vertex_count);
m_point_cloud.colors().reserve(expected_vertex_count);
m_point_cloud.reflectances().reserve(expected_vertex_count);
}
template<bool Normal, bool Color, bool Reflectance>
void assets::generic_3dtk_loader<Normal, Color, Reflectance>::parser_context::reset()
void assets::detail::generic_3dtk_loader<Normal, Color, Reflectance>::parser_context::reset()
{
m_mesh.clear();
m_position_buffer.clear();
m_normal_buffer.clear();
m_tex_coord_buffer.clear();
m_vertex_comp_indices_to_vertex_index.clear();
m_point_cloud.clear();
}
template<bool Normal, bool Color, bool Reflectance>
void assets::generic_3dtk_loader<Normal, Color, Reflectance>::parser_context::operator()(lookup_type::const_pointer entry) noexcept
void assets::detail::generic_3dtk_loader<Normal, Color, Reflectance>::parser_context::operator()(lookup_type::const_pointer entry) noexcept
{
// TODO look up pose
const auto& [ filename, id ] = *entry;
auto in = std::ifstream{ filename };
if (not in.is_open())
std::size_t pose_index;
if (auto res = parse_scan_index(filename))
{
ztu::logger::warn("Cannot open obj file %.", filename);
return;
}
std::size_t scan_index;
if (auto res = parse_index(filename))
{
scan_index = *res;
pose_index = *res;
}
else [[unlikely]]
{
@@ -113,40 +94,77 @@ void assets::generic_3dtk_loader<Normal, Color, Reflectance>::parser_context::op
error.category().name(),
error.message()
);
}
auto pose = asset_lookup.poses.find();
const auto id_it = id_lookup.find(filename);
if (id_it != id_lookup.end()) [[unlikely]]
{
return;
}
in.open(filename);
if (in.is_open())
auto pose_path = entry->first;
pose_path.replace_extension(".pose");
if (pose_path != m_last_pose_path)
{
if ((error = this->read_point_file(filename, buffer))) {
return error;
if (const auto pose_list_id_it = m_id_lookups->pose_lists.find(pose_path); pose_list_id_it != m_id_lookups->pose_lists.end())
{
m_last_pose_path = pose_path;
const auto pose_list_id = pose_list_id_it->second;
// TODO What if the netry is not there we look it pup again and again
// TODO Insert maybe a bool was_valid
if (const auto [ pose_list_it, found ] = m_id_lookups->poses.find(pose_list_id); found)
{
m_last_pose_list = pose_list_it->second;
}
else
{
m_last_pose_list = {};
ztu::logger::error("No matching pose found in store for %.", pose_path);
return;
}
}
else
{
ztu::logger::error("No matching pose registered in lookup %.", pose_path);
return;
}
}
else if (m_last_pose_list.data() == nullptr)
{
ztu::logger::error("No matching pose found in store for %.", pose_path);
return;
}
this->transform_point_cloud(buffer.positions(), pose);
reset();
const auto id = store.add(std::move(buffer));
id_lookup.emplace_hint(id_it, filename, id);
pose_data pose;
if (pose_index < m_last_pose_list.size())
{
pose = m_last_pose_list[pose_index];
}
else
{
ztu::logger::error("Cannot open 3dtk file %", filename);
ztu::logger::error(
"Pose index % of 3dtk file % is out of range for its pose file %. Proceeding with identity pose.",
pose_index,
filename,
m_last_pose_path
);
pose = glm::identity<pose_data>();
}
in.close();
if (const auto e = parse_file(filename, m_point_cloud))
{
ztu::logger::error("Could not parse 3dtk file %: %", filename, e.message());
return;
}
transform_point_cloud(m_point_cloud.positions(), pose);
m_stores->point_clouds.insert(id, m_point_cloud);
}
template<bool Normal, bool Color, bool Reflectance>
ztu::result<pose_prefetch_lookup::index_type> generic_3dtk_loader<Normal, Color, Reflectance>::parse_index(
ztu::result<std::size_t> assets::detail::generic_3dtk_loader<Normal, Color, Reflectance>::parse_scan_index(
const std::string_view filename
) {
static constexpr auto prefix = std::string_view{ "scan" };
@@ -155,164 +173,26 @@ ztu::result<pose_prefetch_lookup::index_type> generic_3dtk_loader<Normal, Color,
if (name_view.length() <= prefix.length()) [[unlikely]]
{
return std::make_error_code(std::errc::invalid_argument);
return std::unexpected{ std::make_error_code(std::errc::invalid_argument) };
}
name_view = name_view.substr(prefix.length());
pose_prefetch_lookup::index_type index;
std::size_t index;
const auto res = std::from_chars(name_view.begin(), name_view.end(), index);
if (res.ec != std::errc{}) [[unlikely]]
{
return std::make_error_code(res.ec);
return std::unexpected{ std::make_error_code(res.ec) };
}
return index;
}
template<bool Normal, bool Color, bool Reflectance>
std::error_code assets::generic_3dtk_loader<Normal, Color, Reflectance>::load(
path_id_lookups& lookups,
data_stores& stores,
bool pedantic = false
) {
namespace fs = std::filesystem;
auto in = std::ifstream{};
auto path_buffer = fs::path{};
auto error = std::error_code{};
const auto load_file = [&](const char* filename)
{
// TODO look up pose
auto scan_index = pose_prefetch_lookup::index_type{};
if (auto res = parse_index(filename))
{
scan_index = *res;
}
else [[unlikely]]
{
error = res.error();
ztu::logger::error(
"Error occurred while parsing scan index in filename %: [%] %",
error.category().name(),
error.message()
);
}
auto pose = asset_lookup.poses.find();
const auto id_it = id_lookup.find(filename);
if (id_it != id_lookup.end()) [[unlikely]]
{
return;
}
in.open(filename);
if (in.is_open())
{
if ((error = this->read_point_file(filename, buffer))) {
return error;
}
this->transform_point_cloud(buffer.positions(), pose);
const auto id = store.add(std::move(buffer));
id_lookup.emplace_hint(id_it, filename, id);
}
else
{
ztu::logger::error("Cannot open 3dtk file %", filename);
}
in.close();
};
for (const auto filename : paths.files)
{
load_file(filename.data());
}
for (const auto directory : paths.directories)
{
directory_buffer.assign(directory.begin(), directory.end());
directory_buffer /= "frames";
const auto directory_exists = not fs::is_directory(directory_buffer, error);
if (error or not directory_exists) [[unlikely]]
{
ztu::logger::error("Could not open point cloud directory %", directory_buffer);
continue;
}
for (const auto& filename : fs::directory_iterator{ directory_buffer }) {
auto point_filename = reinterpret_cast<const fs::path&>(filename);
if (point_filename.extension() != ".3d")
{
continue;
}
load_file(filename.c_str());
}
}
return {};
}
template<bool Normal, bool Color, bool Reflectance>
std::error_code generic_3dtk_loader<Normal, Color, Reflectance>::load_directory(
dynamic_data_loader_ctx& ctx,
dynamic_point_cloud_store& store,
const std::filesystem::path& path,
const bool pedantic
) {
namespace fs = std::filesystem;
std::error_code error;
const auto directory_exists = not fs::is_directory(path, error);
if (error)
{
return error;
}
if (not directory_exists)
{
return make_error_code(std::errc::no_such_file_or_directory);
}
for (const auto& filename : fs::directory_iterator{ path / "frames" }) {
auto point_filename = reinterpret_cast<const fs::path&>(filename);
if (point_filename.extension() != ".3d") {
continue;
}
if ((error = load(ctx, store, point_filename, pedantic)))
{
ztu::logger::error(
"Error while loading point cloud '%': [%] %",
point_filename,
error.category().name(),
error.message()
);
}
}
return {};
}
template<typename T, std::size_t Count>
std::error_code read_vector(std::string_view& line, std::array<T, Count>& vec) {
for (auto& component : vec)
template<int L>
std::error_code read_vector(std::string_view& line, z3d::vec<L, float>& vec) {
for (int i{}; i != L; ++i)
{
auto it = line.begin();
@@ -321,7 +201,7 @@ std::error_code read_vector(std::string_view& line, std::array<T, Count>& vec) {
const auto [ ptr, ec ] = std::from_chars(
it, line.end(),
component,
vec[i],
std::chars_format::hex
);
@@ -330,8 +210,9 @@ std::error_code read_vector(std::string_view& line, std::array<T, Count>& vec) {
return std::make_error_code(ec);
}
if (minus) {
component *= -1.0;
if (minus)
{
vec[i] *= -1.0;
}
line = { ptr + sizeof(' '), line.end() };
@@ -341,15 +222,16 @@ std::error_code read_vector(std::string_view& line, std::array<T, Count>& vec) {
}
template<bool Normal, bool Color, bool Reflectance>
std::error_code generic_3dtk_loader<Normal, Color, Reflectance>::read_point_file(
std::error_code assets::detail::generic_3dtk_loader<Normal, Color, Reflectance>::parse_file(
const std::filesystem::path& filename,
dynamic_point_cloud_data& point_cloud
data_type& point_cloud
) {
std::error_code error;
auto in = std::ifstream(filename);
if (not in.is_open()) {
if (not in.is_open())
{
return std::make_error_code(static_cast<std::errc>(errno));
}
@@ -381,18 +263,6 @@ std::error_code generic_3dtk_loader<Normal, Color, Reflectance>::read_point_file
return count;
}();
ztu::u32 component_count;
std::chars_format float_format;
if ((error = analyze_component_format(line, component_count, float_format)))
{
return error;
}
if (component_count != expected_component_count)
{
return std::make_error_code(std::errc::invalid_argument);
}
auto& positions = point_cloud.positions();
auto& normals = point_cloud.normals();
@@ -432,12 +302,12 @@ std::error_code generic_3dtk_loader<Normal, Color, Reflectance>::read_point_file
if constexpr (Reflectance)
{
point_cloud_vertex_components::reflectance reflectance;
z3d::vec<1, point_cloud_vertex_components::reflectance> reflectance;
if ((error = read_vector(line_view, reflectance)))
{
return error;
}
reflectances.push_back(reflectance);
reflectances.push_back(reflectance[0]);
}
}
while (std::getline(in, line));
@@ -445,110 +315,13 @@ std::error_code generic_3dtk_loader<Normal, Color, Reflectance>::read_point_file
return {};
}
std::error_code base_3dtk_loader::read_pose_file(
const std::filesystem::path& filename,
glm::mat4& pose
template<bool Normal, bool Color, bool Reflectance>
void assets::detail::generic_3dtk_loader<Normal, Color, Reflectance>::transform_point_cloud(
std::span<point_cloud_vertex_components::position> points,
const pose_data& pose
) {
auto in = std::ifstream(filename);
if (not in.is_open()) {
return std::make_error_code(static_cast<std::errc>(errno));
}
std::string line;
std::array<glm::vec3, 2> numbers{};
for (std::size_t row{}; row != 2; ++row) {
std::getline(in, line);
auto it = line.cbegin().base();
auto end = line.cend().base();
for (glm::vec3::length_type col{}; col != 3; ++col) {
const auto [ ptr, ec ] = std::from_chars(
it, end,
numbers[row][col],
std::chars_format::general
);
if (ec != std::errc{}) {
return std::make_error_code(ec);
}
it = ptr + 1; // skip space in between components
}
}
const auto& translation = numbers[0];
auto& angles = numbers[1];
angles *= static_cast<float>(M_PI / 180.0);
pose = (
glm::translate(glm::identity<glm::mat4>(), translation) *
glm::eulerAngleXYZ(angles[0], angles[1], angles[2])
);
return {};
}
std::error_code base_3dtk_loader::analyze_component_format(
const std::string& line,
ztu::u32& component_count,
std::chars_format& format
) {
auto begin = line.cbegin().base();
auto end = line.cend().base();
format = std::chars_format::general;
component_count = 0;
float buffer;
for (auto it = begin; it < end; it += sizeof(' '))
for (auto& point : points)
{
it += *it == '-' or *it == '+';
std::chars_format current_format;
if (*it == '0' and std::next(it) < end and *std::next(it) == 'x')
{
it += 2; // skip '0x'
current_format = std::chars_format::hex;
}
else
{
current_format = std::chars_format::general;
}
if (it == begin and current_format != format)
{
return std::make_error_code(std::errc::invalid_argument);
}
const auto [next_it, err] = std::from_chars(it, end, buffer, current_format);
if (err != std::errc())
{
return std::make_error_code(err);
}
it = next_it;
format = current_format;
++component_count;
}
return {};
}
void base_3dtk_loader::transform_point_cloud(
std::span<point_cloud_vertex_components::position::value_type> points,
const glm::mat4& pose
) {
for (auto& [ x, y, z ] : points) {
auto vec = glm::vec4{ x, y, z, 1.0f };
vec = pose * vec;
x = vec.x;
y = vec.y;
z = vec.z;
point = pose * glm::vec4{ point, 1.0f };
}
}

62
source/assets/file_parsers/kitti_parser.cpp
View File

@@ -14,13 +14,11 @@
assets::kitti_parser::parser_context::parser_context(
const pose_list_id_lookup& pose_list_lookup,
const pose_list_store& pose_list_store,
store_type& m_store
path_id_lookups& pose_list_lookup,
data_stores& stores
) :
m_pose_list_lookup{ &pose_list_lookup },
m_pose_list_store{ &pose_list_store },
m_store{ &m_store }
m_stores{ &stores }
{
constexpr auto expected_vertex_count = 8192;
m_buffer.positions().reserve(expected_vertex_count);
@@ -37,20 +35,28 @@ void assets::kitti_parser::parser_context::operator()(lookup_type::const_pointer
{
const auto& [ filename, id ] = *entry;
pose_data pose;
if (const auto pose_path = get_pose_path(filename); not pose_path)
std::size_t pose_index;
if (const auto res = frame_id_from_filename(filename.c_str()))
{
if (pose_path != m_last_pose_path)
pose_index = res.value;
}
else
{
ztu::logger::error("Could not parse frame id from kitti filename %: %", filename, res.error());
return;
}
if (const auto pose_path = get_pose_path(filename))
{
if (*pose_path != m_last_pose_path)
{
if (const auto pose_list_id_it = m_pose_list_lookup->find(*pose_path); pose_list_id_it != m_pose_list_lookup->end())
if (const auto pose_list_id_it = m_pose_list_lookup->pose_lists.find(*pose_path); pose_list_id_it != m_pose_list_lookup->pose_lists.end())
{
m_last_pose_path = *pose_path;
const auto pose_list_id = pose_list_id_it->second;
if (const auto [ pose_list_it, found ] = m_pose_list_store->find(pose_list_id); found)
if (const auto [ pose_list_it, found ] = m_pose_list_lookup->poses.find(pose_list_id); found)
{
m_last_pose_path = *pose_path;
m_last_pose_list = pose_list_it->second;
}
else
@@ -74,15 +80,32 @@ void assets::kitti_parser::parser_context::operator()(lookup_type::const_pointer
reset();
if (const auto e = load_point_file(filename, m_buffer))
pose_data pose;
if (pose_index < m_last_pose_list.size())
{
ztu::logger::error("Could not load kitti file %: %", filename, e.message());
pose = m_last_pose_list[pose_index];
}
else
{
ztu::logger::error(
"Pose index % of kitti file % is out of range for its pose file %. Proceeding with identity pose.",
pose_index,
filename,
m_last_pose_path
);
pose = glm::identity<pose_data>();
}
if (const auto e = parse_file(filename, m_buffer))
{
ztu::logger::error("Could not parse kitti file %: %", filename, e.message());
return;
}
transform_point_cloud(m_buffer.positions(), pose);
m_store->insert(id, m_buffer);
m_stores->point_clouds.insert(id, m_buffer);
}
ztu::result<std::filesystem::path> assets::kitti_parser::parent_directory(
@@ -138,8 +161,6 @@ std::error_code assets::kitti_parser::load(
data_stores& stores,
bool pedantic
) {
namespace fs = std::filesystem;
m_path_buffer.clear();
lookups.point_clouds.by_extension(".bin", m_path_buffer);
@@ -148,9 +169,8 @@ std::error_code assets::kitti_parser::load(
m_path_buffer.begin(),
m_path_buffer.end(),
parser_context{
lookups.pose_lists,
stores.pose_lists,
stores.point_clouds
lookups,
stores
}
);
@@ -167,7 +187,7 @@ void assets::kitti_parser::transform_point_cloud(
}
}
std::error_code assets::kitti_parser::load_point_file(
std::error_code assets::kitti_parser::parse_file(
const std::filesystem::path& filename,
point_cloud_data& point_cloud
) {

194
source/assets/file_parsers/threedtk_pose_loader.cpp
View File

@@ -1,137 +1,111 @@
#include "assets/file_parsers/threedtk_pose_loader.hpp"
#include <fstream>
#include <execution>
#include <glm/ext/matrix_transform.hpp>
#include <glm/gtx/euler_angles.hpp>
#include "util/logger.hpp"
#include <fstream>
inline std::error_code threedtk_pose_loader::parse_transform_info(
std::ifstream& in,
std::string& line,
std::array<glm::vec3, 2>& transform_info
std::error_code assets::threedtk_pose_loader::prefetch(
path_id_lookups& lookups
) {
return {};
}
std::error_code assets::threedtk_pose_loader::load(
path_id_lookups& lookups,
data_stores& stores,
bool pedantic
) {
m_path_buffer.clear();
lookups.poses.by_extension(".pose", m_path_buffer);
std::for_each(
std::execution::parallel_unsequenced_policy{},
m_path_buffer.begin(),
m_path_buffer.end(),
parser_context{
lookups,
stores
}
);
return {};
}
assets::threedtk_pose_loader::parser_context::parser_context(
path_id_lookups& m_id_lookups,
data_stores& m_stores
) :
m_id_lookups{ &m_id_lookups },
m_stores{ &m_stores }
{
m_line_buffer.reserve(64);
}
void assets::threedtk_pose_loader::parser_context::operator()(lookup_type::const_pointer entry) noexcept
{
const auto& [ filename, id ] = *entry;
if (const auto res = parse_file(filename, m_line_buffer))
{
m_stores->poses.insert(id, res.value());
}
else
{
const auto error = res.error();
ztu::logger::error(
"Error while parsing 3dtk pose file %: [%] %",
filename,
error.category().name(),
error.message()
);
}
}
ztu::result<assets::pose_data> assets::threedtk_pose_loader::parse_file(
const std::filesystem::path& filename,
std::string& line_buffer
) {
auto in = std::ifstream{ filename };
if (not in.is_open())
{
return std::unexpected{ std::make_error_code(std::errc::no_such_file_or_directory) };
}
std::array<glm::vec3, 2> transform_vecs{};
for (std::size_t row{}; row != 2; ++row) {
std::getline(in, line);
std::getline(in, line_buffer);
auto it = line.cbegin().base();
auto end = line.cend().base();
auto it = line_buffer.cbegin().base();
auto end = line_buffer.cend().base();
for (glm::vec3::length_type col{}; col != 3; ++col) {
const auto [ ptr, ec ] = std::from_chars(
it, end,
transform_info[row][col],
transform_vecs[row][col],
std::chars_format::general
);
if (ec != std::errc{}) {
return std::make_error_code(ec);
return std::unexpected{ std::make_error_code(ec) };
}
it = ptr + 1; // skip space in between components
}
}
}
inline ztu::result<pose_prefetch_lookup::index_type> threedtk_pose_loader::parse_index(
const std::string_view filename
) {
static constexpr auto prefix = std::string_view{ "scan" };
auto name_view = filename.substr(0, name_view.find('.'));
if (name_view.length() <= prefix.length()) [[unlikely]]
{
return std::make_error_code(std::errc::invalid_argument);
}
name_view = name_view.substr(prefix.length());
pose_prefetch_lookup::index_type index;
const auto res = std::from_chars(name_view.begin(), name_view.end(), index);
if (res.ec != std::errc{}) [[unlikely]]
{
return std::make_error_code(res.ec);
}
return index;
}
void threedtk_pose_loader::load(
const ztu::string_list& filenames,
dynamic_pose_store& store,
pose_prefetch_lookup& id_lookup
) {
auto filename_buffer = std::string{};
auto in = std::ifstream{};
auto line = std::string{};
auto pose_buffer = dynamic_pose_buffer{};
for (const auto filename : filenames)
{
pose_prefetch_lookup::index_type index;
if (const auto res = parse_index(filename))
{
index = *res;
}
else
{
const auto error = res.error();
ztu::logger::error(
"Error while parsing 3dtk pose file index %: [%] %",
filename,
error.category().name(),
error.message()
);
}
filename_buffer = filename;
in.open(filename_buffer.c_str());
if (not in.is_open()) {
ztu::logger::error("Cannot open 3dtk pose file %", filename);
continue;
}
std::array<glm::vec3, 2> transform_info{};
const auto error = parse_transform_info(in, line, transform_info);
in.close();
if (error)
{
ztu::logger::error(
"Error while parsing 3dtk pose file %: [%] %",
filename,
error.category().name(),
error.message()
);
continue;
}
const auto& translation = transform_info[0];
auto& angles = transform_info[1];
angles *= static_cast<float>(M_PI / 180.0);
pose_buffer = (
glm::translate(glm::identity<glm::mat4>(), translation) *
glm::eulerAngleXYZ(angles[0], angles[1], angles[2])
);
const auto id = store.add(pose_buffer);
id_lookup.emplace(filename, index, id);
}
const auto& translation = transform_vecs[0];
auto& angles = transform_vecs[1];
angles *= static_cast<float>(M_PI / 180.0);
return (
glm::translate(glm::identity<glm::mat4>(), translation) *
glm::eulerAngleXYZ(angles[0], angles[1], angles[2])
);
}