zy4n/Z3D
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

In the middle of multithreading parsers.

Browse Source
This commit is contained in:
zy4n
2025-03-30 22:38:06 +02:00
parent d18b40a7fc
commit 144126ee7a
80 changed files with 2904 additions and 1450 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
include/assets/file_parsers/glsl_loader.hpp
View File

@@ -1,35 +0,0 @@
#pragma once
#include <filesystem>
#include <vector>
#include <string>
#include "assets/data_stores.hpp"
#include "assets/prefetch_lookup.hpp"
#include "assets/prefetch_queue.hpp"
#include "assets/prefetch_lookups/mesh_prefetch_lookup.hpp"
#include "assets/data/shader_source_data.hpp"
#include "util/string_list.hpp"
namespace assets
{
struct glsl_loader
{
static constexpr auto name = std::string_view("glsl");
[[nodiscard]] static std::error_code prefetch(
const file_dir_list& paths,
prefetch_queue& queue
);
[[nodiscard]] static std::error_code load(
shader_source_data& buffer,
const file_dir_list& paths,
prefetch_lookup& id_lookup,
shader_source_store& store,
bool pedantic = false
);
};
}

122
include/assets/file_parsers/glsl_parser.hpp Normal file
View File

@@ -0,0 +1,122 @@
#pragma once
#include <filesystem>
#include "assets/path_id_lookups.hpp"
#include "assets/data_stores/shader_source_store.hpp"
#include "assets/data/shader_source_data.hpp"
namespace assets
{
struct glsl_parser
{
static constexpr auto name = std::string_view("glsl");
using data_type = shader_source_data;
using store_type = shader_source_store;
using lookup_type = shader_source_id_lookup;
[[nodiscard]] std::error_code prefetch(
path_id_lookups& lookups
);
[[nodiscard]] std::error_code load(
const path_id_lookups& lookups,
store_type& store,
bool pedantic = false
);
protected:
class parser_context
{
public:
parser_context(
store_type& m_store,
std::mutex& m_store_mutex
);
void operator()(lookup_type::const_pointer entry) noexcept;
protected:
void reset();
void tokenize_declarations();
[[nodiscard]] bool parse_metadata_from_tokens();
void remove_metadata_declarations();
private:
store_type* m_store;
std::mutex* m_store_mutex;
shader_source_data m_buffer{};
std::vector<std::string_view> m_value_buffer{};
std::vector<std::size_t> m_declaration_value_count_buffer{};
std::array<std::size_t, 4> m_declaration_type_index_buffer{};
};
[[nodiscard]] static std::error_code read_file(
const std::filesystem::path& filename,
std::vector<char>& source
);
static void tokenize_declarations(
std::string_view source,
std::vector<std::string_view>& value_buffer,
std::vector<std::size_t>& declaration_value_count_buffer,
std::array<std::size_t, 4>& declaration_type_index_buffer
);
static [[nodiscard]] bool parse_metadata_from_tokens(
const std::vector<std::string_view>& value_buffer,
const std::vector<std::size_t>& declaration_value_count_buffer,
const std::array<std::size_t, 4>& declaration_type_index_buffer,
shader_source_data& buffer
);
static void remove_metadata_declarations(
const std::vector<std::string_view>& value_buffer,
const std::vector<std::size_t>& declaration_value_count_buffer,
const std::array<std::size_t, 4>& declaration_type_index_buffer,
std::vector<char>& source
);
[[nodiscard]] static bool parse_geometry_declaration(
std::span<const std::string_view> values,
model_geometry::types& geometry_type
);
[[nodiscard]] static bool parse_stage_declaration(
std::span<const std::string_view> values,
shader_components::stage& stage
);
[[nodiscard]] static bool parse_components_declaration(
std::span<const std::string_view> values,
shader_source_data& buffer
);
[[nodiscard]] static bool parse_static_enable_declaration(
std::span<const std::string_view> values,
shader_source_data& buffer
);
[[nodiscard]] static bool parse_dynamic_enable_declaration(
std::span<const std::string_view> values,
shader_source_data& buffer
);
[[nodiscard]] static bool parse_component_tokens(
std::span<const std::string_view> values,
model_geometry::types geometry_type,
shader_components::flags& components
);
private:
std::vector<std::string_view> m_value_token_buffer;
std::vector<std::size_t> m_declaration_token_count_buffer;
std::array<std::size_t, 4> m_declaration_type_index_buffer;
std::vector<lookup_type::const_pointer> m_path_buffer;
};
}

19
include/assets/file_parsers/interface.txt
View File

@@ -1,19 +0,0 @@
struct X {
[[nodiscard]] static std::error_code prefetch(
const file_dir_list& paths,
prefetch_queue& queue
);
[[nodiscard]] static std::error_code load(
// space stuff that has to persist
dynamic_X_buffer& buffer,
const file_dir_list& paths,
dynamic_X_store& store,
prefetch_lookup& id_lookup,
bool pedantic = false
);
}

59
include/assets/file_parsers/kitti_loader.hpp
View File

@@ -1,59 +0,0 @@
#pragma once
#include <filesystem>
#include <vector>
#include <span>
#include "assets/data_stores.hpp"
#include "assets/components/point_cloud_vertex_components.hpp"
#include "assets/data/point_cloud_data.hpp"
#include "assets/data_stores/point_cloud_store.hpp"
#include "assets/prefetch_lookup.hpp"
#include "assets/prefetch_queue.hpp"
#include "glm/mat4x4.hpp"
#include "util/result.hpp"
namespace assets
{
struct kitti_loader
{
static constexpr auto name = std::string_view("kitti");
[[nodiscard]] static std::error_code prefetch(
const file_dir_list& paths,
prefetch_queue& queue
);
[[nodiscard]] static std::error_code load(
point_cloud_data& buffer,
const file_dir_list& paths,
prefetch_lookup& id_lookup,
shader_source_store& store,
bool pedantic = false
);
private:
inline static constexpr auto frame_folder = "frames";
[[nodiscard]] static ztu::result<std::size_t> frame_id_from_filename(
std::string_view filename
);
[[nodiscard]] static std::error_code load_point_file(
const std::filesystem::path& filename,
dynamic_point_cloud_buffer& point_cloud
);
static void transform_point_cloud(
std::span<point_cloud_vertex_components::position> points,
const glm::mat4& pose
);
[[nodiscard]] static ztu::result<std::string_view> parent_directory(std::string_view path);
};
}

98
include/assets/file_parsers/kitti_parser.hpp Normal file
View File

@@ -0,0 +1,98 @@
#pragma once
#include <filesystem>
#include "assets/data_stores.hpp"
#include "assets/path_id_lookups.hpp"
#include "assets/components/point_cloud_vertex_components.hpp"
#include "assets/data/point_cloud_data.hpp"
#include "assets/data/pose_data.hpp"
#include "assets/data_stores/point_cloud_store.hpp"
#include "assets/data_stores/pose_list_store.hpp"
#include "assets/data_views/pose_list_view.hpp"
#include "util/result.hpp"
namespace assets
{
struct kitti_parser
{
static constexpr auto name = std::string_view("kitti");
using data_type = point_cloud_data;
using store_type = point_cloud_store;
using lookup_type = point_cloud_id_lookup;
[[nodiscard]] std::error_code prefetch(
path_id_lookups& lookups
);
[[nodiscard]] std::error_code load(
path_id_lookups& lookups,
data_stores& stores,
bool pedantic = false
);
protected:
static constexpr auto frame_folder = "frames";
class parser_context
{
public:
parser_context(
const pose_list_id_lookup& pose_list_lookup,
const pose_list_store& pose_list_store,
store_type& m_store,
std::mutex& m_store_mutex
);
void operator()(lookup_type::const_pointer entry) noexcept;
protected:
void reset();
void tokenize_declarations();
[[nodiscard]] bool parse_metadata_from_tokens();
void remove_metadata_declarations();
private:
pose_list_id_lookup const* m_pose_list_lookup;
pose_list_store const* m_pose_list_store;
store_type* m_store;
std::mutex* m_store_mutex;
data_type m_buffer{};
std::filesystem::path m_last_pose_path{};
pose_list_view m_last_pose_list{};
};
[[nodiscard]] static ztu::result<std::size_t> frame_id_from_filename(
std::string_view filename
);
[[nodiscard]] static std::error_code load_point_file(
const std::filesystem::path& filename,
data_type& point_cloud
);
static void transform_point_cloud(
z3d::array_view<point_cloud_vertex_components::position> points,
const pose_data& pose
);
[[nodiscard]] static ztu::result<std::filesystem::path> parent_directory(
const std::filesystem::path& path
);
#
[[nodiscard]] static ztu::result<std::filesystem::path> get_pose_path(
const std::filesystem::path& path
);
private:
std::vector<lookup_type::const_pointer> m_path_buffer;
};
}

37
include/assets/file_parsers/kitti_pose_loader.hpp
View File

@@ -1,37 +0,0 @@
#pragma once
#include <filesystem>
#include "assets/data_stores.hpp"
#include "assets/prefetch_lookup.hpp"
#include "assets/prefetch_queue.hpp"
#include "util/string_list.hpp"
#include "../data_stores"
#include "assets/data"
#include "assets/prefetch_lookups/pose_prefetch_lookup.hpp"
struct kitti_pose_loader
{
static constexpr auto name = std::string_view("kitti_pose");
[[nodiscard]] static std::error_code prefetch(
const file_dir_list& paths,
prefetch_queue& queue
);
[[nodiscard]] static std::error_code load(
dynamic_pose_buffer& buffer,
const file_dir_list& paths,
prefetch_lookup& id_lookup,
shader_source_store& store,
bool pedantic = false
);
private:
static constexpr auto pose_filename = std::string_view{ "pose.txt" };
static std::error_code parse_pose(
std::ifstream& in,
dynamic_pose_buffer& pose
);
};

66
include/assets/file_parsers/kitti_pose_parser.hpp Normal file
View File

@@ -0,0 +1,66 @@
#pragma once
#include <filesystem>
#include "assets/path_id_lookups.hpp"
#include "assets/data_stores/pose_list_store.hpp"
#include "assets/data/pose_list_data.hpp"
namespace assets
{
struct kitti_pose_parser
{
static constexpr auto name = std::string_view("kitti_pose");
using data_type = pose_list_data;
using store_type = pose_list_store;
using lookup_type = pose_list_id_lookup;
[[nodiscard]] std::error_code prefetch(
path_id_lookups& lookups
);
[[nodiscard]] std::error_code parse(
const path_id_lookups& lookups,
store_type& store,
bool pedantic = false
);
private:
static constexpr auto pose_filename = std::string_view{ "pose.txt" };
class parser_context
{
public:
parser_context(
store_type& m_store,
std::mutex& m_store_mutex
);
void operator()(lookup_type::const_pointer entry) noexcept;
protected:
void reset();
private:
store_type* m_store;
std::mutex* m_store_mutex;
data_type m_buffer{};
};
static std::error_code parse_file(
const std::filesystem::path& filename,
data_type &poses
);
static std::error_code parse_pose(
std::ifstream& in,
pose_data& pose
);
private:
std::vector<lookup_type::const_pointer> m_path_buffer;
};
}

14
include/assets/file_parsers/mtl_loader.hpp
View File

@@ -3,15 +3,17 @@
#include <filesystem>
#include <system_error>
#include "assets/data_stores.hpp"
#include "assets/prefetch_lookup.hpp"
#include "assets/prefetch_queue.hpp"
#include "assets/data"
#include "../data_stores"
#include "../data_stores"
#include "util/string_lookup.hpp"
#include "assets/data/material_data.hpp"
#include "assets/data/material_library_data.hpp"
#include "assets/data_stores/material_store.hpp"
#include "assets/data_stores/material_library_store.hpp"
#include "util/result.hpp"
namespace assets
{
namespace mtl_loader_error
{
enum class codes {
@@ -54,3 +56,5 @@ protected:
ztu::string_list& texture_filenames
);
};
}

11
include/assets/file_parsers/threedtk_pose_loader.hpp
View File

@@ -2,14 +2,15 @@
#include <filesystem>
#include "assets/data_stores.hpp"
#include "assets/prefetch_lookup.hpp"
#include "assets/prefetch_queue.hpp"
#include "assets/data"
#include "util/string_list.hpp"
#include "assets/data/pose_data.hpp"
#include "util/result.hpp"
#include "assets/prefetch_lookups/pose_prefetch_lookup.hpp"
namespace assets
{
struct threedtk_pose_loader
{
static constexpr auto name = std::string_view("3dtk_pose");
@@ -20,7 +21,7 @@ struct threedtk_pose_loader
);
[[nodiscard]] static std::error_code load(
dynamic_pose_buffer& buffer,
pose_data& buffer,
const file_dir_list& paths,
prefetch_lookup& id_lookup,
shader_source_store& store,
@@ -38,3 +39,5 @@ protected:
std::string_view filename
);
};
}