zy4n/Z3D
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Made assets::data_stores thread safe.

Browse Source
This commit is contained in:
zy4n
2025-03-31 20:13:26 +02:00
parent 144126ee7a
commit 0acfe36118
39 changed files with 1772 additions and 1069 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
source/assets/file_parsers/glsl_parser.cpp
View File

@@ -72,6 +72,35 @@ namespace assets::language
};
}
std::error_code assets::glsl_parser::prefetch(
path_id_lookups&
) {
// Nothing to prefetch
return {};
}
std::error_code assets::glsl_parser::load(
const path_id_lookups& lookups,
shader_source_store& store,
bool
) {
m_path_buffer.clear();
lookups.shader_sources.by_extension(".glsl", m_path_buffer);
auto store_mutex = std::mutex{};
std::for_each(
std::execution::parallel_unsequenced_policy{},
m_path_buffer.begin(),
m_path_buffer.end(),
parser_context{ store, store_mutex }
);
return {};
}
assets::glsl_parser::parser_context::parser_context(
store_type& m_store,
std::mutex& m_store_mutex
@@ -115,7 +144,7 @@ void assets::glsl_parser::parser_context::operator()(lookup_type::const_pointer
{
auto lock = std::lock_guard{ *m_store_mutex };
m_store->add(id, m_buffer);
m_store->insert(id, m_buffer);
}
}
@@ -322,35 +351,6 @@ std::error_code assets::glsl_parser::read_file(
return {};
}
std::error_code assets::glsl_parser::prefetch(
path_id_lookups&
) {
// Nothing to prefetch
return {};
}
std::error_code assets::glsl_parser::load(
const path_id_lookups& lookups,
shader_source_store& store,
bool
) {
namespace fs = std::filesystem;
m_path_buffer.clear();
lookups.shader_sources.by_extension(".glsl", m_path_buffer);
auto store_mutex = std::mutex{};
std::for_each(
std::execution::parallel_unsequenced_policy{},
m_path_buffer.begin(),
m_path_buffer.end(),
parser_context{ store, store_mutex }
);
return {};
}
bool assets::glsl_parser::parse_geometry_declaration(
const std::span<const std::string_view> values,
model_geometry::types& geometry_type

4
source/assets/file_parsers/kitti_parser.cpp
View File

@@ -28,7 +28,7 @@ assets::kitti_parser::parser_context::parser_context(
m_buffer.normals().reserve(8192);
m_buffer.colors().reserve(8192);
}
5
void assets::kitti_parser::parser_context::reset()
{
m_buffer.clear();
@@ -85,7 +85,7 @@ void assets::kitti_parser::parser_context::operator()(lookup_type::const_pointer
{
auto lock = std::lock_guard{ *m_store_mutex };
m_store->add(id, m_buffer);
m_store->insert(id, m_buffer);
}
}

710
source/assets/file_parsers/mtl_loader.cpp → source/assets/file_parsers/mtl_parser.cpp
View File

@@ -1,4 +1,4 @@
#include "assets/file_parsers/mtl_loader.hpp"
#include "assets/file_parsers/mtl_parser.hpp"
#include <charconv>
#include <fstream>
@@ -7,20 +7,20 @@
#include "util/logger.hpp"
#include "util/for_each.hpp"
#include "util/line_parser.hpp"
#include <execution>
#include "../../../include/assets/data_loaders"
namespace mtl_loader_error
namespace assets::mtl_parser_error
{
struct category : std::error_category
{
[[nodiscard]] const char* name() const noexcept override
{
return "mtl_loader";
return "mtl_parser";
}
[[nodiscard]] std::string message(int ev) const override
{
// TODO these are incomplete
switch (static_cast<codes>(ev))
{
using enum codes;
@@ -47,27 +47,360 @@ struct category : std::error_category
}
};
} // namespace mesh_loader_error
} // namespace mtl_parser_error
inline std::error_category& connector_error_category()
inline std::error_category& mtl_parser_error_category()
{
static mtl_loader_error::category category;
static assets::mtl_parser_error::category category;
return category;
}
namespace mtl_loader_error
namespace assets::mtl_parser_error
{
inline std::error_code make_error_code(codes e)
{
return { static_cast<int>(e), mtl_parser_error_category() };
}
} // namespace mtl_loader_error
std::error_code assets::mtl_parser::prefetch(
path_id_lookups& lookups
) {
namespace fs = std::filesystem;
using mtl_parser_error::codes;
using mtl_parser_error::make_error_code;
auto buffer = std::vector<char>(8 * 1024, '\0');
auto in = std::ifstream{};
auto path_buffer = fs::path{};
auto filename_buffer = fs::path{};
const auto process_file = [&]()
{
in.open(filename_buffer);
if (not in.is_open()) {
ztu::logger::error("Could not open .mtl file '%'", filename_buffer);
return;
}
filename_buffer.remove_filename();
find_textures(buffer, path_buffer, filename_buffer, in, queue.texture.files);
in.close();
};
// TODO properly extract
for (const auto file : paths.files)
{
filename_buffer.assign(file);
process_file();
}
for (const auto directory : paths.directories)
{
for (const auto& file : fs::directory_iterator{ directory }) {
filename_buffer.assign(file.path());
// Avoid heap allocation of .extension()
if (not std::string_view(filename_buffer.c_str()).ends_with(".obj"))
{
continue;
}
process_file();
}
}
return {};
}
std::error_code assets::mtl_parser::load(
path_id_lookups& lookups,
data_stores& stores,
bool pedantic
) {
m_path_buffer.clear();
lookups.material_libraries.by_extension(".mtl", m_path_buffer);
auto store_mutex = std::mutex{};
// TODO who protects the material id lookup
// TODO who protects the material store
std::for_each(
std::execution::parallel_unsequenced_policy{},
m_path_buffer.begin(),
m_path_buffer.end(),
parser_context{
lookups.textures,
lookups.materials,
stores.materials,
stores.material_libraries,
store_mutex
}
);
return {};
}
inline std::error_category& connector_error_category()
{
static assets::mtl_parser_error::category category;
return category;
}
namespace assets::mtl_parser_error
{
inline std::error_code make_error_code(const codes e)
{
return { static_cast<int>(e), connector_error_category() };
}
} // namespace mtl_loader_error
assets::mtl_parser::parser_context::parser_context(
const texture_id_lookup& texture_id_lookup,
material_id_lookup& material_id_lookup,
material_store& material_store,
store_type& store,
std::mutex& store_mutex
) :
m_texture_id_lookup{ &texture_id_lookup },
m_material_id_lookup{ &material_id_lookup },
m_material_store{ &material_store },
m_store{ &store },
m_store_mutex{ &store_mutex }
{
m_buffer.reserve(32);
}
void assets::mtl_parser::parser_context::reset()
{
m_buffer.clear();
}
void assets::mtl_parser::parser_context::operator()(lookup_type::const_pointer entry) noexcept
{
using mtl_parser_error::codes;
using mtl_parser_error::make_error_code;
namespace fs = std::filesystem;
const auto& [ filename, id ] = *entry;
// TODO unroll stuff
auto in = std::ifstream{ filename };
if (not in.is_open())
{
ztu::logger::warn("Cannot open mtl file %.", filename);
return;
}
const auto base_dir = fs::canonical(fs::path(filename).parent_path());
auto name = std::string{};
auto material = material_data{};
const auto push_material = [&]()
{
if (not name.empty())
{
const auto [ id_it, is_new ] = m_material_id_lookup->try_emplace(filename / name);
const auto material_id = id_it->second;
m_material_store->emplace(id, material);
m_buffer.emplace(name, material_id);
}
name = {};
material = {};
};
const auto ec = ztu::parse_lines<codes>(
in,
pedantic,
ztu::make_line_parser("newmtl ", ztu::is_not_repeating, [&](const auto& param)
{
push_material();
name = param;
return codes::ok;
}),
ztu::make_line_parser("Ka ", ztu::is_not_repeating, [&](const auto& param)
{
material_components::ambient_filter ambient_filter;
if (parse_numeric_vector(param, ambient_filter) != std::errc{}) [[unlikely]]
{
return codes::malformed_ambient_color; // TODO rename to filter
}
material.ambient_filter() = ambient_filter;
material.component_flags |= material_components::flags::ambient_filter;
return codes::ok;
}),
ztu::make_line_parser("Kd ", ztu::is_not_repeating, [&](const auto& param)
{
material_components::diffuse_filter diffuse_filter;
if (parse_numeric_vector(param, diffuse_filter) != std::errc{}) [[unlikely]]
{
return codes::malformed_diffuse_color;
}
material.diffuse_filter() = diffuse_filter;
material.component_flags |= material_components::flags::diffuse_filter;
return codes::ok;
}),
ztu::make_line_parser("Ks ", ztu::is_not_repeating, [&](const auto& param)
{
material_components::specular_filter specular_filter;
if (parse_numeric_vector(param, specular_filter) != std::errc{}) [[unlikely]]
{
return codes::malformed_specular_color;
}
material.specular_filter() = specular_filter;
material.component_flags |= material_components::flags::specular_filter;
return codes::ok;
}),
ztu::make_line_parser("Ns ", ztu::is_not_repeating, [&](const auto& param)
{
z3d::vec<1, material_components::shininess> shininess;
if (parse_numeric_vector(param, shininess) != std::errc{}) [[unlikely]]
{
return codes::malformed_specular_exponent;
}
material.shininess() = shininess[0];
material.component_flags |= material_components::flags::shininess;
return codes::ok;
}),
ztu::make_line_parser("d ", ztu::is_not_repeating, [&](const auto& param)
{
z3d::vec<1, material_components::alpha> alpha{};
if (parse_numeric_vector(param, alpha) != std::errc{}) [[unlikely]]
{
return codes::malformed_dissolve;
}
material.alpha() = alpha[0];
material.component_flags |= material_components::flags::alpha;
return codes::ok;
}),
ztu::make_line_parser("map_Ka ", ztu::is_not_repeating, [&](const auto& param)
{
if (const auto texture_id = fetch_texture_id(param, "ambient filter"))
{
material.ambient_filter_texture_id() = *texture_id;
material.component_flags |= material_components::flags::ambient_filter_texture;
}
return codes::ok;
}),
ztu::make_line_parser("map_Kd ", ztu::is_not_repeating, [&](const auto& param)
{
if (const auto texture_id = fetch_texture_id(param, "diffuse filter"))
{
material.diffuse_filter_texture_id() = *texture_id;
material.component_flags |= material_components::flags::diffuse_filter_texture;
}
return codes::ok;
}),
ztu::make_line_parser("map_Ks ", ztu::is_not_repeating, [&](const auto& param)
{
if (const auto texture_id = fetch_texture_id(param, "specular filter"))
{
material.specular_filter_texture_id() = *texture_id;
material.component_flags |= material_components::flags::specular_filter_texture;
}
return codes::ok;
}),
ztu::make_line_parser("map_Ns ", ztu::is_not_repeating, [&](const auto& param)
{
if (const auto texture_id = fetch_texture_id(param, "shininess"))
{
material.shininess_texture_id() = *texture_id;
material.component_flags |= material_components::flags::shininess_texture;
}
return codes::ok;
}),
ztu::make_line_parser("map_d ", ztu::is_not_repeating, [&](const auto& param)
{
if (const auto texture_id = fetch_texture_id(param, "alpha"))
{
material.alpha_texture_id() = *texture_id;
material.component_flags |= material_components::flags::alpha_texture;
}
return codes::ok;
}),
ztu::make_line_parser("bump ", ztu::is_not_repeating, [&](const auto& param)
{
if (const auto texture_id = fetch_texture_id(param, "bump"))
{
material.bump_texture_id() = *texture_id;
material.component_flags |= material_components::flags::bump_texture;
}
return codes::ok;
})
);
if (ec != codes::ok)
{
const auto e = make_error_code(ec);
ztu::logger::error("Error while parsing mtl file %: %", filename, e.message());
}
push_material();
}
std::optional<assets::texture_id> assets::mtl_parser::parser_context::fetch_texture_id(
const std::filesystem::path& mtl_dir,
std::string_view filename,
std::string_view texture_type_name
) {
auto texture_filename = std::filesystem::path(filename);
if (texture_filename.is_relative())
{
texture_filename = mtl_dir / texture_filename;
}
const auto texture_id_it = m_texture_id_lookup->find(texture_filename);
if (texture_id_it == m_texture_id_lookup->end())
{
ztu::logger::warn(
"%-texture at % has not been registered and can therefor not be used.",
texture_type_name,
texture_filename
);
return std::nullopt;
}
return texture_id_it->second;
}
template<typename T, std::size_t Count>
std::errc parse_numeric_vector(std::string_view param, std::array<T, Count>& values)
{
auto it = param.begin(), end = param.end();
const auto end = param.end();
auto it = param.begin();
for (auto& value : values)
{
@@ -90,7 +423,9 @@ std::errc parse_numeric_vector(std::string_view param, std::array<T, Count>& val
};
void mtl_loader::find_textures(
void assets::mtl_parser::find_textures(
std::span<char> buffer,
std::filesystem::path& path_buffer,
const std::filesystem::path& base_directory,
@@ -230,356 +565,3 @@ void mtl_loader::find_textures(
}
} while (not in.eof());
}
std::error_code mtl_loader::prefetch(
const file_dir_list& paths,
prefetch_queue& queue
) {
namespace fs = std::filesystem;
using mtl_loader_error::codes;
using mtl_loader_error::make_error_code;
auto buffer = std::vector<char>(8 * 1024, '\0');
auto in = std::ifstream{};
auto path_buffer = fs::path{};
auto filename_buffer = fs::path{};
const auto process_file = [&]()
{
in.open(filename_buffer);
if (not in.is_open()) {
ztu::logger::error("Could not open .mtl file '%'", filename_buffer);
return;
}
filename_buffer.remove_filename();
find_textures(buffer, path_buffer, filename_buffer, in, queue.texture.files);
in.close();
};
for (const auto file : paths.files)
{
filename_buffer.assign(file);
process_file();
}
for (const auto directory : paths.directories)
{
for (const auto& file : fs::directory_iterator{ directory }) {
filename_buffer.assign(file.path());
// Avoid heap allocation of .extension()
if (not std::string_view(filename_buffer.c_str()).ends_with(".obj"))
{
continue;
}
process_file();
}
}
return {};
}
std::error_code mtl_loader::load_directory(
dynamic_data_loader_ctx& ctx,
dynamic_material_store& store,
material_components::flags enabled_components,
const std::filesystem::path& path,
const bool pedantic
) {
namespace fs = std::filesystem;
if (not fs::exists(path))
{
return make_error_code(std::errc::no_such_file_or_directory);
}
for (const auto& file : fs::directory_iterator{ path })
{
const auto& file_path = file.path();
if (file_path.extension() != ".obj")
{
continue;
}
if (const auto e = load(
ctx,
store,
enabled_components,
path,
pedantic
)) {
ztu::logger::error(
"Error while loading obj file '%': [%] %",
file_path,
e.category().name(),
e.message()
);
}
}
return {};
}
std::error_code mtl_loader::load(
dynamic_data_loader_ctx& ctx,
dynamic_material_store& store,
material_components::flags enabled_components,
const std::filesystem::path& filename,
const bool pedantic
) {
using mtl_loader_error::codes;
using mtl_loader_error::make_error_code;
using flags = material_components::flags;
const auto component_disabled = [&](const material_components::flags component) {
return (enabled_components & component) == flags::none;
};
// TODO unroll stuff
const auto textures_disabled = component_disabled(flags::ambient_filter_texture);
const auto surface_properties_disabled = component_disabled(flags::surface_properties);
const auto transparencies_disabled = component_disabled(flags::transparency);
auto in = std::ifstream{ filename };
if (not in.is_open()) {
return make_error_code(codes::cannot_open_file);
}
namespace fs = std::filesystem;
const auto directory = fs::canonical(fs::path(filename).parent_path());
auto name = std::string{};
auto material = dynamic_material_data{};
const auto push_material = [&]()
{
if (not name.empty())
{
const auto id = store.add(std::move(material));
m_id_lookup.emplace(std::move(name), id);
}
name = std::string{};
material = dynamic_material_data{};
};
const auto load_texture = [&](
const std::string_view path,
std::string_view texture_type_name,
auto&& f
) {
auto texture_filename = fs::path(path);
if (texture_filename.is_relative())
{
texture_filename = directory / texture_filename;
}
const auto extension = texture_filename.extension().string();
auto texture_type = std::string_view{ extension };
if (not texture_type.empty() and texture_type.front() == '.')
{
texture_type = texture_type.substr(1);
}
if (const auto loader_id = ctx.texture_loader.find_loader(texture_type))
{
if (auto res = ctx.texture_loader.read(
ctx,
*loader_id,
texture_filename,
pedantic
)) {
f(*res);
}
else
{
const auto error = res.error();
ztu::logger::warn(
"Error while loading % texture '%': [%] %",
texture_type_name,
path,
error.category().name(),
error.message()
);
}
}
else
{
ztu::logger::warn(
"Failed to load % texture '%' because extension is not supported.",
texture_type_name,
path
);
}
};
const auto ec = ztu::parse_lines<codes>(
in,
pedantic,
ztu::make_line_parser("newmtl ", ztu::is_not_repeating, [&](const auto& param)
{
push_material();
name = param;
return codes::ok;
}),
ztu::make_line_parser("Ka ", ztu::is_not_repeating, [&](const auto& param)
{
material_components::ambient_filter ambient_filter;
if (parse_numeric_vector(param, ambient_filter) != std::errc{}) [[unlikely]]
{
return codes::malformed_ambient_color; // TODO rename to filter
}
material.ambient_filter = mbient_filter;
material.components() |= flags::ambient_filter;
return codes::ok;
}),
ztu::make_line_parser("Kd ", ztu::is_not_repeating, [&](const auto& param)
{
if (surface_properties_disabled) return codes::ok;
auto& properties = material.initialized_surface_properties();
if (parse_numeric_vector(param, properties.diffuse_filter) != std::errc{}) [[unlikely]]
{
return codes::malformed_diffuse_color;
}
material.components() |= flags::surface_properties;
return codes::ok;
}),
ztu::make_line_parser("Ks ", ztu::is_not_repeating, [&](const auto& param)
{
if (surface_properties_disabled) return codes::ok;
auto& properties = material.initialized_surface_properties();
if (parse_numeric_vector(param, properties.specular_filter) != std::errc{}) [[unlikely]]
{
return codes::malformed_specular_color;
}
material.components() |= flags::surface_properties;
return codes::ok;
}),
ztu::make_line_parser("Ns ", ztu::is_not_repeating, [&](const auto& param)
{
if (surface_properties_disabled) return codes::ok;
auto& properties = material.initialized_surface_properties();
std::array<float, 1> shininess{};
if (parse_numeric_vector(param, shininess) != std::errc{}) [[unlikely]]
{
return codes::malformed_specular_exponent;
}
properties.shininess = shininess.front();
material.components() |= flags::surface_properties;
return codes::ok;
}),
ztu::make_line_parser("d ", ztu::is_not_repeating, [&](const auto& param)
{
if (transparencies_disabled) return codes::ok;
std::array<float, 1> transparency{};
if (parse_numeric_vector(param, transparency) != std::errc{}) [[unlikely]]
{
return codes::malformed_dissolve;
}
material.transparency().emplace(transparency.front());
material.components() |= flags::transparency;
return codes::ok;
}),
ztu::make_line_parser("map_Ka ", ztu::is_not_repeating, [&](const auto& param)
{
if (textures_disabled) return codes::ok;
load_texture(param, "ambient color", [&](const auto id) {
material.specular_filter_texture_id() = id;
material.components() |= flags::ambient_filter_texture;
});
return codes::ok;
}),
ztu::make_line_parser("map_Kd ", ztu::is_not_repeating, [&](const auto& param)
{
if (textures_disabled) return codes::ok;
load_texture(param, "diffuse color", [&](const auto id) {
material.diffuse_filter_texture_id() = id;
material.components() |= flags::diffuse_filter_texture;
});
return codes::ok;
}),
ztu::make_line_parser("map_Ks ", ztu::is_not_repeating, [&](const auto& param)
{
if (textures_disabled) return codes::ok;
load_texture(param, "specular color", [&](const auto id) {
material.specular_filter_texture_id() = id;
material.components() |= flags::specular_filter_texture;
});
return codes::ok;
}),
ztu::make_line_parser("map_Ns ", ztu::is_not_repeating, [&](const auto& param)
{
if (textures_disabled) return codes::ok;
load_texture(param, "shininess", [&](const auto id) {
material.shininess_texture_id() = id;
material.components() |= flags::shininess_texture;
});
return codes::ok;
}),
ztu::make_line_parser("map_d ", ztu::is_not_repeating, [&](const auto& param)
{
if (textures_disabled) return codes::ok;
load_texture(param, "alpha", [&](const auto id) {
material.alpha_texture_id() = id;
material.components() |= flags::alpha_texture;
});
return codes::ok;
}),
ztu::make_line_parser("bump ", ztu::is_not_repeating, [&](const auto& param)
{
if (textures_disabled) return codes::ok;
load_texture(param, "bump", [&](const auto id) {
material.bump_texture_id() = id;
material.components() |= flags::bump_texture;
});
return codes::ok;
})
);
if (ec != codes::ok)
{
return make_error_code(ec);
}
push_material();
return {};
}