zy4n/Z3D
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

worked on obj and mtl prefetching nad parsing

Browse Source
This commit is contained in:
ZY4N
2024-12-26 20:20:02 +01:00
parent b385b3b1c8
commit 447146b7f5
16 changed files with 606 additions and 376 deletions
Download Patch File Download Diff File Expand all files Collapse all files

440
source/assets/data_loaders/obj_loader.cpp
View File

@@ -14,30 +14,37 @@
#include "util/line_parser.hpp"
namespace obj_loader_error {
struct category : std::error_category {
namespace obj_loader_error
{
struct category : std::error_category
{
[[nodiscard]] const char* name() const noexcept override {
return "connector";
return "obj_loader";
}
[[nodiscard]] std::string message(int ev) const override {
switch (static_cast<codes>(ev)) {
[[nodiscard]] std::string message(int ev) const override
{
switch (static_cast<codes>(ev))
{
using enum codes;
case obj_cannot_open_file:
case cannot_open_file:
return "Cannot open given obj file.";
case obj_malformed_vertex:
case malformed_vertex:
return "File contains malformed 'v' statement.";
case obj_malformed_texture_coordinate:
case malformed_texture_coordinate:
return "File contains malformed 'vt' statement.";
case obj_malformed_normal:
case malformed_normal:
return "File contains malformed 'vn' statement.";
case obj_malformed_face:
case malformed_face:
return "File contains malformed 'f' statement.";
case obj_face_index_out_of_range:
case face_index_out_of_range:
return "Face index out of range.";
case obj_unknown_line_begin:
case unknown_line_begin:
return "Unknown obj line begin.";
case use_material_without_material_library:
return "'usemtl' statement before material library loaded.";
case unknown_material_name:
return "No matching material name found in material library.";
default:
using namespace std::string_literals;
return "unrecognized error ("s + std::to_string(ev) + ")";
@@ -47,41 +54,25 @@ struct category : std::error_category {
} // namespace mesh_loader_error
inline std::error_category& connector_error_category() {
inline std::error_category& connector_error_category()
{
static obj_loader_error::category category;
return category;
}
namespace obj_loader_error {
inline std::error_code make_error_code(codes e) {
namespace obj_loader_error
{
inline std::error_code make_error_code(codes e)
{
return { static_cast<int>(e), connector_error_category() };
}
} // namespace mesh_loader_error
using vertex_type = std::array<dynamic_mesh_buffer::index_type, 3>;
struct indexed_vertex_type
{
vertex_type vertex;
ztu::u32 buffer_index;
friend auto operator<=>(const indexed_vertex_type& a, const indexed_vertex_type& b) {
return a.vertex <=> b.vertex;
}
bool operator==(const indexed_vertex_type& other) const noexcept {
return other.vertex == vertex;
}
};
// TODO add compile time selection and unrolling
template<typename T, std::size_t Count>
std::errc parse_numeric_vector(std::string_view param, std::array<T, Count>& values) {
auto it = param.begin(), end = param.end();
std::errc parse_numeric_vector(std::string_view param, std::array<T, Count>& values)
{
auto it = param.begin();
const auto end = param.end();
for (auto& value : values)
{
@@ -97,15 +88,17 @@ std::errc parse_numeric_vector(std::string_view param, std::array<T, Count>& val
return ec;
}
it = ptr + 1; // skip space in between components
it = ptr + 1; // Skip space in between components.
}
return {};
};
void find_materials(
void obj_loader::find_materials(
std::span<char> buffer,
std::filesystem::path& path_buffer,
const std::filesystem::path& base_directory,
std::ifstream& in,
ztu::string_list& material_filenames
) {
@@ -113,13 +106,13 @@ void find_materials(
const auto buffer_view = std::string_view(buffer);
// Add linebreak to simplify line begin search
// Add linebreak to simplify line begin search.
buffer.front() = '\n';
auto leftover = std::size_t{ 1 };
do
{
// Keep some old characters to continue matching interrupted sequence
// Keep some old characters to continue matching interrupted sequence.
std::copy(buffer.end() - leftover, buffer.end(), buffer.begin());
in.read(buffer.data() + leftover, buffer.size() - leftover);
@@ -137,23 +130,25 @@ void find_materials(
if (filename_end != std::string_view::npos)
{
const auto length = filename_end - filename_begin;
const auto material_filename = str.substr(filename_begin, length);
const auto filename = str.substr(filename_begin, length);
// TODO get base dir from param figure out how to avoid heap
if (material_filename.is_relative())
path_buffer.assign(filename);
if (path_buffer.is_relative())
{
material_filename = directory / material_filename;
path_buffer = base_directory;
path_buffer /= filename;
}
material_filenames.push_back(material_filename);
material_filenames.push_back(path_buffer.c_str());
pos = filename_end;
}
else // string exceeds the buffer
else // String match exceeds buffer.
{
if (pos == 0) [[unlikely]]
{
std::cout << "Ignoring string match, as it exceeds buffer size." << std::endl;
ztu::logger::error("Ignoring string match, as it exceeds buffer size of % characters.", buffer.size());
leftover = 0;
}
else
@@ -179,145 +174,139 @@ std::error_code obj_loader::prefetch(
auto in = std::ifstream{};
const auto parse_file = [&](const char* filename)
auto path_buffer = fs::path{};
auto filename_buffer = fs::path{};
const auto process_file = [&]()
{
in.open(filename);
in.open(filename_buffer);
if (not in.is_open()) {
ztu::logger::error("Could not open .obj file '%'", filename);
ztu::logger::error("Could not open .obj file '%'", filename_buffer);
return;
}
find_materials(buffer, in, queue.mtl_queue.files);
filename_buffer.remove_filename();
find_materials(buffer, path_buffer, filename_buffer, in, queue.mtl_queue.files);
in.close();
};
for (const auto file : paths.files)
{
// `file` is null-terminates by list.
parse_file(file.data());
filename_buffer.assign(file);
process_file();
}
for (const auto directory : paths.directories)
{
for (const auto& file : fs::directory_iterator{ directory }) {
const auto& file_path = std::string_view{ file.path().c_str() };
filename_buffer.assign(file.path());
// TODO remove heap allocation
if (not file_path.ends_with(".obj"))
// Avoid heap allocation of .extension()
if (not std::string_view(filename_buffer.c_str()).ends_with(".obj"))
{
continue;
}
// Null terminated by fs::path.
parse_file(file_path.data());
process_file();
}
}
return {};
}
std::error_code obj_loader::load_directory(
dynamic_data_loader_ctx& ctx,
dynamic_mesh_store& store,
components::mesh_vertex::flags enabled_components,
const std::filesystem::path& path,
const bool pedantic
) {
namespace fs = std::filesystem;
if (not fs::exists(path))
{
return make_error_code(std::errc::no_such_file_or_directory);
}
for (const auto& file : fs::directory_iterator{ path })
{
const auto& file_path = file.path();
if (file_path.extension() != ".obj")
{
continue;
}
if (const auto e = load(
ctx,
store,
enabled_components,
path,
pedantic
)) {
ztu::logger::error(
"Error while loading obj file '%': [%] %",
file_path,
e.category().name(),
e.message()
);
}
}
return {};
}
[[nodiscard]] static std::error_code load(
components::mesh_vertex::flags enabled_components,
std::error_code obj_loader::load(
dynamic_mesh_buffer& buffer,
const file_dir_list& paths,
prefetch_lookup& id_lookup,
dynamic_data_store& store,
bool pedantic = false
bool pedantic
) {
namespace fs = std::filesystem;
auto position_buffer = buffer.positions();
auto normal_buffer = buffer.normals();
auto tex_coord_buffer = buffer.tex_coords();
auto read_buffer = dynamic_mesh_buffer{};
auto path_buffer = fs::path{};
auto vertex_ids = std::set<indexed_vertex_type>{};
auto in = std::ifstream{};
std::set<indexed_vertex_type> vertex_ids;
const auto parse_file = [&](const char* filename)
auto filename_buffer = fs::path{};
const auto process_file = [&]()
{
in.open(filename);
in.open(filename_buffer);
if (not in.is_open()) {
ztu::logger::error("Could not open .obj file '%'", filename);
ztu::logger::error("Could not open .obj file '%'", filename_buffer);
return;
}
filename_buffer.remove_filename();
vertex_ids.clear();
// parse file
const auto error = parse_file(
read_buffer,
buffer,
path_buffer,
filename_buffer,
vertex_ids,
in,
id_lookup,
store,
pedantic
);
if (error)
{
ztu::logger::error(
"Error occurred while parsing .obj file: [%] %",
error.category().name(),
error.message()
);
}
in.close();
};
for (const auto file : paths.files)
{
// `file` is null-terminates by list.
parse_file(file.data());
filename_buffer.assign(file);
process_file();
}
for (const auto directory : paths.directories)
{
for (const auto& file : fs::directory_iterator{ directory }) {
const auto& file_path = std::string_view{ file.path().c_str() };
filename_buffer.assign(file.path());
// TODO remove heap allocation
if (not file_path.ends_with(".obj"))
// Avoid heap allocation of .extension()
if (not std::string_view(filename_buffer.c_str()).ends_with(".obj"))
{
continue;
}
// Null terminated by fs::path.
parse_file(file_path.data());
process_file();
}
}
return {};
}
// TODO refactor so there is a function like parse_normals etc.
std::error_code obj_loader::load(
components::mesh_vertex::flags enabled_components,
dynamic_mesh_buffer& buffer,
const char* filename,
std::error_code obj_loader::parse_file(
dynamic_mesh_buffer& read_buffer,
dynamic_mesh_buffer& mesh_buffer,
std::filesystem::path& path_buffer,
const std::filesystem::path& base_directory,
std::set<indexed_vertex_type>& vertex_ids,
std::ifstream& in,
prefetch_lookup& id_lookup,
dynamic_data_store& store,
bool pedantic
@@ -325,94 +314,47 @@ std::error_code obj_loader::load(
using obj_loader_error::codes;
using obj_loader_error::make_error_code;
auto in = std::ifstream{ filename };
if (not in.is_open()) {
return make_error_code(codes::obj_cannot_open_file);
}
read_buffer.clear();
mesh_buffer.clear();
vertex_ids.clear();
namespace fs = std::filesystem;
const auto directory = fs::path(filename).parent_path();
// Buffers for storing the vertex component definitions.
auto& position_buffer = read_buffer.positions();
auto& normal_buffer = read_buffer.normals();
auto& tex_coord_buffer = read_buffer.tex_coords();
// Each vertex of a face can represent a unique combination of vertex-/texture-/normal-coordinates.
// But some combinations may occur more than once, for example on every corner of a cube 3 triangles will
// reference the exact same corner vertex.
// To get the best rendering performance and lowest final memory footprint these duplicates
// need to be removed. So this std::set lookup is used to identify the aforementioned duplicates
// and only push unique combinations to the buffers.
std::set<indexed_vertex_type> vertex_ids;
auto& positions = mesh_buffer.positions();
auto& normals = mesh_buffer.normals();
auto& tex_coords = mesh_buffer.tex_coords();
auto& triangles = mesh_buffer.triangles();
// Buffers
// TODO find out if this is still relevant
auto position_buffer = buffer.positions();
auto normal_buffer = buffer.normals();
auto tex_coord_buffer = buffer.tex_coords();
std::unordered_map<std::string, ztu::u32> material_name_lookup;
std::string material_name;
const auto push_mesh = [&](const bool clear_buffers = false) {
if (not buffer.positions().empty())
const auto push_mesh = [&](const bool clear_read_buffer = false)
{
if (not triangles.empty())
{
// Copy buffers instead of moving to keep capacity for further parsing
// and have the final buffers be shrunk to size.
if (not material_name.empty()) {
if (const auto id = material_loader.find_id(material_name))
{
buffer.material_id() = *id;
}
else
{
ztu::logger::warn(
"Could not find material '%'.",
material_name
);
}
}
ztu::logger::debug("Parsed % positions.", positions.size());
ztu::logger::debug("Parsed % normals.", normals.size());
ztu::logger::debug("Parsed % tex_coords.", tex_coords.size());
ztu::logger::debug("Parsed % triangles.", triangles.size());
ztu::logger::debug("Parsed % positions.", mesh.positions().size());
ztu::logger::debug("Parsed % normals.", mesh.normals().size());
ztu::logger::debug("Parsed % tex_coords.", mesh.tex_coords().size());
if (not buffer.positions().empty())
{
buffer.components() |= components::mesh_vertex::flags::position;
}
if (not buffer.normals().empty())
{
buffer.components() |= components::mesh_vertex::flags::normal;
}
if (not buffer.tex_coords().empty())
{
buffer.components() |= components::mesh_vertex::flags::tex_coord;
}
ztu::logger::debug("Pushing obj mesh with % triangles.", buffer.triangles().size());
store.meshes.add(buffer);
// Copy buffer into store and keep capacity.
store.meshes.add(mesh_buffer);
}
if (clear_buffers)
if (clear_read_buffer)
{
position_buffer.clear();
normal_buffer.clear();
tex_coord_buffer.clear();
read_buffer.clear();
}
buffer.clear();
mesh_buffer.clear();
vertex_ids.clear();
material_name.clear();
};
const auto find_or_push_vertex = [&](const vertex_type& vertex) -> ztu::u32 {
const auto find_or_push_vertex = [&](const vertex_type& vertex) -> index_type
{
auto indexed_vid = indexed_vertex_type{
.vertex = vertex,
.buffer_index = static_cast<ztu::u32>(buffer.positions().size())
.buffer_index = static_cast<index_type>(positions.size())
};
// Search through sorted lookup to check if index combination is unique
@@ -422,92 +364,86 @@ std::error_code obj_loader::load(
{
const auto& [ position_index, tex_coord_index, normal_index ] = vertex;
// If index is out of range, push default constructed value.
// Not ideal, but better than out of range indices.
auto& position = positions.emplace_back();
if (position_index < position_buffer.size())
{
buffer.positions().emplace_back(position_buffer[position_index]);
position = position_buffer[position_index];
}
auto& normal = normals.emplace_back();
if (normal_index < normal_buffer.size())
{
buffer.normals().emplace_back(normal_buffer[normal_index]);
normal = normal_buffer[normal_index];
}
auto& tex_coord = tex_coords.emplace_back();
if (tex_coord_index < tex_coord_buffer.size())
{
buffer.tex_coords().emplace_back(tex_coord_buffer[tex_coord_index]);
tex_coord = tex_coord_buffer[tex_coord_index];
}
}
return id_it->buffer_index;
};
using flags = components::mesh_vertex::flags;
const auto component_disabled = [&](const flags component) {
return (enabled_components & component) == flags::none;
};
const auto positions_disabled = component_disabled(flags::position);
const auto normals_disabled = component_disabled(flags::normal);
const auto tex_coords_disabled = component_disabled(flags::tex_coord);
auto curr_material_library_it = dynamic_material_library_store::iterator_type{};
auto has_material_library = false;
const auto ec = ztu::parse_lines<codes>(
in,
pedantic,
ztu::make_line_parser("v ", ztu::is_repeating, [&](const auto& param)
make_line_parser("v ", ztu::is_repeating, [&](const auto& param)
{
if (positions_disabled) return codes::ok;
components::mesh_vertex::position position;
if (parse_numeric_vector(param, position) != std::errc{}) [[unlikely]]
{
return codes::obj_malformed_vertex;
return codes::malformed_vertex;
}
position_buffer.push_back(position);
return codes::ok;
}),
ztu::make_line_parser("vt ", ztu::is_repeating, [&](const auto& param) {
if (tex_coords_disabled) return codes::ok;
make_line_parser("vt ", ztu::is_repeating, [&](const auto& param)
{
components::mesh_vertex::tex_coord coord;
if (parse_numeric_vector(param, coord) != std::errc{}) [[unlikely]]
{
return codes::obj_malformed_texture_coordinate;
return codes::malformed_texture_coordinate;
}
tex_coord_buffer.push_back(coord);
return codes::ok;
}),
ztu::make_line_parser("vn ", ztu::is_repeating, [&](const auto& param)
make_line_parser("vn ", ztu::is_repeating, [&](const auto& param)
{
if (normals_disabled) return codes::ok;
components::mesh_vertex::normal normal;
if (parse_numeric_vector(param, normal) != std::errc{}) [[unlikely]]
{
return codes::obj_malformed_normal;
return codes::malformed_normal;
}
normal_buffer.push_back(normal);
return codes::ok;
}),
ztu::make_line_parser("o ", ztu::is_not_repeating, [&](const auto&)
make_line_parser("o ", ztu::is_not_repeating, [&](const auto&)
{
push_mesh(); // Name is currently ignored
return codes::ok;
}),
ztu::make_line_parser("f ", ztu::is_repeating, [&](const auto& param)
make_line_parser("f ", ztu::is_repeating, [&](const auto& param)
{
const auto begin = param.begin().base();
const auto end = param.end().base();
auto vertex = vertex_type{};
ztu::u32 first_index{}, prev_index{};
index_type first_index{}, prev_index{};
auto vertex_count = std::size_t{};
@@ -525,11 +461,11 @@ std::error_code obj_loader::load(
const auto [ptr, ec] = std::from_chars(it, end, component_index);
if (ec != std::errc()) [[unlikely]]
{
// Discard whole face if one index is malformed
return codes::obj_malformed_face;
// Discard whole face if one index is malformed.
return codes::malformed_face;
}
--component_index; // Indices start at one
--component_index; // Convert to zero based index.
it = ptr;
if (it == end or *it != '/')
@@ -544,14 +480,14 @@ std::error_code obj_loader::load(
if (it != end and *it != ' ') [[unlikely]]
{
return codes::obj_malformed_face;
return codes::malformed_face;
}
const auto curr_index = find_or_push_vertex(vertex);
if (vertex_count >= 3)
{
auto& triangle = mesh.triangles().emplace_back();
auto& triangle = triangles.emplace_back();
triangle[0] = first_index;
triangle[1] = prev_index;
triangle[2] = curr_index;
@@ -566,34 +502,50 @@ std::error_code obj_loader::load(
return codes::ok;
}),
ztu::make_line_parser("usemtl ", ztu::is_not_repeating, [&](const auto& param)
make_line_parser("usemtl ", ztu::is_not_repeating, [&](const auto& param)
{
push_mesh(false);
material_name = param;
if (not has_material_library) [[unlikely]]
{
return codes::use_material_without_material_library;
}
const auto material_id_it = curr_material_library_it->find(param);
if (material_id_it == curr_material_library_it->end()) [[unlikely]]
{
return codes::unknown_material_name;
}
mesh_buffer.material_id() = material_id_it->second;
return codes::ok;
}),
ztu::make_line_parser("mtllib ", ztu::is_not_repeating, [&](const auto& param)
make_line_parser("mtllib ", ztu::is_not_repeating, [&](const auto& param)
{
auto material_filename = fs::path(param);
if (material_filename.is_relative())
path_buffer.assign(param);
if (path_buffer.is_relative())
{
material_filename = directory / material_filename;
path_buffer = base_directory;
path_buffer /= param; // TODO Doesn't thi allocate an extra path?!?
}
if (const auto error = ctx.material_loader.read(
ctx,
mtl_loader_id,
material_filename,
pedantic
)) {
const auto material_library_id_it = id_lookup.material_libraries.find(path_buffer);
if (material_library_id_it != id_lookup.material_libraries.end()) [[likely]]
{
const auto material_library_id = material_library_id_it->second;
std::tie(curr_material_library_it, has_material_library) = store.material_libraries.find(material_library_id);
}
else [[unlikely]]
{
ztu::logger::warn(
"Error occurred while loading mtl files '%': [%] %",
material_filename,
error.category().name(),
error.message()
"Could not find a matching material library with path '%'. Proceeding with default material.",
param
);
has_material_library = false;
}
})
);