zy4n/Z3D
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

worked on obj and mtl prefetching nad parsing

Browse Source
This commit is contained in:
ZY4N
2024-12-26 20:20:02 +01:00
parent b385b3b1c8
commit 447146b7f5
16 changed files with 606 additions and 376 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
include/assets/data_loaders/mtl_loader.hpp
View File

@@ -12,30 +12,25 @@
#include "util/string_lookup.hpp"
#include "util/result.hpp"
namespace mtl_loader_error {
namespace mtl_loader_error
{
enum class codes {
ok = 0,
mtl_cannot_open_file,
mtl_cannot_open_texture,
mtl_malformed_ambient_color,
mtl_malformed_diffuse_color,
mtl_malformed_specular_color,
mtl_malformed_specular_exponent,
mtl_malformed_dissolve,
mlt_unknown_line_begin
cannot_open_file,
cannot_open_texture,
malformed_ambient_color,
malformed_diffuse_color,
malformed_specular_color,
malformed_specular_exponent,
malformed_dissolve,
unknown_line_begin
};
} // namespace mtl_loader_error
class mtl_loader {
public:
struct mtl_loader
{
static constexpr auto name = std::string_view("mtl");
std::optional<dynamic_material_store::id_type> find_id(std::string_view name);
void clear_name_lookup();
[[nodiscard]] static std::error_code prefetch(
const file_dir_list& paths,
prefetch_queue& queue
@@ -50,7 +45,12 @@ public:
bool pedantic = false
);
private:
ztu::string_lookup<dynamic_material_store::id_type> m_id_lookup;
protected:
static void find_textures(
std::span<char> buffer,
std::filesystem::path& path_buffer,
const std::filesystem::path& base_directory,
std::ifstream& in,
ztu::string_list& texture_filenames
);
};

56
include/assets/data_loaders/obj_loader.hpp
View File

@@ -6,18 +6,21 @@
#include "assets/dynamic_data_loaders/dynamic_material_loader.hpp"
#include "assets/dynamic_data_stores/dynamic_mesh_store.hpp"
#include "assets/prefetch_lookup.hpp"
#include <set>
namespace obj_loader_error {
enum class codes {
ok = 0,
obj_cannot_open_file,
obj_malformed_vertex,
obj_malformed_texture_coordinate,
obj_malformed_normal,
obj_malformed_face,
obj_face_index_out_of_range,
obj_unknown_line_begin
cannot_open_file,
malformed_vertex,
malformed_texture_coordinate,
malformed_normal,
malformed_face,
face_index_out_of_range,
unknown_line_begin,
use_material_without_material_library,
unknown_material_name
};
} // namespace obj_loader_error
@@ -32,11 +35,48 @@ struct obj_loader {
);
[[nodiscard]] static std::error_code load(
components::mesh_vertex::flags enabled_components,
dynamic_mesh_buffer& buffer,
const file_dir_list& paths,
prefetch_lookup& id_lookup,
dynamic_data_store& store,
bool pedantic = false
);
protected:
using index_type = dynamic_mesh_buffer::index_type;
using vertex_type = std::array<index_type, 3>;
struct indexed_vertex_type
{
vertex_type vertex;
index_type buffer_index;
friend auto operator<=>(const indexed_vertex_type& a, const indexed_vertex_type& b) {
return a.vertex <=> b.vertex;
}
bool operator==(const indexed_vertex_type& other) const noexcept {
return other.vertex == vertex;
}
};
static void find_materials(
std::span<char> buffer,
std::filesystem::path& path_buffer,
const std::filesystem::path& base_directory,
std::ifstream& in,
ztu::string_list& material_filenames
);
[[nodiscard]] static std::error_code obj_loader::parse_file(
dynamic_mesh_buffer& read_buffer,
dynamic_mesh_buffer& mesh_buffer,
std::filesystem::path& path_buffer,
const std::filesystem::path& base_directory,
std::set<indexed_vertex_type>& vertex_ids,
std::ifstream& in,
prefetch_lookup& id_lookup,
dynamic_data_store& store,
bool pedantic
);
};

34
include/assets/dynamic_read_buffers/dynamic_material_buffer.hpp
View File

@@ -11,23 +11,25 @@ struct dynamic_material_buffer {
components::material::surface_properties& initialized_surface_properties();
[[nodiscard]] std::optional<components::material::surface_properties>& surface_properties();
[[nodiscard]] std::optional<components::material::transparency>& transparency();
[[nodiscard]] std::optional<dynamic_texture_store::id_type>& ambient_color_texture_id();
[[nodiscard]] std::optional<dynamic_texture_store::id_type>& diffuse_color_texture_id();
[[nodiscard]] std::optional<dynamic_texture_store::id_type>& specular_color_texture_id();
[[nodiscard]] std::optional<dynamic_texture_store::id_type>& shininess_texture_id();
[[nodiscard]] std::optional<dynamic_texture_store::id_type>& alpha_texture_id();
[[nodiscard]] std::optional<dynamic_texture_store::id_type>& bump_texture_id();
[[nodiscard]] inline std::optional<components::material::surface_properties>& surface_properties();
[[nodiscard]] inline std::optional<components::material::transparency>& transparency();
[[nodiscard]] inline std::optional<dynamic_texture_store::id_type>& ambient_color_texture_id();
[[nodiscard]] inline std::optional<dynamic_texture_store::id_type>& diffuse_color_texture_id();
[[nodiscard]] inline std::optional<dynamic_texture_store::id_type>& specular_color_texture_id();
[[nodiscard]] inline std::optional<dynamic_texture_store::id_type>& shininess_texture_id();
[[nodiscard]] inline std::optional<dynamic_texture_store::id_type>& alpha_texture_id();
[[nodiscard]] inline std::optional<dynamic_texture_store::id_type>& bump_texture_id();
[[nodiscard]] const std::optional<components::material::surface_properties>& surface_properties() const;
[[nodiscard]] const std::optional<components::material::transparency>& transparency() const;
[[nodiscard]] const std::optional<dynamic_texture_store::id_type>& ambient_color_texture_id() const;
[[nodiscard]] const std::optional<dynamic_texture_store::id_type>& diffuse_color_texture_id() const;
[[nodiscard]] const std::optional<dynamic_texture_store::id_type>& specular_color_texture_id() const;
[[nodiscard]] const std::optional<dynamic_texture_store::id_type>& shininess_texture_id() const;
[[nodiscard]] const std::optional<dynamic_texture_store::id_type>& alpha_texture_id() const;
[[nodiscard]] const std::optional<dynamic_texture_store::id_type>& bump_texture_id() const;
[[nodiscard]] inline const std::optional<components::material::surface_properties>& surface_properties() const;
[[nodiscard]] inline const std::optional<components::material::transparency>& transparency() const;
[[nodiscard]] inline const std::optional<dynamic_texture_store::id_type>& ambient_color_texture_id() const;
[[nodiscard]] inline const std::optional<dynamic_texture_store::id_type>& diffuse_color_texture_id() const;
[[nodiscard]] inline const std::optional<dynamic_texture_store::id_type>& specular_color_texture_id() const;
[[nodiscard]] inline const std::optional<dynamic_texture_store::id_type>& shininess_texture_id() const;
[[nodiscard]] inline const std::optional<dynamic_texture_store::id_type>& alpha_texture_id() const;
[[nodiscard]] inline const std::optional<dynamic_texture_store::id_type>& bump_texture_id() const;
inline void clear();
std::tuple<
std::optional<components::material::surface_properties>,

32
include/assets/dynamic_read_buffers/dynamic_mesh_buffer.hpp
View File

@@ -5,7 +5,7 @@
#include "util/uix.hpp"
#include "assets/components/mesh_vertex_components.hpp"
#include "assets/dynamic_read_buffers/dynamic_vertex_buffer.hpp"
#include "generic/dynamic_vertex_buffer.hpp"
#include "assets/dynamic_data_stores/dynamic_material_store.hpp"
class dynamic_mesh_buffer : public dynamic_vertex_buffer<
@@ -20,21 +20,23 @@ public:
using index_type = ztu::u32;
using triangle_type = std::array<index_type, 3>;
[[nodiscard]] std::vector<components::mesh_vertex::position>& positions();
[[nodiscard]] std::vector<components::mesh_vertex::normal>& normals();
[[nodiscard]] std::vector<components::mesh_vertex::tex_coord>& tex_coords();
[[nodiscard]] std::vector<components::mesh_vertex::color>& colors();
[[nodiscard]] std::vector<components::mesh_vertex::reflectance>& reflectances();
[[nodiscard]] std::vector<triangle_type>& triangles();
[[nodiscard]] auto& material_id();
[[nodiscard]] inline std::vector<components::mesh_vertex::position>& positions();
[[nodiscard]] inline std::vector<components::mesh_vertex::normal>& normals();
[[nodiscard]] inline std::vector<components::mesh_vertex::tex_coord>& tex_coords();
[[nodiscard]] inline std::vector<components::mesh_vertex::color>& colors();
[[nodiscard]] inline std::vector<components::mesh_vertex::reflectance>& reflectances();
[[nodiscard]] inline std::vector<triangle_type>& triangles();
[[nodiscard]] inline auto& material_id();
[[nodiscard]] const std::vector<components::mesh_vertex::position>& positions() const;
[[nodiscard]] const std::vector<components::mesh_vertex::normal>& normals() const;
[[nodiscard]] const std::vector<components::mesh_vertex::tex_coord>& tex_coords() const;
[[nodiscard]] const std::vector<components::mesh_vertex::color>& colors() const;
[[nodiscard]] const std::vector<components::mesh_vertex::reflectance>& reflectances() const;
[[nodiscard]] const std::vector<triangle_type>& triangles() const;
[[nodiscard]] const auto& material_id() const;
[[nodiscard]] inline const std::vector<components::mesh_vertex::position>& positions() const;
[[nodiscard]] inline const std::vector<components::mesh_vertex::normal>& normals() const;
[[nodiscard]] inline const std::vector<components::mesh_vertex::tex_coord>& tex_coords() const;
[[nodiscard]] inline const std::vector<components::mesh_vertex::color>& colors() const;
[[nodiscard]] inline const std::vector<components::mesh_vertex::reflectance>& reflectances() const;
[[nodiscard]] inline const std::vector<triangle_type>& triangles() const;
[[nodiscard]] inline const auto& material_id() const;
inline void clear();
private:
std::vector<triangle_type> m_triangles{};

20
include/assets/dynamic_read_buffers/dynamic_point_cloud_buffer.hpp
View File

@@ -4,7 +4,7 @@
#include <array>
#include <vector>
#include "assets/dynamic_read_buffers/dynamic_vertex_buffer.hpp"
#include "generic/dynamic_vertex_buffer.hpp"
class dynamic_point_cloud_buffer : public dynamic_vertex_buffer<
components::point_cloud_vertex::flags,
@@ -14,15 +14,17 @@ class dynamic_point_cloud_buffer : public dynamic_vertex_buffer<
components::point_cloud_vertex::reflectance
> {
public:
[[nodiscard]] std::vector<components::point_cloud_vertex::position>& positions();
[[nodiscard]] std::vector<components::point_cloud_vertex::normal>& normals();
[[nodiscard]] std::vector<components::point_cloud_vertex::color>& colors();
[[nodiscard]] std::vector<components::point_cloud_vertex::reflectance>& reflectances();
[[nodiscard]] inline std::vector<components::point_cloud_vertex::position>& positions();
[[nodiscard]] inline std::vector<components::point_cloud_vertex::normal>& normals();
[[nodiscard]] inline std::vector<components::point_cloud_vertex::color>& colors();
[[nodiscard]] inline std::vector<components::point_cloud_vertex::reflectance>& reflectances();
[[nodiscard]] const std::vector<components::point_cloud_vertex::position>& positions() const;
[[nodiscard]] const std::vector<components::point_cloud_vertex::normal>& normals() const;
[[nodiscard]] const std::vector<components::point_cloud_vertex::color>& colors() const;
[[nodiscard]] const std::vector<components::point_cloud_vertex::reflectance>& reflectances() const;
[[nodiscard]] inline const std::vector<components::point_cloud_vertex::position>& positions() const;
[[nodiscard]] inline const std::vector<components::point_cloud_vertex::normal>& normals() const;
[[nodiscard]] inline const std::vector<components::point_cloud_vertex::color>& colors() const;
[[nodiscard]] inline const std::vector<components::point_cloud_vertex::reflectance>& reflectances() const;
inline void clear();
};
#define INCLUDE_DYNAMIC_TEXTURE_DATA_IMPLEMENTATION

6
include/assets/dynamic_read_buffers/dynamic_shader_buffer.hpp
View File

@@ -7,4 +7,10 @@ struct dynamic_shader_buffer
{
std::vector<char> source{};
GLenum type{ GL_INVALID_ENUM };
inline void clear()
{
source.clear();
type = GL_INVALID_ENUM;
}
};

44
include/assets/dynamic_read_buffers/dynamic_texture_buffer.hpp
View File

@@ -21,50 +21,52 @@ public:
dynamic_texture_buffer() = default;
dynamic_texture_buffer(
inline dynamic_texture_buffer(
std::unique_ptr<value_type>&& data,
dim_type width,
dim_type height,
components::texture::flags components
);
);;
dynamic_texture_buffer(const dynamic_texture_buffer&);
inline dynamic_texture_buffer(const dynamic_texture_buffer&);
dynamic_texture_buffer(dynamic_texture_buffer&&) noexcept;
inline dynamic_texture_buffer(dynamic_texture_buffer&&) noexcept;
[[nodiscard]] dynamic_texture_buffer& operator=(const dynamic_texture_buffer&);
[[nodiscard]] inline dynamic_texture_buffer& operator=(const dynamic_texture_buffer&);
[[nodiscard]] dynamic_texture_buffer& operator=(dynamic_texture_buffer&&) noexcept;
[[nodiscard]] inline dynamic_texture_buffer& operator=(dynamic_texture_buffer&&) noexcept;
[[nodiscard]] components::texture::flags components() const;
[[nodiscard]] inline components::texture::flags components() const;
[[nodiscard]] dim_type width() const;
[[nodiscard]] inline dim_type width() const;
[[nodiscard]] dim_type height() const;
[[nodiscard]] inline dim_type height() const;
[[nodiscard]] std::pair<dim_type, dim_type> dimensions() const;
[[nodiscard]] inline std::pair<dim_type, dim_type> dimensions() const;
[[nodiscard]] size_type pixel_count() const;
[[nodiscard]] inline size_type pixel_count() const;
[[nodiscard]] size_type component_count() const;
[[nodiscard]] inline size_type component_count() const;
[[nodiscard]] size_type size() const;
[[nodiscard]] inline size_type size() const;
[[nodiscard]] const_iterator begin() const;
[[nodiscard]] inline const_iterator begin() const;
[[nodiscard]] iterator begin();
[[nodiscard]] inline iterator begin();
[[nodiscard]] const_iterator end() const;
[[nodiscard]] inline const_iterator end() const;
[[nodiscard]] iterator end();
[[nodiscard]] inline iterator end();
[[nodiscard]] const_iterator cbegin() const;
[[nodiscard]] inline const_iterator cbegin() const;
[[nodiscard]] const_iterator cend() const;
[[nodiscard]] inline const_iterator cend() const;
[[nodiscard]] const_pointer data() const;
[[nodiscard]] inline const_pointer data() const;
[[nodiscard]] pointer data();
[[nodiscard]] inline pointer data();
void inline clear();
private:
std::unique_ptr<value_type[]> m_data{ nullptr };

15
include/assets/dynamic_read_buffers/dynamic_vertex_buffer.hpp → include/assets/dynamic_read_buffers/generic/dynamic_vertex_buffer.hpp
View File

@@ -8,8 +8,15 @@
template<typename C, typename... Ts>
struct dynamic_vertex_buffer {
std::tuple<std::vector<Ts>...> vertices{};
};
#define INCLUDE_DYNAMIC_MODEL_DATA_IMPLEMENTATION
#include "assets/dynamic_read_buffers/dynamic_texture_buffer.ipp"
#undef INCLUDE_DYNAMIC_MODEL_DATA_IMPLEMENTATION
protected:
inline void clear_vertices()
{
std::apply(
[](auto&... vertex_opt) {
(vertex_opt.clear(), ...);
},
vertices
);
}
};

2
source/assets/data_loaders/kitti_loader.cpp
View File

@@ -208,7 +208,7 @@ std::error_code kitti_loader::load(
}
for (const auto [ filename, pose_it ] : std::ranges::views::zip_view(processed_filenames, pose_its))
for (const auto [ filename, pose_it ] : std::ranges::views::zip(processed_filenames, pose_its))
{
buffer.clear();

258
source/assets/data_loaders/mtl_loader.cpp
View File

@@ -10,31 +10,35 @@
#include "assets/dynamic_data_loaders/dynamic_texture_loader.hpp"
namespace mtl_loader_error {
struct category : std::error_category {
[[nodiscard]] const char* name() const noexcept override {
return "connector";
namespace mtl_loader_error
{
struct category : std::error_category
{
[[nodiscard]] const char* name() const noexcept override
{
return "mtl_loader";
}
[[nodiscard]] std::string message(int ev) const override {
switch (static_cast<codes>(ev)) {
[[nodiscard]] std::string message(int ev) const override
{
switch (static_cast<codes>(ev))
{
using enum codes;
case mtl_cannot_open_file:
case cannot_open_file:
return "Cannot open mtl file.";
case mtl_cannot_open_texture:
case cannot_open_texture:
return "Cannot open texture file.";
case mtl_malformed_ambient_color:
case malformed_ambient_color:
return "File contains malformed 'Ka' statement.";
case mtl_malformed_diffuse_color:
case malformed_diffuse_color:
return "File contains malformed 'Kd' statement.";
case mtl_malformed_specular_color:
case malformed_specular_color:
return "File contains malformed 'Ks' statement.";
case mtl_malformed_specular_exponent:
case malformed_specular_exponent:
return "File contains malformed 'Ns' statement.";
case mtl_malformed_dissolve:
case malformed_dissolve:
return "File contains malformed 'd' statement.";
case mlt_unknown_line_begin:
case unknown_line_begin:
return "Unknown mtl line begin";
default:
using namespace std::string_literals;
@@ -45,22 +49,24 @@ struct category : std::error_category {
} // namespace mesh_loader_error
inline std::error_category& connector_error_category() {
inline std::error_category& connector_error_category()
{
static mtl_loader_error::category category;
return category;
}
namespace mtl_loader_error {
inline std::error_code make_error_code(codes e) {
namespace mtl_loader_error
{
inline std::error_code make_error_code(codes e)
{
return { static_cast<int>(e), connector_error_category() };
}
} // namespace mtl_loader_error
template<typename T, std::size_t Count>
std::errc parse_numeric_vector(std::string_view param, std::array<T, Count>& values) {
std::errc parse_numeric_vector(std::string_view param, std::array<T, Count>& values)
{
auto it = param.begin(), end = param.end();
for (auto& value : values)
@@ -83,20 +89,202 @@ std::errc parse_numeric_vector(std::string_view param, std::array<T, Count>& val
return {};
};
std::optional<dynamic_material_store::id_type> mtl_loader::find_id(std::string_view name)
{
const auto it = m_id_lookup.find(name);
if (it == m_id_lookup.end())
void mtl_loader::find_textures(
std::span<char> buffer,
std::filesystem::path& path_buffer,
const std::filesystem::path& base_directory,
std::ifstream& in,
ztu::string_list& texture_filenames
) {
using namespace std::string_view_literals;
// TODO 'bump' is missing!!!
static constexpr auto keyword = "\nmap_"sv;
using long_postfix_type = std::array<char, 2>;
static constexpr auto postfix_length = std::tuple_size_v<long_postfix_type>;
static constexpr auto make_postfix = [](const std::string_view str) static constexpr
{
return it->second;
auto postfix = long_postfix_type{};
assert(str.length() >= postfix_length);
std::copy_n(str.begin(), postfix.size(), postfix.begin());
return postfix;
};
static constexpr auto postfixes = std::array{
make_postfix("d "),
make_postfix("Ka"),
make_postfix("Kd"),
make_postfix("Ks"),
make_postfix("Ns")
};
const auto buffer_view = std::string_view(buffer);
// Add linebreak to simplify line begin search.
buffer.front() = '\n';
auto leftover = std::size_t{ 1 };
enum class match {
exact,
overflowed,
none
};
const auto check_match = [](std::string_view& potential_match) static -> match
{
std::cout << '\'' << potential_match.substr(0, std::min(40ul, potential_match.size())) << '\'' << std::endl;
if (potential_match.length() < postfix_length)
{
return match::overflowed;
}
return std::nullopt;
const auto postfix = make_postfix(potential_match);
// Optimized for SIMD.
if (not std::ranges::contains(postfixes, postfix))
{
return match::none;
}
void mtl_loader::clear_name_lookup() {
m_id_lookup.clear();
const auto long_match = postfix.back() != ' ';
if (long_match and (
potential_match.length() < postfix_length + sizeof(' ') or
potential_match[postfix_length] != ' '
)) {
return match::overflowed;
}
const auto actual_postfix_length = std::size_t{ 1 } + static_cast<std::size_t>(long_match);
const auto filename_begin = actual_postfix_length + sizeof(' ');
const auto filename_end = potential_match.find('\n', filename_begin);
if (filename_end == std::string_view::npos)
{
return match::overflowed;
}
const auto length = filename_end - filename_begin;
potential_match = potential_match.substr(filename_begin, length);
return match::exact;
};
do
{
// Keep some old characters to continue matching interrupted sequence.
std::copy(buffer.end() - leftover, buffer.end(), buffer.begin());
in.read(buffer.data() + leftover, buffer.size() - leftover);
const auto str = buffer_view.substr(0, leftover + in.gcount());
auto pos = std::string_view::size_type{};
while ((pos = str.find(keyword, pos)) != std::string_view::npos)
{
const auto keyword_end = pos + keyword.size();
auto potential_match = str.substr(keyword_end);
const auto match_type = check_match(potential_match);
if (match_type == match::exact)
{
path_buffer.assign(potential_match);
if (path_buffer.is_relative())
{
path_buffer = base_directory;
path_buffer /= potential_match;
}
texture_filenames.push_back(path_buffer.c_str());
pos += potential_match.size();
leftover = 0;
}
else if (match_type == match::overflowed)
{
if (pos == 0) [[unlikely]]
{
ztu::logger::error("Ignoring string match, as it exceeds buffer size of % characters.", buffer.size());
leftover = 0;
}
else
{
leftover = str.size() - pos;
}
break;
}
else
{
leftover = keyword.size();
}
}
} while (not in.eof());
}
std::error_code mtl_loader::prefetch(
const file_dir_list& paths,
prefetch_queue& queue
) {
namespace fs = std::filesystem;
using mtl_loader_error::codes;
using mtl_loader_error::make_error_code;
auto buffer = std::vector<char>(8 * 1024, '\0');
auto in = std::ifstream{};
auto path_buffer = fs::path{};
auto filename_buffer = fs::path{};
const auto process_file = [&]()
{
in.open(filename_buffer);
if (not in.is_open()) {
ztu::logger::error("Could not open .mtl file '%'", filename_buffer);
return;
}
filename_buffer.remove_filename();
find_textures(buffer, path_buffer, filename_buffer, in, queue.texture.files);
in.close();
};
for (const auto file : paths.files)
{
filename_buffer.assign(file);
process_file();
}
for (const auto directory : paths.directories)
{
for (const auto& file : fs::directory_iterator{ directory }) {
filename_buffer.assign(file.path());
// Avoid heap allocation of .extension()
if (not std::string_view(filename_buffer.c_str()).ends_with(".obj"))
{
continue;
}
process_file();
}
}
return {};
}
std::error_code mtl_loader::load_directory(
@@ -164,7 +352,7 @@ std::error_code mtl_loader::load(
auto in = std::ifstream{ filename };
if (not in.is_open()) {
return make_error_code(codes::mtl_cannot_open_file);
return make_error_code(codes::cannot_open_file);
}
namespace fs = std::filesystem;
@@ -252,7 +440,7 @@ std::error_code mtl_loader::load(
auto& properties = material.initialized_surface_properties();
if (parse_numeric_vector(param, properties.ambient_filter) != std::errc{}) [[unlikely]]
{
return codes::mtl_malformed_ambient_color;
return codes::malformed_ambient_color;
}
material.components() |= flags::surface_properties;
@@ -266,7 +454,7 @@ std::error_code mtl_loader::load(
auto& properties = material.initialized_surface_properties();
if (parse_numeric_vector(param, properties.diffuse_filter) != std::errc{}) [[unlikely]]
{
return codes::mtl_malformed_diffuse_color;
return codes::malformed_diffuse_color;
}
material.components() |= flags::surface_properties;
@@ -280,7 +468,7 @@ std::error_code mtl_loader::load(
auto& properties = material.initialized_surface_properties();
if (parse_numeric_vector(param, properties.specular_filter) != std::errc{}) [[unlikely]]
{
return codes::mtl_malformed_specular_color;
return codes::malformed_specular_color;
}
material.components() |= flags::surface_properties;
@@ -295,7 +483,7 @@ std::error_code mtl_loader::load(
std::array<float, 1> shininess{};
if (parse_numeric_vector(param, shininess) != std::errc{}) [[unlikely]]
{
return codes::mtl_malformed_specular_exponent;
return codes::malformed_specular_exponent;
}
properties.shininess = shininess.front();
@@ -310,7 +498,7 @@ std::error_code mtl_loader::load(
std::array<float, 1> transparency{};
if (parse_numeric_vector(param, transparency) != std::errc{}) [[unlikely]]
{
return codes::mtl_malformed_dissolve;
return codes::malformed_dissolve;
}
material.transparency().emplace(transparency.front());

460
source/assets/data_loaders/obj_loader.cpp
View File

@@ -14,30 +14,37 @@
#include "util/line_parser.hpp"
namespace obj_loader_error {
struct category : std::error_category {
namespace obj_loader_error
{
struct category : std::error_category
{
[[nodiscard]] const char* name() const noexcept override {
return "connector";
return "obj_loader";
}
[[nodiscard]] std::string message(int ev) const override {
switch (static_cast<codes>(ev)) {
[[nodiscard]] std::string message(int ev) const override
{
switch (static_cast<codes>(ev))
{
using enum codes;
case obj_cannot_open_file:
case cannot_open_file:
return "Cannot open given obj file.";
case obj_malformed_vertex:
case malformed_vertex:
return "File contains malformed 'v' statement.";
case obj_malformed_texture_coordinate:
case malformed_texture_coordinate:
return "File contains malformed 'vt' statement.";
case obj_malformed_normal:
case malformed_normal:
return "File contains malformed 'vn' statement.";
case obj_malformed_face:
case malformed_face:
return "File contains malformed 'f' statement.";
case obj_face_index_out_of_range:
case face_index_out_of_range:
return "Face index out of range.";
case obj_unknown_line_begin:
case unknown_line_begin:
return "Unknown obj line begin.";
case use_material_without_material_library:
return "'usemtl' statement before material library loaded.";
case unknown_material_name:
return "No matching material name found in material library.";
default:
using namespace std::string_literals;
return "unrecognized error ("s + std::to_string(ev) + ")";
@@ -47,41 +54,25 @@ struct category : std::error_category {
} // namespace mesh_loader_error
inline std::error_category& connector_error_category() {
inline std::error_category& connector_error_category()
{
static obj_loader_error::category category;
return category;
}
namespace obj_loader_error {
inline std::error_code make_error_code(codes e) {
namespace obj_loader_error
{
inline std::error_code make_error_code(codes e)
{
return { static_cast<int>(e), connector_error_category() };
}
} // namespace mesh_loader_error
using vertex_type = std::array<dynamic_mesh_buffer::index_type, 3>;
struct indexed_vertex_type
{
vertex_type vertex;
ztu::u32 buffer_index;
friend auto operator<=>(const indexed_vertex_type& a, const indexed_vertex_type& b) {
return a.vertex <=> b.vertex;
}
bool operator==(const indexed_vertex_type& other) const noexcept {
return other.vertex == vertex;
}
};
// TODO add compile time selection and unrolling
template<typename T, std::size_t Count>
std::errc parse_numeric_vector(std::string_view param, std::array<T, Count>& values) {
auto it = param.begin(), end = param.end();
std::errc parse_numeric_vector(std::string_view param, std::array<T, Count>& values)
{
auto it = param.begin();
const auto end = param.end();
for (auto& value : values)
{
@@ -97,15 +88,17 @@ std::errc parse_numeric_vector(std::string_view param, std::array<T, Count>& val
return ec;
}
it = ptr + 1; // skip space in between components
it = ptr + 1; // Skip space in between components.
}
return {};
};
void find_materials(
void obj_loader::find_materials(
std::span<char> buffer,
std::filesystem::path& path_buffer,
const std::filesystem::path& base_directory,
std::ifstream& in,
ztu::string_list& material_filenames
) {
@@ -113,13 +106,13 @@ void find_materials(
const auto buffer_view = std::string_view(buffer);
// Add linebreak to simplify line begin search
// Add linebreak to simplify line begin search.
buffer.front() = '\n';
auto leftover = std::size_t{ 1 };
do
{
// Keep some old characters to continue matching interrupted sequence
// Keep some old characters to continue matching interrupted sequence.
std::copy(buffer.end() - leftover, buffer.end(), buffer.begin());
in.read(buffer.data() + leftover, buffer.size() - leftover);
@@ -137,23 +130,25 @@ void find_materials(
if (filename_end != std::string_view::npos)
{
const auto length = filename_end - filename_begin;
const auto material_filename = str.substr(filename_begin, length);
const auto filename = str.substr(filename_begin, length);
// TODO get base dir from param figure out how to avoid heap
if (material_filename.is_relative())
path_buffer.assign(filename);
if (path_buffer.is_relative())
{
material_filename = directory / material_filename;
path_buffer = base_directory;
path_buffer /= filename;
}
material_filenames.push_back(material_filename);
material_filenames.push_back(path_buffer.c_str());
pos = filename_end;
}
else // string exceeds the buffer
else // String match exceeds buffer.
{
if (pos == 0) [[unlikely]]
{
std::cout << "Ignoring string match, as it exceeds buffer size." << std::endl;
ztu::logger::error("Ignoring string match, as it exceeds buffer size of % characters.", buffer.size());
leftover = 0;
}
else
@@ -179,145 +174,139 @@ std::error_code obj_loader::prefetch(
auto in = std::ifstream{};
const auto parse_file = [&](const char* filename)
auto path_buffer = fs::path{};
auto filename_buffer = fs::path{};
const auto process_file = [&]()
{
in.open(filename);
in.open(filename_buffer);
if (not in.is_open()) {
ztu::logger::error("Could not open .obj file '%'", filename);
ztu::logger::error("Could not open .obj file '%'", filename_buffer);
return;
}
find_materials(buffer, in, queue.mtl_queue.files);
filename_buffer.remove_filename();
find_materials(buffer, path_buffer, filename_buffer, in, queue.mtl_queue.files);
in.close();
};
for (const auto file : paths.files)
{
// `file` is null-terminates by list.
parse_file(file.data());
filename_buffer.assign(file);
process_file();
}
for (const auto directory : paths.directories)
{
for (const auto& file : fs::directory_iterator{ directory }) {
const auto& file_path = std::string_view{ file.path().c_str() };
filename_buffer.assign(file.path());
// TODO remove heap allocation
if (not file_path.ends_with(".obj"))
// Avoid heap allocation of .extension()
if (not std::string_view(filename_buffer.c_str()).ends_with(".obj"))
{
continue;
}
// Null terminated by fs::path.
parse_file(file_path.data());
process_file();
}
}
return {};
}
std::error_code obj_loader::load_directory(
dynamic_data_loader_ctx& ctx,
dynamic_mesh_store& store,
components::mesh_vertex::flags enabled_components,
const std::filesystem::path& path,
const bool pedantic
) {
namespace fs = std::filesystem;
if (not fs::exists(path))
{
return make_error_code(std::errc::no_such_file_or_directory);
}
for (const auto& file : fs::directory_iterator{ path })
{
const auto& file_path = file.path();
if (file_path.extension() != ".obj")
{
continue;
}
if (const auto e = load(
ctx,
store,
enabled_components,
path,
pedantic
)) {
ztu::logger::error(
"Error while loading obj file '%': [%] %",
file_path,
e.category().name(),
e.message()
);
}
}
return {};
}
[[nodiscard]] static std::error_code load(
components::mesh_vertex::flags enabled_components,
std::error_code obj_loader::load(
dynamic_mesh_buffer& buffer,
const file_dir_list& paths,
prefetch_lookup& id_lookup,
dynamic_data_store& store,
bool pedantic = false
bool pedantic
) {
namespace fs = std::filesystem;
auto position_buffer = buffer.positions();
auto normal_buffer = buffer.normals();
auto tex_coord_buffer = buffer.tex_coords();
auto read_buffer = dynamic_mesh_buffer{};
auto path_buffer = fs::path{};
auto vertex_ids = std::set<indexed_vertex_type>{};
auto in = std::ifstream{};
std::set<indexed_vertex_type> vertex_ids;
const auto parse_file = [&](const char* filename)
auto filename_buffer = fs::path{};
const auto process_file = [&]()
{
in.open(filename);
in.open(filename_buffer);
if (not in.is_open()) {
ztu::logger::error("Could not open .obj file '%'", filename);
ztu::logger::error("Could not open .obj file '%'", filename_buffer);
return;
}
filename_buffer.remove_filename();
vertex_ids.clear();
// parse file
const auto error = parse_file(
read_buffer,
buffer,
path_buffer,
filename_buffer,
vertex_ids,
in,
id_lookup,
store,
pedantic
);
if (error)
{
ztu::logger::error(
"Error occurred while parsing .obj file: [%] %",
error.category().name(),
error.message()
);
}
in.close();
};
for (const auto file : paths.files)
{
// `file` is null-terminates by list.
parse_file(file.data());
filename_buffer.assign(file);
process_file();
}
for (const auto directory : paths.directories)
{
for (const auto& file : fs::directory_iterator{ directory }) {
const auto& file_path = std::string_view{ file.path().c_str() };
filename_buffer.assign(file.path());
// TODO remove heap allocation
if (not file_path.ends_with(".obj"))
// Avoid heap allocation of .extension()
if (not std::string_view(filename_buffer.c_str()).ends_with(".obj"))
{
continue;
}
// Null terminated by fs::path.
parse_file(file_path.data());
process_file();
}
}
return {};
}
// TODO refactor so there is a function like parse_normals etc.
std::error_code obj_loader::load(
components::mesh_vertex::flags enabled_components,
dynamic_mesh_buffer& buffer,
const char* filename,
std::error_code obj_loader::parse_file(
dynamic_mesh_buffer& read_buffer,
dynamic_mesh_buffer& mesh_buffer,
std::filesystem::path& path_buffer,
const std::filesystem::path& base_directory,
std::set<indexed_vertex_type>& vertex_ids,
std::ifstream& in,
prefetch_lookup& id_lookup,
dynamic_data_store& store,
bool pedantic
@@ -325,94 +314,47 @@ std::error_code obj_loader::load(
using obj_loader_error::codes;
using obj_loader_error::make_error_code;
auto in = std::ifstream{ filename };
if (not in.is_open()) {
return make_error_code(codes::obj_cannot_open_file);
}
namespace fs = std::filesystem;
const auto directory = fs::path(filename).parent_path();
// Each vertex of a face can represent a unique combination of vertex-/texture-/normal-coordinates.
// But some combinations may occur more than once, for example on every corner of a cube 3 triangles will
// reference the exact same corner vertex.
// To get the best rendering performance and lowest final memory footprint these duplicates
// need to be removed. So this std::set lookup is used to identify the aforementioned duplicates
// and only push unique combinations to the buffers.
std::set<indexed_vertex_type> vertex_ids;
// Buffers
// TODO find out if this is still relevant
auto position_buffer = buffer.positions();
auto normal_buffer = buffer.normals();
auto tex_coord_buffer = buffer.tex_coords();
std::unordered_map<std::string, ztu::u32> material_name_lookup;
std::string material_name;
const auto push_mesh = [&](const bool clear_buffers = false) {
if (not buffer.positions().empty())
{
// Copy buffers instead of moving to keep capacity for further parsing
// and have the final buffers be shrunk to size.
if (not material_name.empty()) {
if (const auto id = material_loader.find_id(material_name))
{
buffer.material_id() = *id;
}
else
{
ztu::logger::warn(
"Could not find material '%'.",
material_name
);
}
}
ztu::logger::debug("Parsed % positions.", mesh.positions().size());
ztu::logger::debug("Parsed % normals.", mesh.normals().size());
ztu::logger::debug("Parsed % tex_coords.", mesh.tex_coords().size());
if (not buffer.positions().empty())
{
buffer.components() |= components::mesh_vertex::flags::position;
}
if (not buffer.normals().empty())
{
buffer.components() |= components::mesh_vertex::flags::normal;
}
if (not buffer.tex_coords().empty())
{
buffer.components() |= components::mesh_vertex::flags::tex_coord;
}
ztu::logger::debug("Pushing obj mesh with % triangles.", buffer.triangles().size());
store.meshes.add(buffer);
}
if (clear_buffers)
{
position_buffer.clear();
normal_buffer.clear();
tex_coord_buffer.clear();
}
buffer.clear();
read_buffer.clear();
mesh_buffer.clear();
vertex_ids.clear();
// Buffers for storing the vertex component definitions.
auto& position_buffer = read_buffer.positions();
auto& normal_buffer = read_buffer.normals();
auto& tex_coord_buffer = read_buffer.tex_coords();
auto& positions = mesh_buffer.positions();
auto& normals = mesh_buffer.normals();
auto& tex_coords = mesh_buffer.tex_coords();
auto& triangles = mesh_buffer.triangles();
const auto push_mesh = [&](const bool clear_read_buffer = false)
{
if (not triangles.empty())
{
ztu::logger::debug("Parsed % positions.", positions.size());
ztu::logger::debug("Parsed % normals.", normals.size());
ztu::logger::debug("Parsed % tex_coords.", tex_coords.size());
ztu::logger::debug("Parsed % triangles.", triangles.size());
// Copy buffer into store and keep capacity.
store.meshes.add(mesh_buffer);
}
if (clear_read_buffer)
{
read_buffer.clear();
}
mesh_buffer.clear();
vertex_ids.clear();
material_name.clear();
};
const auto find_or_push_vertex = [&](const vertex_type& vertex) -> ztu::u32 {
const auto find_or_push_vertex = [&](const vertex_type& vertex) -> index_type
{
auto indexed_vid = indexed_vertex_type{
.vertex = vertex,
.buffer_index = static_cast<ztu::u32>(buffer.positions().size())
.buffer_index = static_cast<index_type>(positions.size())
};
// Search through sorted lookup to check if index combination is unique
@@ -422,92 +364,86 @@ std::error_code obj_loader::load(
{
const auto& [ position_index, tex_coord_index, normal_index ] = vertex;
// If index is out of range, push default constructed value.
// Not ideal, but better than out of range indices.
auto& position = positions.emplace_back();
if (position_index < position_buffer.size())
{
buffer.positions().emplace_back(position_buffer[position_index]);
position = position_buffer[position_index];
}
auto& normal = normals.emplace_back();
if (normal_index < normal_buffer.size())
{
buffer.normals().emplace_back(normal_buffer[normal_index]);
normal = normal_buffer[normal_index];
}
auto& tex_coord = tex_coords.emplace_back();
if (tex_coord_index < tex_coord_buffer.size())
{
buffer.tex_coords().emplace_back(tex_coord_buffer[tex_coord_index]);
tex_coord = tex_coord_buffer[tex_coord_index];
}
}
return id_it->buffer_index;
};
using flags = components::mesh_vertex::flags;
const auto component_disabled = [&](const flags component) {
return (enabled_components & component) == flags::none;
};
const auto positions_disabled = component_disabled(flags::position);
const auto normals_disabled = component_disabled(flags::normal);
const auto tex_coords_disabled = component_disabled(flags::tex_coord);
auto curr_material_library_it = dynamic_material_library_store::iterator_type{};
auto has_material_library = false;
const auto ec = ztu::parse_lines<codes>(
in,
pedantic,
ztu::make_line_parser("v ", ztu::is_repeating, [&](const auto& param)
make_line_parser("v ", ztu::is_repeating, [&](const auto& param)
{
if (positions_disabled) return codes::ok;
components::mesh_vertex::position position;
if (parse_numeric_vector(param, position) != std::errc{}) [[unlikely]]
{
return codes::obj_malformed_vertex;
return codes::malformed_vertex;
}
position_buffer.push_back(position);
return codes::ok;
}),
ztu::make_line_parser("vt ", ztu::is_repeating, [&](const auto& param) {
if (tex_coords_disabled) return codes::ok;
make_line_parser("vt ", ztu::is_repeating, [&](const auto& param)
{
components::mesh_vertex::tex_coord coord;
if (parse_numeric_vector(param, coord) != std::errc{}) [[unlikely]]
{
return codes::obj_malformed_texture_coordinate;
return codes::malformed_texture_coordinate;
}
tex_coord_buffer.push_back(coord);
return codes::ok;
}),
ztu::make_line_parser("vn ", ztu::is_repeating, [&](const auto& param)
make_line_parser("vn ", ztu::is_repeating, [&](const auto& param)
{
if (normals_disabled) return codes::ok;
components::mesh_vertex::normal normal;
if (parse_numeric_vector(param, normal) != std::errc{}) [[unlikely]]
{
return codes::obj_malformed_normal;
return codes::malformed_normal;
}
normal_buffer.push_back(normal);
return codes::ok;
}),
ztu::make_line_parser("o ", ztu::is_not_repeating, [&](const auto&)
make_line_parser("o ", ztu::is_not_repeating, [&](const auto&)
{
push_mesh(); // Name is currently ignored
return codes::ok;
}),
ztu::make_line_parser("f ", ztu::is_repeating, [&](const auto& param)
make_line_parser("f ", ztu::is_repeating, [&](const auto& param)
{
const auto begin = param.begin().base();
const auto end = param.end().base();
auto vertex = vertex_type{};
ztu::u32 first_index{}, prev_index{};
index_type first_index{}, prev_index{};
auto vertex_count = std::size_t{};
@@ -525,11 +461,11 @@ std::error_code obj_loader::load(
const auto [ptr, ec] = std::from_chars(it, end, component_index);
if (ec != std::errc()) [[unlikely]]
{
// Discard whole face if one index is malformed
return codes::obj_malformed_face;
// Discard whole face if one index is malformed.
return codes::malformed_face;
}
--component_index; // Indices start at one
--component_index; // Convert to zero based index.
it = ptr;
if (it == end or *it != '/')
@@ -544,14 +480,14 @@ std::error_code obj_loader::load(
if (it != end and *it != ' ') [[unlikely]]
{
return codes::obj_malformed_face;
return codes::malformed_face;
}
const auto curr_index = find_or_push_vertex(vertex);
if (vertex_count >= 3)
{
auto& triangle = mesh.triangles().emplace_back();
auto& triangle = triangles.emplace_back();
triangle[0] = first_index;
triangle[1] = prev_index;
triangle[2] = curr_index;
@@ -566,34 +502,50 @@ std::error_code obj_loader::load(
return codes::ok;
}),
ztu::make_line_parser("usemtl ", ztu::is_not_repeating, [&](const auto& param)
make_line_parser("usemtl ", ztu::is_not_repeating, [&](const auto& param)
{
push_mesh(false);
material_name = param;
if (not has_material_library) [[unlikely]]
{
return codes::use_material_without_material_library;
}
const auto material_id_it = curr_material_library_it->find(param);
if (material_id_it == curr_material_library_it->end()) [[unlikely]]
{
return codes::unknown_material_name;
}
mesh_buffer.material_id() = material_id_it->second;
return codes::ok;
}),
ztu::make_line_parser("mtllib ", ztu::is_not_repeating, [&](const auto& param)
make_line_parser("mtllib ", ztu::is_not_repeating, [&](const auto& param)
{
auto material_filename = fs::path(param);
if (material_filename.is_relative())
path_buffer.assign(param);
if (path_buffer.is_relative())
{
material_filename = directory / material_filename;
path_buffer = base_directory;
path_buffer /= param; // TODO Doesn't thi allocate an extra path?!?
}
if (const auto error = ctx.material_loader.read(
ctx,
mtl_loader_id,
material_filename,
pedantic
)) {
const auto material_library_id_it = id_lookup.material_libraries.find(path_buffer);
if (material_library_id_it != id_lookup.material_libraries.end()) [[likely]]
{
const auto material_library_id = material_library_id_it->second;
std::tie(curr_material_library_it, has_material_library) = store.material_libraries.find(material_library_id);
}
else [[unlikely]]
{
ztu::logger::warn(
"Error occurred while loading mtl files '%': [%] %",
material_filename,
error.category().name(),
error.message()
"Could not find a matching material library with path '%'. Proceeding with default material.",
param
);
has_material_library = false;
}
})
);

2
source/assets/dynamic_data_loaders/dynamic_texture_loader.cpp
View File

@@ -7,8 +7,6 @@
#pragma warning(push, 0)
#endif
#define STB_IMAGE_IMPLEMENTATION
#define STB_IMAGE_STATIC
#include "stb_image.h"
#if defined(__GNUC__) || defined(__GNUG__)

11
source/assets/dynamic_read_buffers/dynamic_material_buffer.ipp
View File

@@ -86,3 +86,14 @@ inline components::material::surface_properties& dynamic_material_buffer::initia
}
return *surface_properties_opt;
}
inline void dynamic_material_buffer::clear()
{
std::apply(
[](auto&... data_opt) {
(data_opt.reset(), ...);
},
data
);
}

7
source/assets/dynamic_read_buffers/dynamic_mesh_buffer.ipp
View File

@@ -71,3 +71,10 @@ inline const auto& dynamic_mesh_buffer::material_id() const
{
return m_material_id;
}
inline void dynamic_mesh_buffer::clear()
{
clear_vertices();
m_triangles.clear();
m_material_id = {};
}

5
source/assets/dynamic_read_buffers/dynamic_point_cloud_buffer.ipp
View File

@@ -42,3 +42,8 @@ inline const std::vector<components::point_cloud_vertex::reflectance>& dynamic_p
{
return std::get<components::point_cloud_vertex::indices::reflectance>(vertices);
}
inline void dynamic_point_cloud_buffer::clear()
{
clear_vertices();
}

8
source/assets/dynamic_read_buffers/dynamic_texture_buffer.ipp
View File

@@ -149,3 +149,11 @@ inline dynamic_texture_buffer::const_iterator dynamic_texture_buffer::cend() con
{
return const_cast<const_iterator>(begin());
}
inline void dynamic_texture_buffer::clear()
{
m_data.reset();
m_width = 0;
m_height = 0;
m_components = {};
}