#include "assets/file_parsers/mtl_parser.hpp" #include #include #include #include "util/logger.hpp" #include "util/for_each.hpp" #include "util/line_parser.hpp" #include namespace assets::mtl_parser_error { struct category : std::error_category { [[nodiscard]] const char* name() const noexcept override { return "mtl_parser"; } [[nodiscard]] std::string message(int ev) const override { // TODO these are incomplete switch (static_cast(ev)) { using enum codes; case cannot_open_file: return "Cannot open mtl file."; case cannot_open_texture: return "Cannot open texture file."; case malformed_ambient_color: return "File contains malformed 'Ka' statement."; case malformed_diffuse_color: return "File contains malformed 'Kd' statement."; case malformed_specular_color: return "File contains malformed 'Ks' statement."; case malformed_specular_exponent: return "File contains malformed 'Ns' statement."; case malformed_dissolve: return "File contains malformed 'd' statement."; case unknown_line_begin: return "Unknown mtl line begin"; default: using namespace std::string_literals; return "unrecognized error ("s + std::to_string(ev) + ")"; } } }; } // namespace mtl_parser_error inline std::error_category& mtl_parser_error_category() { static assets::mtl_parser_error::category category; return category; } namespace assets::mtl_parser_error { inline std::error_code make_error_code(codes e) { return { static_cast(e), mtl_parser_error_category() }; } } // namespace mtl_loader_error std::error_code assets::mtl_parser::prefetch( path_id_lookups& lookups ) { namespace fs = std::filesystem; using mtl_parser_error::codes; using mtl_parser_error::make_error_code; auto buffer = std::vector(8 * 1024, '\0'); auto in = std::ifstream{}; auto path_buffer = fs::path{}; auto filename_buffer = fs::path{}; const auto process_file = [&]() { in.open(filename_buffer); if (not in.is_open()) { ztu::logger::error("Could not open .mtl file '%'", filename_buffer); return; } filename_buffer.remove_filename(); find_textures(buffer, path_buffer, filename_buffer, in, queue.texture.files); in.close(); }; // TODO properly extract for (const auto file : paths.files) { filename_buffer.assign(file); process_file(); } for (const auto directory : paths.directories) { for (const auto& file : fs::directory_iterator{ directory }) { filename_buffer.assign(file.path()); // Avoid heap allocation of .extension() if (not std::string_view(filename_buffer.c_str()).ends_with(".obj")) { continue; } process_file(); } } return {}; } std::error_code assets::mtl_parser::load( path_id_lookups& lookups, data_stores& stores, bool pedantic ) { m_path_buffer.clear(); lookups.material_libraries.by_extension(".mtl", m_path_buffer); auto store_mutex = std::mutex{}; // TODO who protects the material id lookup // TODO who protects the material store std::for_each( std::execution::parallel_unsequenced_policy{}, m_path_buffer.begin(), m_path_buffer.end(), parser_context{ lookups.textures, lookups.materials, stores.materials, stores.material_libraries, store_mutex } ); return {}; } inline std::error_category& connector_error_category() { static assets::mtl_parser_error::category category; return category; } namespace assets::mtl_parser_error { inline std::error_code make_error_code(const codes e) { return { static_cast(e), connector_error_category() }; } } // namespace mtl_loader_error assets::mtl_parser::parser_context::parser_context( const texture_id_lookup& texture_id_lookup, material_id_lookup& material_id_lookup, material_store& material_store, store_type& store, std::mutex& store_mutex ) : m_texture_id_lookup{ &texture_id_lookup }, m_material_id_lookup{ &material_id_lookup }, m_material_store{ &material_store }, m_store{ &store }, m_store_mutex{ &store_mutex } { m_buffer.reserve(32); } void assets::mtl_parser::parser_context::reset() { m_buffer.clear(); } void assets::mtl_parser::parser_context::operator()(lookup_type::const_pointer entry) noexcept { using mtl_parser_error::codes; using mtl_parser_error::make_error_code; namespace fs = std::filesystem; const auto& [ filename, id ] = *entry; // TODO unroll stuff auto in = std::ifstream{ filename }; if (not in.is_open()) { ztu::logger::warn("Cannot open mtl file %.", filename); return; } const auto base_dir = fs::canonical(fs::path(filename).parent_path()); auto name = std::string{}; auto material = material_data{}; const auto push_material = [&]() { if (not name.empty()) { const auto [ id_it, is_new ] = m_material_id_lookup->try_emplace(filename / name); const auto material_id = id_it->second; m_material_store->emplace(id, material); m_buffer.emplace(name, material_id); } name = {}; material = {}; }; const auto ec = ztu::parse_lines( in, pedantic, ztu::make_line_parser("newmtl ", ztu::is_not_repeating, [&](const auto& param) { push_material(); name = param; return codes::ok; }), ztu::make_line_parser("Ka ", ztu::is_not_repeating, [&](const auto& param) { material_components::ambient_filter ambient_filter; if (parse_numeric_vector(param, ambient_filter) != std::errc{}) [[unlikely]] { return codes::malformed_ambient_color; // TODO rename to filter } material.ambient_filter() = ambient_filter; material.component_flags |= material_components::flags::ambient_filter; return codes::ok; }), ztu::make_line_parser("Kd ", ztu::is_not_repeating, [&](const auto& param) { material_components::diffuse_filter diffuse_filter; if (parse_numeric_vector(param, diffuse_filter) != std::errc{}) [[unlikely]] { return codes::malformed_diffuse_color; } material.diffuse_filter() = diffuse_filter; material.component_flags |= material_components::flags::diffuse_filter; return codes::ok; }), ztu::make_line_parser("Ks ", ztu::is_not_repeating, [&](const auto& param) { material_components::specular_filter specular_filter; if (parse_numeric_vector(param, specular_filter) != std::errc{}) [[unlikely]] { return codes::malformed_specular_color; } material.specular_filter() = specular_filter; material.component_flags |= material_components::flags::specular_filter; return codes::ok; }), ztu::make_line_parser("Ns ", ztu::is_not_repeating, [&](const auto& param) { z3d::vec<1, material_components::shininess> shininess; if (parse_numeric_vector(param, shininess) != std::errc{}) [[unlikely]] { return codes::malformed_specular_exponent; } material.shininess() = shininess[0]; material.component_flags |= material_components::flags::shininess; return codes::ok; }), ztu::make_line_parser("d ", ztu::is_not_repeating, [&](const auto& param) { z3d::vec<1, material_components::alpha> alpha{}; if (parse_numeric_vector(param, alpha) != std::errc{}) [[unlikely]] { return codes::malformed_dissolve; } material.alpha() = alpha[0]; material.component_flags |= material_components::flags::alpha; return codes::ok; }), ztu::make_line_parser("map_Ka ", ztu::is_not_repeating, [&](const auto& param) { if (const auto texture_id = fetch_texture_id(param, "ambient filter")) { material.ambient_filter_texture_id() = *texture_id; material.component_flags |= material_components::flags::ambient_filter_texture; } return codes::ok; }), ztu::make_line_parser("map_Kd ", ztu::is_not_repeating, [&](const auto& param) { if (const auto texture_id = fetch_texture_id(param, "diffuse filter")) { material.diffuse_filter_texture_id() = *texture_id; material.component_flags |= material_components::flags::diffuse_filter_texture; } return codes::ok; }), ztu::make_line_parser("map_Ks ", ztu::is_not_repeating, [&](const auto& param) { if (const auto texture_id = fetch_texture_id(param, "specular filter")) { material.specular_filter_texture_id() = *texture_id; material.component_flags |= material_components::flags::specular_filter_texture; } return codes::ok; }), ztu::make_line_parser("map_Ns ", ztu::is_not_repeating, [&](const auto& param) { if (const auto texture_id = fetch_texture_id(param, "shininess")) { material.shininess_texture_id() = *texture_id; material.component_flags |= material_components::flags::shininess_texture; } return codes::ok; }), ztu::make_line_parser("map_d ", ztu::is_not_repeating, [&](const auto& param) { if (const auto texture_id = fetch_texture_id(param, "alpha")) { material.alpha_texture_id() = *texture_id; material.component_flags |= material_components::flags::alpha_texture; } return codes::ok; }), ztu::make_line_parser("bump ", ztu::is_not_repeating, [&](const auto& param) { if (const auto texture_id = fetch_texture_id(param, "bump")) { material.bump_texture_id() = *texture_id; material.component_flags |= material_components::flags::bump_texture; } return codes::ok; }) ); if (ec != codes::ok) { const auto e = make_error_code(ec); ztu::logger::error("Error while parsing mtl file %: %", filename, e.message()); } push_material(); } std::optional assets::mtl_parser::parser_context::fetch_texture_id( const std::filesystem::path& mtl_dir, std::string_view filename, std::string_view texture_type_name ) { auto texture_filename = std::filesystem::path(filename); if (texture_filename.is_relative()) { texture_filename = mtl_dir / texture_filename; } const auto texture_id_it = m_texture_id_lookup->find(texture_filename); if (texture_id_it == m_texture_id_lookup->end()) { ztu::logger::warn( "%-texture at % has not been registered and can therefor not be used.", texture_type_name, texture_filename ); return std::nullopt; } return texture_id_it->second; } template std::errc parse_numeric_vector(std::string_view param, std::array& values) { const auto end = param.end(); auto it = param.begin(); for (auto& value : values) { if (it >= end) { return std::errc::invalid_argument; } const auto [ptr, ec] = std::from_chars(it, end, value); if (ec != std::errc{}) { return ec; } it = ptr + 1; // skip space in between components } return {}; }; void assets::mtl_parser::find_textures( std::span buffer, std::filesystem::path& path_buffer, const std::filesystem::path& base_directory, std::ifstream& in, ztu::string_list& texture_filenames ) { using namespace std::string_view_literals; // TODO 'bump' is missing!!! static constexpr auto keyword = "\nmap_"sv; using long_postfix_type = std::array; static constexpr auto postfix_length = std::tuple_size_v; static constexpr auto make_postfix = [](const std::string_view str) static constexpr { auto postfix = long_postfix_type{}; assert(str.length() >= postfix_length); std::copy_n(str.begin(), postfix.size(), postfix.begin()); return postfix; }; static constexpr auto postfixes = std::array{ make_postfix("d "), make_postfix("Ka"), make_postfix("Kd"), make_postfix("Ks"), make_postfix("Ns") }; const auto buffer_view = std::string_view(buffer); // Add linebreak to simplify line begin search. buffer.front() = '\n'; auto leftover = std::size_t{ 1 }; enum class match { exact, overflowed, none }; const auto check_match = [](std::string_view& potential_match) static -> match { std::cout << '\'' << potential_match.substr(0, std::min(40ul, potential_match.size())) << '\'' << std::endl; if (potential_match.length() < postfix_length) { return match::overflowed; } const auto postfix = make_postfix(potential_match); // Optimized for SIMD. if (not std::ranges::contains(postfixes, postfix)) { return match::none; } const auto long_match = postfix.back() != ' '; if (long_match and ( potential_match.length() < postfix_length + sizeof(' ') or potential_match[postfix_length] != ' ' )) { return match::overflowed; } const auto actual_postfix_length = std::size_t{ 1 } + static_cast(long_match); const auto filename_begin = actual_postfix_length + sizeof(' '); const auto filename_end = potential_match.find('\n', filename_begin); if (filename_end == std::string_view::npos) { return match::overflowed; } const auto length = filename_end - filename_begin; potential_match = potential_match.substr(filename_begin, length); return match::exact; }; do { // Keep some old characters to continue matching interrupted sequence. std::copy(buffer.end() - leftover, buffer.end(), buffer.begin()); in.read(buffer.data() + leftover, buffer.size() - leftover); const auto str = buffer_view.substr(0, leftover + in.gcount()); auto pos = std::string_view::size_type{}; while ((pos = str.find(keyword, pos)) != std::string_view::npos) { const auto keyword_end = pos + keyword.size(); auto potential_match = str.substr(keyword_end); const auto match_type = check_match(potential_match); if (match_type == match::exact) { path_buffer.assign(potential_match); if (path_buffer.is_relative()) { path_buffer = base_directory; path_buffer /= potential_match; } texture_filenames.push_back(path_buffer.c_str()); pos += potential_match.size(); leftover = 0; } else if (match_type == match::overflowed) { if (pos == 0) [[unlikely]] { ztu::logger::error("Ignoring string match, as it exceeds buffer size of % characters.", buffer.size()); leftover = 0; } else { leftover = str.size() - pos; } break; } else { leftover = keyword.size(); } } } while (not in.eof()); }