Ported the obj parser.
This commit is contained in:
zy4n
@@ -63,58 +63,47 @@ inline std::error_code make_error_code(codes e)
|
||||
}
|
||||
} // namespace mtl_loader_error
|
||||
|
||||
template<int L, typename T>
|
||||
std::errc parse_numeric_vector(std::string_view param, z3d::vec<L, T>& vec)
|
||||
{
|
||||
auto it = param.begin();
|
||||
const auto end = param.end();
|
||||
|
||||
for (int i{}; i != L; ++i)
|
||||
{
|
||||
if (it >= end)
|
||||
{
|
||||
return std::errc::invalid_argument;
|
||||
}
|
||||
|
||||
const auto [ptr, ec] = std::from_chars(it, end, vec[i]);
|
||||
|
||||
if (ec != std::errc{})
|
||||
{
|
||||
return ec;
|
||||
}
|
||||
|
||||
it = ptr + 1; // Skip space in between components.
|
||||
}
|
||||
|
||||
return {};
|
||||
};
|
||||
|
||||
|
||||
std::error_code assets::mtl_parser::prefetch(
|
||||
path_id_lookups& lookups
|
||||
) {
|
||||
namespace fs = std::filesystem;
|
||||
using mtl_parser_error::codes;
|
||||
using mtl_parser_error::make_error_code;
|
||||
m_path_buffer.clear();
|
||||
lookups.material_libraries.by_extension(".mtl", m_path_buffer);
|
||||
|
||||
auto buffer = std::vector<char>(8 * 1024, '\0');
|
||||
|
||||
auto in = std::ifstream{};
|
||||
|
||||
auto path_buffer = fs::path{};
|
||||
auto filename_buffer = fs::path{};
|
||||
|
||||
const auto process_file = [&]()
|
||||
{
|
||||
in.open(filename_buffer);
|
||||
if (not in.is_open()) {
|
||||
ztu::logger::error("Could not open .mtl file '%'", filename_buffer);
|
||||
return;
|
||||
std::for_each(
|
||||
std::execution::parallel_unsequenced_policy{},
|
||||
m_path_buffer.begin(),
|
||||
m_path_buffer.end(),
|
||||
prefetcher_context{
|
||||
lookups
|
||||
}
|
||||
|
||||
filename_buffer.remove_filename();
|
||||
|
||||
find_textures(buffer, path_buffer, filename_buffer, in, queue.texture.files);
|
||||
|
||||
in.close();
|
||||
};
|
||||
|
||||
// TODO properly extract
|
||||
for (const auto file : paths.files)
|
||||
{
|
||||
filename_buffer.assign(file);
|
||||
process_file();
|
||||
}
|
||||
|
||||
for (const auto directory : paths.directories)
|
||||
{
|
||||
for (const auto& file : fs::directory_iterator{ directory }) {
|
||||
|
||||
filename_buffer.assign(file.path());
|
||||
|
||||
// Avoid heap allocation of .extension()
|
||||
if (not std::string_view(filename_buffer.c_str()).ends_with(".obj"))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
process_file();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
return {};
|
||||
}
|
||||
@@ -127,20 +116,14 @@ std::error_code assets::mtl_parser::load(
|
||||
m_path_buffer.clear();
|
||||
lookups.material_libraries.by_extension(".mtl", m_path_buffer);
|
||||
|
||||
auto store_mutex = std::mutex{};
|
||||
// TODO who protects the material id lookup
|
||||
// TODO who protects the material store
|
||||
|
||||
std::for_each(
|
||||
std::execution::parallel_unsequenced_policy{},
|
||||
m_path_buffer.begin(),
|
||||
m_path_buffer.end(),
|
||||
parser_context{
|
||||
lookups.textures,
|
||||
lookups.materials,
|
||||
stores.materials,
|
||||
stores.material_libraries,
|
||||
store_mutex
|
||||
lookups,
|
||||
stores,
|
||||
}
|
||||
);
|
||||
|
||||
@@ -149,31 +132,169 @@ std::error_code assets::mtl_parser::load(
|
||||
|
||||
|
||||
|
||||
inline std::error_category& connector_error_category()
|
||||
assets::mtl_parser::prefetcher_context::prefetcher_context(
|
||||
path_id_lookups& lookups
|
||||
) :
|
||||
m_lookups{ &lookups }
|
||||
{
|
||||
static assets::mtl_parser_error::category category;
|
||||
return category;
|
||||
m_buffer.resize(8192);
|
||||
}
|
||||
|
||||
namespace assets::mtl_parser_error
|
||||
void assets::mtl_parser::prefetcher_context::operator()(lookup_type::const_pointer entry) noexcept
|
||||
{
|
||||
inline std::error_code make_error_code(const codes e)
|
||||
{
|
||||
return { static_cast<int>(e), connector_error_category() };
|
||||
}
|
||||
} // namespace mtl_loader_error
|
||||
namespace fs = std::filesystem;
|
||||
using namespace std::string_view_literals;
|
||||
|
||||
// TODO 'bump' is missing!!!
|
||||
|
||||
static constexpr auto keyword = "\nmap_"sv;
|
||||
|
||||
using long_postfix_type = std::array<char, 2>;
|
||||
|
||||
static constexpr auto postfix_length = std::tuple_size_v<long_postfix_type>;
|
||||
|
||||
static constexpr auto make_postfix = [](const std::string_view str) static constexpr
|
||||
{
|
||||
auto postfix = long_postfix_type{};
|
||||
assert(str.length() >= postfix_length);
|
||||
std::copy_n(str.begin(), postfix.size(), postfix.begin());
|
||||
return postfix;
|
||||
};
|
||||
|
||||
static constexpr auto postfixes = std::array{
|
||||
make_postfix("d "),
|
||||
make_postfix("Ka"),
|
||||
make_postfix("Kd"),
|
||||
make_postfix("Ks"),
|
||||
make_postfix("Ns")
|
||||
};
|
||||
|
||||
const auto buffer_view = std::string_view(m_buffer);
|
||||
|
||||
// Add linebreak to simplify line begin search.
|
||||
m_buffer.front() = '\n';
|
||||
auto leftover = std::size_t{ 1 };
|
||||
|
||||
enum class match {
|
||||
exact,
|
||||
overflowed,
|
||||
none
|
||||
};
|
||||
|
||||
const auto check_match = [](std::string_view& potential_match) static -> match
|
||||
{
|
||||
std::cout << '\'' << potential_match.substr(0, std::min(40ul, potential_match.size())) << '\'' << std::endl;
|
||||
|
||||
if (potential_match.length() < postfix_length)
|
||||
{
|
||||
return match::overflowed;
|
||||
}
|
||||
|
||||
const auto postfix = make_postfix(potential_match);
|
||||
|
||||
// Optimized for SIMD.
|
||||
if (not std::ranges::contains(postfixes, postfix))
|
||||
{
|
||||
return match::none;
|
||||
}
|
||||
|
||||
const auto long_match = postfix.back() != ' ';
|
||||
|
||||
if (long_match and (
|
||||
potential_match.length() < postfix_length + sizeof(' ') or
|
||||
potential_match[postfix_length] != ' '
|
||||
)) {
|
||||
return match::overflowed;
|
||||
}
|
||||
|
||||
const auto actual_postfix_length = std::size_t{ 1 } + static_cast<std::size_t>(long_match);
|
||||
|
||||
const auto filename_begin = actual_postfix_length + sizeof(' ');
|
||||
const auto filename_end = potential_match.find('\n', filename_begin);
|
||||
|
||||
if (filename_end == std::string_view::npos)
|
||||
{
|
||||
return match::overflowed;
|
||||
}
|
||||
|
||||
const auto length = filename_end - filename_begin;
|
||||
potential_match = potential_match.substr(filename_begin, length);
|
||||
|
||||
return match::exact;
|
||||
};
|
||||
|
||||
const auto& [ filename, id ] = *entry;
|
||||
|
||||
auto path_buffer = fs::path{};
|
||||
const auto base_dir = fs::canonical(fs::path(filename).parent_path());
|
||||
|
||||
auto in = std::ifstream{ filename };
|
||||
if (not in.is_open())
|
||||
{
|
||||
ztu::logger::warn("Cannot open obj file %.", filename);
|
||||
return;
|
||||
}
|
||||
|
||||
do
|
||||
{
|
||||
// Keep some old characters to continue matching interrupted sequence.
|
||||
std::copy(m_buffer.end() - leftover, m_buffer.end(), m_buffer.begin());
|
||||
|
||||
in.read(m_buffer.data() + leftover, m_buffer.size() - leftover);
|
||||
|
||||
const auto str = buffer_view.substr(0, leftover + in.gcount());
|
||||
|
||||
auto pos = std::string_view::size_type{};
|
||||
while ((pos = str.find(keyword, pos)) != std::string_view::npos)
|
||||
{
|
||||
const auto keyword_end = pos + keyword.size();
|
||||
auto potential_match = str.substr(keyword_end);
|
||||
|
||||
const auto match_type = check_match(potential_match);
|
||||
|
||||
if (match_type == match::exact)
|
||||
{
|
||||
path_buffer.assign(potential_match);
|
||||
|
||||
if (path_buffer.is_relative())
|
||||
{
|
||||
path_buffer = base_dir;
|
||||
path_buffer /= potential_match;
|
||||
}
|
||||
|
||||
m_lookups->textures.try_emplace(path_buffer);
|
||||
|
||||
pos += potential_match.size();
|
||||
|
||||
leftover = 0;
|
||||
}
|
||||
else if (match_type == match::overflowed)
|
||||
{
|
||||
if (pos == 0) [[unlikely]]
|
||||
{
|
||||
ztu::logger::error("Ignoring string match, as it exceeds buffer size of % characters.", m_buffer.size());
|
||||
leftover = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
leftover = str.size() - pos;
|
||||
}
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
leftover = keyword.size();
|
||||
}
|
||||
}
|
||||
} while (not in.eof());
|
||||
}
|
||||
|
||||
assets::mtl_parser::parser_context::parser_context(
|
||||
const texture_id_lookup& texture_id_lookup,
|
||||
material_id_lookup& material_id_lookup,
|
||||
material_store& material_store,
|
||||
store_type& store
|
||||
path_id_lookups& lookups,
|
||||
data_stores& stores
|
||||
) :
|
||||
m_texture_id_lookup{ &texture_id_lookup },
|
||||
m_material_id_lookup{ &material_id_lookup },
|
||||
m_material_store{ &material_store },
|
||||
m_store{ &store }
|
||||
m_lookups{ &lookups },
|
||||
m_stores{ &stores }
|
||||
{
|
||||
m_buffer.reserve(32);
|
||||
}
|
||||
@@ -191,6 +312,7 @@ void assets::mtl_parser::parser_context::operator()(lookup_type::const_pointer e
|
||||
|
||||
const auto& [ filename, id ] = *entry;
|
||||
|
||||
|
||||
// TODO unroll stuff
|
||||
|
||||
auto in = std::ifstream{ filename };
|
||||
@@ -209,11 +331,11 @@ void assets::mtl_parser::parser_context::operator()(lookup_type::const_pointer e
|
||||
{
|
||||
if (not name.empty())
|
||||
{
|
||||
const auto [ id_it, is_new ] = m_material_id_lookup->try_emplace(filename / name);
|
||||
const auto [ id_it, is_new ] = m_lookups->materials.try_emplace(filename / name);
|
||||
|
||||
const auto material_id = id_it->second;
|
||||
|
||||
m_material_store->insert(id, material);
|
||||
m_stores->materials.insert(material_id, material);
|
||||
|
||||
m_buffer.emplace(name, material_id);
|
||||
}
|
||||
@@ -223,7 +345,7 @@ void assets::mtl_parser::parser_context::operator()(lookup_type::const_pointer e
|
||||
|
||||
const auto ec = ztu::parse_lines<codes>(
|
||||
in,
|
||||
pedantic,
|
||||
true, // TODO pedantic,
|
||||
ztu::make_line_parser("newmtl ", ztu::is_not_repeating, [&](const auto& param)
|
||||
{
|
||||
push_material();
|
||||
@@ -298,7 +420,7 @@ void assets::mtl_parser::parser_context::operator()(lookup_type::const_pointer e
|
||||
}),
|
||||
ztu::make_line_parser("map_Ka ", ztu::is_not_repeating, [&](const auto& param)
|
||||
{
|
||||
if (const auto texture_id = fetch_texture_id(param, "ambient filter"))
|
||||
if (const auto texture_id = fetch_texture_id(base_dir, param, "ambient filter"))
|
||||
{
|
||||
material.ambient_filter_texture_id() = *texture_id;
|
||||
material.component_flags |= material_components::flags::ambient_filter_texture;
|
||||
@@ -308,7 +430,7 @@ void assets::mtl_parser::parser_context::operator()(lookup_type::const_pointer e
|
||||
}),
|
||||
ztu::make_line_parser("map_Kd ", ztu::is_not_repeating, [&](const auto& param)
|
||||
{
|
||||
if (const auto texture_id = fetch_texture_id(param, "diffuse filter"))
|
||||
if (const auto texture_id = fetch_texture_id(base_dir, param, "diffuse filter"))
|
||||
{
|
||||
material.diffuse_filter_texture_id() = *texture_id;
|
||||
material.component_flags |= material_components::flags::diffuse_filter_texture;
|
||||
@@ -318,7 +440,7 @@ void assets::mtl_parser::parser_context::operator()(lookup_type::const_pointer e
|
||||
}),
|
||||
ztu::make_line_parser("map_Ks ", ztu::is_not_repeating, [&](const auto& param)
|
||||
{
|
||||
if (const auto texture_id = fetch_texture_id(param, "specular filter"))
|
||||
if (const auto texture_id = fetch_texture_id(base_dir, param, "specular filter"))
|
||||
{
|
||||
material.specular_filter_texture_id() = *texture_id;
|
||||
material.component_flags |= material_components::flags::specular_filter_texture;
|
||||
@@ -328,7 +450,7 @@ void assets::mtl_parser::parser_context::operator()(lookup_type::const_pointer e
|
||||
}),
|
||||
ztu::make_line_parser("map_Ns ", ztu::is_not_repeating, [&](const auto& param)
|
||||
{
|
||||
if (const auto texture_id = fetch_texture_id(param, "shininess"))
|
||||
if (const auto texture_id = fetch_texture_id(base_dir, param, "shininess"))
|
||||
{
|
||||
material.shininess_texture_id() = *texture_id;
|
||||
material.component_flags |= material_components::flags::shininess_texture;
|
||||
@@ -338,7 +460,7 @@ void assets::mtl_parser::parser_context::operator()(lookup_type::const_pointer e
|
||||
}),
|
||||
ztu::make_line_parser("map_d ", ztu::is_not_repeating, [&](const auto& param)
|
||||
{
|
||||
if (const auto texture_id = fetch_texture_id(param, "alpha"))
|
||||
if (const auto texture_id = fetch_texture_id(base_dir, param, "alpha"))
|
||||
{
|
||||
material.alpha_texture_id() = *texture_id;
|
||||
material.component_flags |= material_components::flags::alpha_texture;
|
||||
@@ -348,7 +470,7 @@ void assets::mtl_parser::parser_context::operator()(lookup_type::const_pointer e
|
||||
}),
|
||||
ztu::make_line_parser("bump ", ztu::is_not_repeating, [&](const auto& param)
|
||||
{
|
||||
if (const auto texture_id = fetch_texture_id(param, "bump"))
|
||||
if (const auto texture_id = fetch_texture_id(base_dir, param, "bump"))
|
||||
{
|
||||
material.bump_texture_id() = *texture_id;
|
||||
material.component_flags |= material_components::flags::bump_texture;
|
||||
@@ -366,7 +488,7 @@ void assets::mtl_parser::parser_context::operator()(lookup_type::const_pointer e
|
||||
|
||||
push_material();
|
||||
|
||||
m_store->insert(id, m_buffer);
|
||||
m_stores->material_libraries.insert(id, m_buffer);
|
||||
}
|
||||
|
||||
std::optional<assets::texture_id> assets::mtl_parser::parser_context::fetch_texture_id(
|
||||
@@ -380,8 +502,8 @@ std::optional<assets::texture_id> assets::mtl_parser::parser_context::fetch_text
|
||||
texture_filename = mtl_dir / texture_filename;
|
||||
}
|
||||
|
||||
const auto texture_id_it = m_texture_id_lookup->find(texture_filename);
|
||||
if (texture_id_it == m_texture_id_lookup->end())
|
||||
const auto texture_id_it = m_lookups->textures.find(texture_filename);
|
||||
if (texture_id_it == m_lookups->textures.end())
|
||||
{
|
||||
ztu::logger::warn(
|
||||
"%-texture at % has not been registered and can therefor not be used.",
|
||||
@@ -393,175 +515,3 @@ std::optional<assets::texture_id> assets::mtl_parser::parser_context::fetch_text
|
||||
|
||||
return texture_id_it->second;
|
||||
}
|
||||
|
||||
|
||||
|
||||
template<typename T, std::size_t Count>
|
||||
std::errc parse_numeric_vector(std::string_view param, std::array<T, Count>& values)
|
||||
{
|
||||
const auto end = param.end();
|
||||
auto it = param.begin();
|
||||
|
||||
for (auto& value : values)
|
||||
{
|
||||
if (it >= end)
|
||||
{
|
||||
return std::errc::invalid_argument;
|
||||
}
|
||||
|
||||
const auto [ptr, ec] = std::from_chars(it, end, value);
|
||||
|
||||
if (ec != std::errc{})
|
||||
{
|
||||
return ec;
|
||||
}
|
||||
|
||||
it = ptr + 1; // skip space in between components
|
||||
}
|
||||
|
||||
return {};
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
||||
void assets::mtl_parser::find_textures(
|
||||
std::span<char> buffer,
|
||||
std::filesystem::path& path_buffer,
|
||||
const std::filesystem::path& base_directory,
|
||||
std::ifstream& in,
|
||||
ztu::string_list& texture_filenames
|
||||
) {
|
||||
using namespace std::string_view_literals;
|
||||
|
||||
// TODO 'bump' is missing!!!
|
||||
|
||||
static constexpr auto keyword = "\nmap_"sv;
|
||||
|
||||
using long_postfix_type = std::array<char, 2>;
|
||||
|
||||
static constexpr auto postfix_length = std::tuple_size_v<long_postfix_type>;
|
||||
|
||||
static constexpr auto make_postfix = [](const std::string_view str) static constexpr
|
||||
{
|
||||
auto postfix = long_postfix_type{};
|
||||
assert(str.length() >= postfix_length);
|
||||
std::copy_n(str.begin(), postfix.size(), postfix.begin());
|
||||
return postfix;
|
||||
};
|
||||
|
||||
static constexpr auto postfixes = std::array{
|
||||
make_postfix("d "),
|
||||
make_postfix("Ka"),
|
||||
make_postfix("Kd"),
|
||||
make_postfix("Ks"),
|
||||
make_postfix("Ns")
|
||||
};
|
||||
|
||||
const auto buffer_view = std::string_view(buffer);
|
||||
|
||||
// Add linebreak to simplify line begin search.
|
||||
buffer.front() = '\n';
|
||||
auto leftover = std::size_t{ 1 };
|
||||
|
||||
enum class match {
|
||||
exact,
|
||||
overflowed,
|
||||
none
|
||||
};
|
||||
|
||||
const auto check_match = [](std::string_view& potential_match) static -> match
|
||||
{
|
||||
std::cout << '\'' << potential_match.substr(0, std::min(40ul, potential_match.size())) << '\'' << std::endl;
|
||||
|
||||
if (potential_match.length() < postfix_length)
|
||||
{
|
||||
return match::overflowed;
|
||||
}
|
||||
|
||||
const auto postfix = make_postfix(potential_match);
|
||||
|
||||
// Optimized for SIMD.
|
||||
if (not std::ranges::contains(postfixes, postfix))
|
||||
{
|
||||
return match::none;
|
||||
}
|
||||
|
||||
const auto long_match = postfix.back() != ' ';
|
||||
|
||||
if (long_match and (
|
||||
potential_match.length() < postfix_length + sizeof(' ') or
|
||||
potential_match[postfix_length] != ' '
|
||||
)) {
|
||||
return match::overflowed;
|
||||
}
|
||||
|
||||
const auto actual_postfix_length = std::size_t{ 1 } + static_cast<std::size_t>(long_match);
|
||||
|
||||
const auto filename_begin = actual_postfix_length + sizeof(' ');
|
||||
const auto filename_end = potential_match.find('\n', filename_begin);
|
||||
|
||||
if (filename_end == std::string_view::npos)
|
||||
{
|
||||
return match::overflowed;
|
||||
}
|
||||
|
||||
const auto length = filename_end - filename_begin;
|
||||
potential_match = potential_match.substr(filename_begin, length);
|
||||
|
||||
return match::exact;
|
||||
};
|
||||
|
||||
do
|
||||
{
|
||||
// Keep some old characters to continue matching interrupted sequence.
|
||||
std::copy(buffer.end() - leftover, buffer.end(), buffer.begin());
|
||||
|
||||
in.read(buffer.data() + leftover, buffer.size() - leftover);
|
||||
|
||||
const auto str = buffer_view.substr(0, leftover + in.gcount());
|
||||
|
||||
auto pos = std::string_view::size_type{};
|
||||
while ((pos = str.find(keyword, pos)) != std::string_view::npos)
|
||||
{
|
||||
const auto keyword_end = pos + keyword.size();
|
||||
auto potential_match = str.substr(keyword_end);
|
||||
|
||||
const auto match_type = check_match(potential_match);
|
||||
|
||||
if (match_type == match::exact)
|
||||
{
|
||||
path_buffer.assign(potential_match);
|
||||
|
||||
if (path_buffer.is_relative())
|
||||
{
|
||||
path_buffer = base_directory;
|
||||
path_buffer /= potential_match;
|
||||
}
|
||||
|
||||
texture_filenames.push_back(path_buffer.c_str());
|
||||
|
||||
pos += potential_match.size();
|
||||
|
||||
leftover = 0;
|
||||
}
|
||||
else if (match_type == match::overflowed)
|
||||
{
|
||||
if (pos == 0) [[unlikely]]
|
||||
{
|
||||
ztu::logger::error("Ignoring string match, as it exceeds buffer size of % characters.", buffer.size());
|
||||
leftover = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
leftover = str.size() - pos;
|
||||
}
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
leftover = keyword.size();
|
||||
}
|
||||
}
|
||||
} while (not in.eof());
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user