In the middle of multithreading parsers.

This commit is contained in:
zy4n
2025-03-30 22:38:06 +02:00
parent d18b40a7fc
commit 144126ee7a
80 changed files with 2904 additions and 1450 deletions

View File

@@ -1,36 +0,0 @@
#include "../../../include/assets/data_parsers"
#include <fstream>
std::error_code glsl_loader::load(
const std::filesystem::path& filename,
std::string& source
) {
auto file = std::ifstream(filename);
if (not file.is_open())
{
return std::make_error_code(std::errc::no_such_file_or_directory);
}
file.seekg(0, std::ios::end);
const auto size = file.tellg();
if (size == 0 or size == std::numeric_limits<std::streamsize>::max())
{
return std::make_error_code(std::errc::invalid_seek);
}
source.reserve(size);
file.seekg(0, std::ios::beg);
source.assign(
std::istreambuf_iterator<char>(file),
std::istreambuf_iterator<char>()
);
file.close();
return {};
}

View File

@@ -0,0 +1,463 @@
#include "assets/file_parsers/glsl_parser.hpp"
#include <fstream>
#include <numeric>
#include "assets/components/mesh_shader_components.hpp"
#include "assets/components/point_cloud_shader_components.hpp"
#include "util/logger.hpp"
#include <mutex>
#include <execution>
namespace assets::language
{
enum class declaration_type : std::size_t
{
stage = 0,
geometry = 1,
features = 2,
static_enable = 3,
dynamic_enable = 4,
invalid = std::numeric_limits<std::size_t>::max()
};
using stage = shader_components::stage;
inline constexpr auto declaration_prefix = std::string_view("\n#pragma ");
inline constexpr auto title_separator = ':';
inline constexpr auto value_separator = ' ';
inline auto declaration_lookup = ztu::string_lookup<declaration_type>{
{ "STAGE", declaration_type::stage },
{ "GEOMETRY", declaration_type::geometry },
{ "FEATURES", declaration_type::features },
{ "STATIC_ENABLE", declaration_type::static_enable },
{ "DYNAMIC_ENABLE", declaration_type::dynamic_enable }
};
inline auto stage_lookup = ztu::string_lookup<shader_components::stage>{
{ "VERTEX", stage::vertex },
{ "TESSELATION_CONTROL", stage::tesselation_control },
{ "TESSELATION_EVALUATION", stage::tesselation_evaluation },
{ "GEOMETRY", stage::geometry },
{ "FRAGMENT", stage::fragment },
};
inline auto geometry_lookup = ztu::string_lookup<model_geometry::types>{
{ "MESH", model_geometry::types::mesh },
{ "POINT_CLOUD", model_geometry::types::point_cloud }
};
inline auto mesh_feature_lookup = ztu::string_lookup<shader_components::flags>{
{ "FACE", static_cast<shader_components::flags>(mesh_shader_components::flags::face) },
{ "LINE", static_cast<shader_components::flags>(mesh_shader_components::flags::line) },
{ "POINT", static_cast<shader_components::flags>(mesh_shader_components::flags::point) },
{ "V_L", static_cast<shader_components::flags>(mesh_shader_components::flags::luminance) },
{ "V_RGB", static_cast<shader_components::flags>(mesh_shader_components::flags::color) },
{ "V_R", static_cast<shader_components::flags>(mesh_shader_components::flags::alpha) },
{ "COLOR_TEXTURE", static_cast<shader_components::flags>(mesh_shader_components::flags::color_texture) },
{ "U_LIGHTING", static_cast<shader_components::flags>(mesh_shader_components::flags::uniform_lighting) },
{ "T_LIGHTING", static_cast<shader_components::flags>(mesh_shader_components::flags::textured_lighting) },
{ "U_RGBA", static_cast<shader_components::flags>(mesh_shader_components::flags::uniform_color) }
};
inline auto point_cloud_feature_lookup = ztu::string_lookup<shader_components::flags>{
{ "SQUARE", static_cast<shader_components::flags>(point_cloud_shader_components::flags::square) },
{ "LIGHTING", static_cast<shader_components::flags>(point_cloud_shader_components::flags::lighting) },
{ "V_L", static_cast<shader_components::flags>(point_cloud_shader_components::flags::luminance) },
{ "V_RGB", static_cast<shader_components::flags>(point_cloud_shader_components::flags::color) },
{ "V_A", static_cast<shader_components::flags>(point_cloud_shader_components::flags::alpha) },
{ "U_RGBA", static_cast<shader_components::flags>(point_cloud_shader_components::flags::uniform_color) },
{ "RAINBOW", static_cast<shader_components::flags>(point_cloud_shader_components::flags::rainbow) }
};
}
assets::glsl_parser::parser_context::parser_context(
store_type& m_store,
std::mutex& m_store_mutex
) : m_store{ &m_store }, m_store_mutex{ &m_store_mutex }
{
m_buffer.source.reserve(4096);
m_value_buffer.reserve(64);
m_declaration_value_count_buffer.reserve(8);
}
void assets::glsl_parser::parser_context::reset()
{
m_buffer.clear();
m_value_buffer.clear();
m_declaration_value_count_buffer.clear();
std::ranges::fill(
m_declaration_type_index_buffer,
static_cast<std::size_t>(language::declaration_type::invalid)
);
}
void assets::glsl_parser::parser_context::operator()(lookup_type::const_pointer entry) noexcept
{
const auto& [ filename, id ] = *entry;
if (const auto e = read_file(filename, m_buffer.source))
{
ztu::logger::warn("Could not load shader source %: %.", filename, e.message());
return;
}
tokenize_declarations();
if (not parse_metadata_from_tokens())
{
ztu::logger::warn("Ignoring shader % as it contains malformed metadata.", id);
return;
}
remove_metadata_declarations();
{
auto lock = std::lock_guard{ *m_store_mutex };
m_store->add(id, m_buffer);
}
}
void assets::glsl_parser::parser_context::tokenize_declarations()
{
auto source_rest = std::string_view( m_buffer.source.data(), m_buffer.source.size() );
auto offset = std::string_view::size_type{};
while ((offset = source_rest.find(language::declaration_prefix)) != std::string_view::npos)
{
const auto current_token_count = m_value_buffer.size();
auto line_end = source_rest.find('\n', offset);
if (line_end == std::string_view::npos)
{
line_end = source_rest.length();
}
auto declaration = source_rest.substr(offset, line_end - offset);
if ((offset = declaration.find(language::title_separator)) == std::string_view::npos)
{
continue;
}
const auto title = declaration.substr(0, offset);
if (const auto it = language::declaration_lookup.find(title); it != language::declaration_lookup.end())
{
const auto declaration_type = static_cast<std::size_t>(it->second);
m_declaration_type_index_buffer[declaration_type] = m_declaration_value_count_buffer.size();
}
else
{
continue;
}
declaration = declaration.substr(offset);
if (not declaration.empty() and declaration.front() == language::value_separator)
{
declaration = declaration.substr(sizeof(language::value_separator), declaration.length());
}
while ((offset = declaration.find(language::value_separator)) != std::string_view::npos)
{
m_value_buffer.emplace_back(declaration.substr(0, offset));
declaration = declaration.substr(offset + sizeof(language::value_separator));
}
if (not declaration.empty())
{
m_value_buffer.emplace_back(declaration);
}
m_declaration_value_count_buffer.emplace_back(
m_value_buffer.size() - current_token_count
);
// Preserve line break
source_rest = source_rest.substr(line_end);
}
}
bool assets::glsl_parser::parser_context::parse_metadata_from_tokens()
{
using namespace std::string_view_literals;
for (const auto [ type, name, parser ] :
{
std::make_tuple(language::declaration_type::stage, "stage"sv, &parse_stage_declaration),
std::make_tuple(language::declaration_type::geometry, "geometry"sv, &parse_geometry_declaration),
std::make_tuple(language::declaration_type::features, "features"sv, &parse_components_declaration),
std::make_tuple(language::declaration_type::static_enable, "static_enable"sv, &parse_static_enable_declaration),
std::make_tuple(language::declaration_type::dynamic_enable, "dynamic_enable"sv, &parse_dynamic_enable_declaration)
}
) {
const auto index = m_declaration_type_index_buffer[static_cast<std::size_t>(type)];
if (index == static_cast<std::size_t>(language::declaration_type::invalid))
{
ztu::logger::warn("Shader metadata error: Missing % declaration.", name);
return false;
}
const auto value_token_offset = std::accumulate(
m_declaration_value_count_buffer.begin(),
m_declaration_value_count_buffer.begin() + index,
std::size_t{}
);
const auto value_token_count = m_declaration_value_count_buffer[index];
if (not parser(std::span(m_value_buffer).subspan(value_token_offset, value_token_count), m_buffer))
{
return false;
}
}
return true;
}
void assets::glsl_parser::parser_context::remove_metadata_declarations()
{
if (m_declaration_value_count_buffer.empty())
{
return;
}
// 1. For every declaration expand the first value view to the size of the line.
// 2. Delete all other value views as they are not needed anymore.
// 3. Delete all declaration lines by overwriting them with the following source code.
// 4. Resize the source vector to the new character count.
const auto source_view = std::string_view{ m_buffer.source.data(), m_buffer.source.size() };
const auto value_span = std::span(m_value_buffer);
auto offset = std::size_t{};
for (auto [ index, count ] : std::ranges::views::enumerate(m_declaration_value_count_buffer))
{
const auto values = value_span.subspan(offset, count);
const auto value = values.front();
std::size_t begin_pos = value.data() - source_view.data();
std::size_t end_pos = begin_pos + value.size();
begin_pos = source_view.rfind('\n', begin_pos);
if (begin_pos == std::string_view::npos)
{
begin_pos = 0;
}
end_pos = source_view.find('\n', end_pos);
if (end_pos == std::string_view::npos)
{
end_pos = source_view.length();
}
else
{
end_pos += sizeof('\n');
}
m_value_buffer[index] = source_view.substr(begin_pos, end_pos - begin_pos);
offset += count;
}
const auto lines = std::span(m_value_buffer.data(), m_declaration_value_count_buffer.size());
const auto& first_line = lines.front();
auto source_end = first_line.begin();
auto line_end = first_line.end();
for (const auto& line : lines.subspan(1))
{
std::copy(line_end, line.begin(), source_end);
source_end += line.size();
line_end = line.end();
}
const auto source_rest_size = m_buffer.source.end().base() - line_end;
std::copy_n(line_end, source_rest_size, source_end);
source_end += source_rest_size;
m_buffer.source.resize(source_end - m_buffer.source.data());
}
std::error_code assets::glsl_parser::read_file(
const std::filesystem::path& filename,
std::vector<char>& source
) {
auto file = std::ifstream{ filename };
if (not file.is_open())
{
return std::make_error_code(std::errc::no_such_file_or_directory);
}
file.seekg(0, std::ios::end);
const std::streampos size = file.tellg();
if (size == 0 or size == std::numeric_limits<std::streampos>::max())
{
return std::make_error_code(std::errc::invalid_seek);
}
file.seekg(0, std::ios::beg);
source.clear();
source.reserve(sizeof('\n') + size);
source.push_back('\n');
std::copy(
std::istreambuf_iterator<char>(file),
std::istreambuf_iterator<char>(),
std::back_inserter(source)
);
file.close();
return {};
}
std::error_code assets::glsl_parser::prefetch(
path_id_lookups&
) {
// Nothing to prefetch
return {};
}
std::error_code assets::glsl_parser::load(
const path_id_lookups& lookups,
shader_source_store& store,
bool
) {
namespace fs = std::filesystem;
m_path_buffer.clear();
lookups.shader_sources.by_extension(".glsl", m_path_buffer);
auto store_mutex = std::mutex{};
std::for_each(
std::execution::parallel_unsequenced_policy{},
m_path_buffer.begin(),
m_path_buffer.end(),
parser_context{ store, store_mutex }
);
return {};
}
bool assets::glsl_parser::parse_geometry_declaration(
const std::span<const std::string_view> values,
model_geometry::types& geometry_type
) {
if (values.size() != 1)
{
ztu::logger::warn("Invalid geometry declaration: Expected exactly one token but got %.", values.size());
return false;
}
const auto value = values.front();
if (const auto it = language::geometry_lookup.find(value); it != language::geometry_lookup.end())
{
geometry_type = it->second;
}
else
{
ztu::logger::warn("Invalid geometry declaration: Unknown geometry %.", value);
return false;
}
return true;
}
bool assets::glsl_parser::parse_stage_declaration(
const std::span<const std::string_view> values,
shader_components::stage& stage
) {
if (values.size() != 1)
{
ztu::logger::warn("Invalid stage declaration: Expected exactly one token but got %.", values.size());
return false;
}
const auto value = values.front();
if (const auto it = language::stage_lookup.find(value); it != language::stage_lookup.end())
{
stage = it->second;
}
else
{
ztu::logger::warn("Invalid stage declaration: Unknown stage %.", value);
return false;
}
return true;
}
bool assets::glsl_parser::parse_components_declaration(
const std::span<const std::string_view> values,
shader_source_data& buffer
) {
return parse_component_tokens(values, buffer.geometry_type, buffer.components);
}
bool assets::glsl_parser::parse_static_enable_declaration(
const std::span<const std::string_view> values,
shader_source_data& buffer
) {
return parse_component_tokens(values, buffer.geometry_type, buffer.static_enable);
}
bool assets::glsl_parser::parse_dynamic_enable_declaration(
const std::span<const std::string_view> values,
shader_source_data& buffer
) {
return parse_component_tokens(values, buffer.geometry_type, buffer.dynamic_enable);
}
bool assets::glsl_parser::parse_component_tokens(
const std::span<const std::string_view> values,
const model_geometry::types geometry_type,
shader_components::flags& components
) {
components = {};
const ztu::string_lookup<shader_components::flags>* component_lookup{};
switch (geometry_type)
{
case model_geometry::types::mesh:
component_lookup = &language::mesh_feature_lookup;
break;
case model_geometry::types::point_cloud:
component_lookup = &language::point_cloud_feature_lookup;
break;
default:
ztu::logger::warn("Internal error: Unknown geometry index %.", static_cast<int>(geometry_type));
return false;
}
for (const auto value : values)
{
if (const auto it = component_lookup->find(value); it != component_lookup->end())
{
components |= it->second;
}
else
{
ztu::logger::warn("Ignoring unknown feature token %.", value);
}
}
return true;
}

View File

@@ -1,301 +0,0 @@
#include "../../../include/assets/data_parsers"
#include "glm/glm.hpp"
#include <charconv>
#include <fstream>
#include <glm/ext/matrix_transform.hpp>
#include "assets/components/point_cloud_vertex_components.hpp"
#include "util/binary_ifstream.hpp"
#include "util/logger.hpp"
ztu::result<std::string_view> kitti_loader::parent_directory(const std::string_view path)
{
const auto sep_index = path.rfind(std::filesystem::path::preferred_separator);
if (sep_index == std::string_view::npos)
{
return std::unexpected(std::make_error_code(std::errc::no_such_file_or_directory));
}
return path.substr(0, sep_index);
};
std::error_code kitti_loader::prefetch(
const file_dir_list& paths,
prefetch_queue& queue
) {
// Directories can simply be passed on
queue.kitti_pose_queue.directories.push_back(paths.directories);
// For files, we just forward the files directory
for (const auto file : queue.kitti_pose_queue.files)
{
if (const auto base_directory = parent_directory(file).and_then(parent_directory))
{
queue.kitti_pose_queue.directories.push_back(*base_directory);
}
else
{
// TODO remove from list
ztu::logger::error("Malformed kitti file path: %.", file);
}
}
return {};
}
std::error_code kitti_loader::load(
dynamic_point_cloud_buffer& buffer,
const file_dir_list& paths,
prefetch_lookup& id_lookup,
dynamic_shader_source_store& store,
bool
) {
namespace fs = std::filesystem;
std::error_code error;
std::vector<dynamic_pose_store::iterator_type> pose_its;
pose_its.reserve(paths.files.size());
auto processed_filenames = ztu::string_list{};
auto path_buffer = fs::path{};
const auto preprocess_filename = [&](
std::string_view path,
const auto& directory,
std::string_view filename,
const pose_prefetch_lookup::directory_iterator& dir_it
) {
const auto pose_index = frame_id_from_filename(filename);
if (not pose_index) [[unlikely]]
{
ztu::logger::error("Could not parse frame id from kitti file path: %.", filename);
return;
}
const auto [ index_it, pose_id_match ] = id_lookup.poses.find_index(dir_it, *pose_index);
if (not pose_id_match) [[unlikely]]
{
ztu::logger::error("No matching pose index (%) found in directory (%).", directory, *pose_index);
return;
}
const auto [ pose_it, pose_match ] = store.poses.find(pose_id_match);
if (not pose_id_match) [[unlikely]]
{
ztu::logger::error("No matching pose found for id: %.", pose_id_match);
return;
}
processed_filenames.push_back(path);
pose_its.push_back(pose_it);
};
for (const auto file : paths.files)
{
path_buffer.assign(file.begin(), file.end());
if (not fs::is_regular_file(path_buffer))
{
ztu::logger::error("Given kitti file does not exist: %.", path_buffer);
continue;
}
const auto sep_index = file.rfind(fs::path::preferred_separator);
if (sep_index == std::string_view::npos) [[unlikely]]
{
ztu::logger::error("Could not parse frame directory from kitti file path: %.", file);
continue;
}
// TODO find out if the path splitting is consistent
const auto base_directory = parent_directory(file.substr(0, sep_index));
if (not base_directory) [[unlikely]]
{
ztu::logger::error("Could not parse base directory from kitti file path: %.", file);
continue;
}
const auto [ dir_it, dir_match ] = id_lookup.poses.find_directory(*base_directory);
if (not dir_match) [[unlikely]]
{
ztu::logger::error("No matching pose directory found for %.", file);
continue;
}
const auto filename = file.substr(sep_index + 1);
preprocess_filename(
file,
*base_directory,
filename,
dir_it
);
}
for (const auto directory : paths.directories)
{
path_buffer.assign(directory.begin(), directory.end());
const auto [ dir_it, dir_match ] = id_lookup.poses.find_directory(path_buffer);
if (not dir_match) [[unlikely]]
{
ztu::logger::error("No matching pose directory found for %.", path_buffer);
continue;
}
path_buffer /= frame_folder;
if (not fs::is_directory(path_buffer))
{
ztu::logger::error("Given kitti directory does not exist: %.", directory);
continue;
}
for (const auto& file : fs::directory_iterator{ path_buffer })
{
const auto file_path = std::string_view{ file.path().c_str() };
if (not file_path.ends_with(".bin"))
{
continue;
}
auto filename_begin = file_path.rfind(fs::path::preferred_separator);
filename_begin = filename_begin == std::string_view::npos ? 0 : filename_begin + 1;
const auto filename_only = file_path.substr(filename_begin);
const auto pose_index = frame_id_from_filename(filename_only);
if (not pose_index) [[unlikely]]
{
ztu::logger::error("Could not parse frame id from kitti filename: %.", filename_only);
continue;
}
const auto [ index_it, pose_id_match ] = id_lookup.poses.find_index(dir_it, *pose_index);
if (not pose_id_match) [[unlikely]]
{
ztu::logger::error("No matching pose index (%) found in directory (%).", directory, *pose_index);
continue;
}
const auto [ pose_it, pose_match ] = store.poses.find(pose_id_match);
if (not pose_id_match) [[unlikely]]
{
ztu::logger::error("No matching pose found for id: %.", pose_id_match);
continue;
}
processed_filenames.push_back(file_path);
pose_its.push_back(pose_it);
preprocess_filename(
file_path,
directory,
filename_only,
dir_it
);
}
}
for (const auto [ filename, pose_it ] : std::ranges::views::zip(processed_filenames, pose_its))
{
buffer.clear();
if ((error = load_point_file(filename, buffer)))
{
return error;
}
transform_point_cloud(buffer.positions(), *pose_it);
store.point_clouds.add(buffer);
}
return {};
}
void kitti_loader::transform_point_cloud(
std::span<point_cloud_vertex_components::position> points,
const glm::mat4& pose
) {
for (auto& [ x, y, z ] : points) {
auto vec = glm::vec4{ x, y, z, 1.0f };
vec = pose * vec;
x = vec.x;
y = vec.y;
z = vec.z;
}
}
std::error_code kitti_loader::load_point_file(
const std::filesystem::path& filename,
dynamic_point_cloud_buffer& point_cloud
) {
auto in = binary_ifstream{};
auto error = std::error_code{};
if ((error == in.open(filename, true)))
{
return error;
}
const auto read_vector = [&in](auto& vector) -> std::error_code
{
for (auto& component : vector)
{
float component32;
if (const auto e = in.read_ieee754<std::endian::little>(component32))
{
return e;
}
component = component32;
}
return {};
};
point_cloud_vertex_components::position position;
auto& positions = point_cloud.positions();
while (not ((error = read_vector(position)))) {
positions.push_back(position);
if ((error = in.skip<float>())) // TODO what am I skipping here?!?
{
break;
}
}
if (static_cast<std::errc>(error.value()) != std::errc::result_out_of_range)
{
return error;
}
return {};
}
ztu::result<std::size_t> kitti_loader::frame_id_from_filename(
std::string_view filename
) {
std::size_t id;
const auto result = std::from_chars(filename.cbegin(), filename.cend(), id);
if (result.ec != std::errc{})
{
return std::unexpected(std::make_error_code(result.ec));
}
return id;
}

View File

@@ -0,0 +1,239 @@
#include "assets/file_parsers/kitti_parser.hpp"
#include "glm/glm.hpp"
#include <charconv>
#include <fstream>
#include <glm/ext/matrix_transform.hpp>
#include "assets/components/point_cloud_vertex_components.hpp"
#include "assets/data/pose_data.hpp"
#include "util/binary_ifstream.hpp"
#include "util/logger.hpp"
#include <execution>
assets::kitti_parser::parser_context::parser_context(
const pose_list_id_lookup& pose_list_lookup,
const pose_list_store& pose_list_store,
store_type& m_store,
std::mutex& m_store_mutex
) :
m_pose_list_lookup{ &pose_list_lookup },
m_pose_list_store{ &pose_list_store },
m_store{ &m_store },
m_store_mutex{ &m_store_mutex }
{
m_buffer.positions().reserve(8192);
m_buffer.normals().reserve(8192);
m_buffer.colors().reserve(8192);
}
void assets::kitti_parser::parser_context::reset()
{
m_buffer.clear();
}
void assets::kitti_parser::parser_context::operator()(lookup_type::const_pointer entry) noexcept
{
const auto& [ filename, id ] = *entry;
pose_data pose;
if (const auto pose_path = get_pose_path(filename); not pose_path)
{
if (pose_path != m_last_pose_path)
{
if (const auto pose_list_id_it = m_pose_list_lookup->find(*pose_path); pose_list_id_it != m_pose_list_lookup->end())
{
m_last_pose_path = *pose_path;
const auto pose_list_id = pose_list_id_it->second;
if (const auto [ pose_list_it, found ] = m_pose_list_store->find(pose_list_id); found)
{
m_last_pose_path = *pose_path;
m_last_pose_list = pose_list_it->second;
}
else
{
ztu::logger::error("No matching pose found in store for %.", *pose_path);
return;
}
}
else
{
ztu::logger::error("No matching pose registered in lookup %.", *pose_path);
return;
}
}
}
else
{
ztu::logger::error("Malformed kitti file path %.", filename);
return;
}
clear();
if (const auto e = load_point_file(filename, m_buffer))
{
ztu::logger::error("Could not load kitti file %: %", filename, e.message());
return;
}
transform_point_cloud(m_buffer.positions(), pose);
{
auto lock = std::lock_guard{ *m_store_mutex };
m_store->add(id, m_buffer);
}
}
ztu::result<std::filesystem::path> assets::kitti_parser::parent_directory(
const std::filesystem::path& path
) {
try
{
return path.parent_path();
}
catch (const std::exception&)
{
return std::unexpected(std::make_error_code(std::errc::no_such_file_or_directory));
}
};
ztu::result<std::filesystem::path> assets::kitti_parser::get_pose_path(
const std::filesystem::path& path
) {
return parent_directory(path)
.and_then(parent_directory)
.and_then(
[](const auto& base_dir)
{
return base_dir / "pose.txt";
}
);
};
std::error_code assets::kitti_parser::prefetch(
path_id_lookups& lookups
) {
m_path_buffer.clear();
lookups.point_clouds.by_extension(".bin", m_path_buffer);
for (const auto entry : m_path_buffer)
{
if (const auto pose_path = pose_path(entry->second))
{
lookups.poses.try_emplace(std::filesystem::absolute(pose_path));
}
else
{
ztu::logger::error("Malformed kitti file path %.", entry->second);
}
}
return {};
}
std::error_code assets::kitti_parser::load(
path_id_lookups& lookups,
data_stores& stores,
bool pedantic
) {
namespace fs = std::filesystem;
m_path_buffer.clear();
lookups.point_clouds.by_extension(".bin", m_path_buffer);
auto store_mutex = std::mutex{};
std::for_each(
std::execution::parallel_unsequenced_policy{},
m_path_buffer.begin(),
m_path_buffer.end(),
parser_context{
lookups.pose_lists,
stores.pose_lists,
stores.point_clouds,
store_mutex
}
);
return {};
}
void assets::kitti_parser::transform_point_cloud(
std::span<point_cloud_vertex_components::position> points,
const glm::mat4& pose
) {
for (auto& point : points)
{
point = pose * glm::vec4{ point, 1.0f };
}
}
std::error_code assets::kitti_parser::load_point_file(
const std::filesystem::path& filename,
point_cloud_data& point_cloud
) {
auto in = binary_ifstream{};
if (const auto e = in.open(filename, true))
{
return e;
}
const auto read_vector = [&in](auto& vector) -> std::error_code
{
for (auto& component : vector)
{
float component32;
if (const auto e = in.read_ieee754<std::endian::little>(component32))
{
return e;
}
component = component32;
}
return {};
};
point_cloud_vertex_components::position position;
auto& positions = point_cloud.positions();
auto parsing_error = std::error_code{};
while (not ((parsing_error = read_vector(position))))
{
positions.push_back(position);
if ((parsing_error = in.skip<float>())) // TODO what am I skipping here?!?
{
break;
}
}
if (static_cast<std::errc>(parsing_error.value()) != std::errc::result_out_of_range)
{
return parsing_error;
}
return {};
}
ztu::result<std::size_t> assets::kitti_parser::frame_id_from_filename(
const std::string_view filename
) {
std::size_t id;
const auto result = std::from_chars(filename.cbegin(), filename.cend(), id);
if (result.ec != std::errc{})
{
return std::unexpected(std::make_error_code(result.ec));
}
return id;
}

View File

@@ -1,120 +0,0 @@
#include "../../../include/assets/data_parsers"
#include "../../../include/assets/read_buffers"
#include <fstream>
#include <glm/ext/matrix_transform.hpp>
#include "util/logger.hpp"
inline std::error_code kitti_pose_loader::parse_pose(
std::ifstream& in,
dynamic_pose_buffer& pose
) {
for (dynamic_pose_buffer::length_type row{}; row != 3; ++row)
{
for (dynamic_pose_buffer::length_type col{}; col != 4; ++col)
{
if (not (in >> pose[row][col]))
{
return std::make_error_code(std::errc::result_out_of_range);
}
}
}
return {};
}
std::error_code kitti_pose_loader::prefetch(
const file_dir_list& paths,
prefetch_queue& queue
) {
// Nothing to be done here
}
std::error_code kitti_pose_loader::load(
dynamic_pose_buffer& buffer,
const file_dir_list& paths,
prefetch_lookup& id_lookup,
dynamic_shader_source_store& store,
bool pedantic
) {
namespace fs = std::filesystem;
auto path_buffer = fs::path{};
auto in = std::ifstream{}; // TODO disable exceptions (for other loaders as well)
// Needed to initialize 4th row and col
buffer = glm::identity<glm::mat4>();
auto processed_filenames = ztu::string_list{};
processed_filenames.reserve(
paths.files.character_count() +
paths.directories.character_count() +
paths.directories.size() * pose_filename.size(),
paths.files.size() + paths.directories.size()
);
const auto preprocess_file = [&]()
{
if (not fs::is_regular_file(path_buffer))
{
ztu::logger::error("Kitti pose file does not exist: %", path_buffer);
return;
}
processed_filenames.push_back(path_buffer.c_str());
};
for (const auto directory : paths.directories) {
path_buffer.assign(directory.begin(), directory.end());
path_buffer /= pose_filename;
preprocess_file();
}
for (const auto file : paths.files) {
path_buffer.assign(file.begin(), file.end());
preprocess_file();
}
for (const auto filename : processed_filenames)
{
// TODO if (not) removing the path separator creates issues.
const auto directory = filename.substr(0, filename.length() - pose_filename.length());
auto [ dir_it, dir_match ] = id_lookup.poses.find_directory(directory);
if (not dir_match) [[unlikely]]
{
dir_it = id_lookup.poses.emplace_dir(dir_it, directory);
}
in.open(filename.data()); // Safe because string list adds null terminator
if (not in.is_open())
{
ztu::logger::error("Cannot open kitti pose file %", path_buffer);
continue;
}
in >> std::skipws;
for (auto i = pose_prefetch_lookup::index_type{}; in.peek() != std::ifstream::traits_type::eof(); ++i)
{
if (const auto error = parse_pose(in, buffer))
{
ztu::logger::error(
"Error occurred while parsing kitti pose % in file %: [%] %",
i,
path_buffer,
error.category().name(),
error.message()
);
continue;
}
const auto id = store.poses.add(buffer);
id_lookup.poses.emplace_hint_dir(dir_it, i, id);
}
in.close();
}
}

View File

@@ -0,0 +1,125 @@
#include "assets/file_parsers/kitti_pose_parser.hpp"
#include <fstream>
#include <execution>
#include <glm/ext/matrix_transform.hpp>
#include "util/logger.hpp"
assets::kitti_pose_parser::parser_context::parser_context(
store_type& m_store,
std::mutex& m_store_mutex
) : m_store{ &m_store }, m_store_mutex{ &m_store_mutex }
{
m_buffer.reserve(128);
}
void assets::kitti_pose_parser::parser_context::reset()
{
m_buffer.clear();
}
void assets::kitti_pose_parser::parser_context::operator()(lookup_type::const_pointer entry) noexcept
{
const auto& [ filename, id ] = *entry;
if (const auto e = parse_file(filename, m_buffer))
{
ztu::logger::warn("Could not load pose_list %: %.", filename, e.message());
return;
}
{
auto lock = std::lock_guard{ *m_store_mutex };
m_store->add(id, m_buffer);
}
}
std::error_code assets::kitti_pose_parser::prefetch(
path_id_lookups& lookups
) {
return {};
}
std::error_code assets::kitti_pose_parser::parse(
const path_id_lookups& lookups,
store_type& store,
bool
) {
namespace fs = std::filesystem;
auto path_buffer = fs::path{};
auto in = std::ifstream{}; // TODO disable exceptions (for other loaders as well)
m_path_buffer.clear();
lookups.pose_lists.by_extension(".glsl", m_path_buffer);
auto store_mutex = std::mutex{};
std::for_each(
std::execution::parallel_unsequenced_policy{},
m_path_buffer.begin(),
m_path_buffer.end(),
parser_context{ store, store_mutex }
);
return {};
}
std::error_code assets::kitti_pose_parser::parse_file(
const std::filesystem::path& filename,
data_type &poses
) {
auto in = std::ifstream{};
in.open(filename);
if (not in.is_open())
{
return std::make_error_code(std::errc::no_such_file_or_directory);
}
in >> std::skipws;
while (in.peek() != std::ifstream::traits_type::eof())
{
auto pose = glm::identity<pose_data>();
if (const auto error = parse_pose(in, pose))
{
ztu::logger::error(
"Error occurred while parsing kitti pose file %: [%] %",
filename,
error.category().name(),
error.message()
);
continue;
}
poses.emplace_back(pose);
}
in.close();
return {};
}
std::error_code assets::kitti_pose_parser::parse_pose(
std::ifstream& in,
pose_data& pose
) {
for (pose_data::length_type row{}; row != 3; ++row)
{
for (pose_data::length_type col{}; col != 4; ++col)
{
if (not (in >> pose[row][col]))
{
return std::make_error_code(std::errc::result_out_of_range);
}
}
}
return {};
}

View File

@@ -1,4 +1,4 @@
#include "../../../include/assets/data_parsers"
#include "assets/file_parsers/mtl_loader.hpp"
#include <charconv>
#include <fstream>
@@ -435,15 +435,14 @@ std::error_code mtl_loader::load(
}),
ztu::make_line_parser("Ka ", ztu::is_not_repeating, [&](const auto& param)
{
if (surface_properties_disabled) return codes::ok;
auto& properties = material.initialized_surface_properties();
if (parse_numeric_vector(param, properties.ambient_filter) != std::errc{}) [[unlikely]]
material_components::ambient_filter ambient_filter;
if (parse_numeric_vector(param, ambient_filter) != std::errc{}) [[unlikely]]
{
return codes::malformed_ambient_color;
return codes::malformed_ambient_color; // TODO rename to filter
}
material.components() |= flags::surface_properties;
material.ambient_filter = mbient_filter;
material.components() |= flags::ambient_filter;
return codes::ok;
}),
@@ -511,7 +510,7 @@ std::error_code mtl_loader::load(
if (textures_disabled) return codes::ok;
load_texture(param, "ambient color", [&](const auto id) {
material.ambient_color_texture_id() = id;
material.specular_filter_texture_id() = id;
material.components() |= flags::ambient_filter_texture;
});
@@ -522,7 +521,7 @@ std::error_code mtl_loader::load(
if (textures_disabled) return codes::ok;
load_texture(param, "diffuse color", [&](const auto id) {
material.diffuse_color_texture_id() = id;
material.diffuse_filter_texture_id() = id;
material.components() |= flags::diffuse_filter_texture;
});
@@ -533,7 +532,7 @@ std::error_code mtl_loader::load(
if (textures_disabled) return codes::ok;
load_texture(param, "specular color", [&](const auto id) {
material.specular_color_texture_id() = id;
material.specular_filter_texture_id() = id;
material.components() |= flags::specular_filter_texture;
});

View File

@@ -1,11 +1,11 @@
#include "../../../include/assets/data_parsers"
#include "assets/file_parsers/obj_loader.hpp"
#include <charconv>
#include <fstream>
#include <array>
#include "assets/components/mesh_vertex_components.hpp"
#include "../../../include/assets/data_loaders"
#include "assets/data_loaders/"
#include "util/logger.hpp"
#include "util/for_each.hpp"

View File

@@ -1,4 +1,4 @@
#include "../../../include/assets/data_parsers"
#include "assets/file_parsers/stl_loader.hpp"
#include "util/binary_ifstream.hpp"
#include "util/unroll_bool_template.hpp"

View File

@@ -1,13 +1,14 @@
#include "../../../include/assets/data_parsers"
#include "assets/file_parsers/threedtk_pose_loader.hpp"
#include "../../../include/assets/read_buffers"
#include <fstream>
#include <glm/ext/matrix_transform.hpp>
#include <glm/gtx/euler_angles.hpp>
#include "util/logger.hpp"
#include <fstream>
inline std::error_code threedtk_pose_loader::parse_transform_info(