zy4n/Z3D
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finally refactored all stores!

Browse Source
This commit is contained in:
zy4n
2025-04-02 17:45:41 +02:00
parent 27977c1738
commit 835c645da4
11 changed files with 469 additions and 648 deletions
Download Patch File Download Diff File Expand all files Collapse all files

413
source/assets/file_parsers/generic/generic_3dtk_loader.ipp
View File

@@ -14,11 +14,11 @@
template<bool Normal, bool Color, bool Reflectance>
std::error_code assets::generic_3dtk_loader<Normal, Color, Reflectance>::prefetch(
std::error_code assets::detail::generic_3dtk_loader<Normal, Color, Reflectance>::prefetch(
path_id_lookups& lookups
) {
m_path_buffer.clear();
lookups.meshes.by_extension(".3d", m_path_buffer);
lookups.point_clouds.by_extension(".3d", m_path_buffer);
auto path_buffer = std::filesystem::path{};
@@ -33,7 +33,7 @@ std::error_code assets::generic_3dtk_loader<Normal, Color, Reflectance>::prefetc
}
template<bool Normal, bool Color, bool Reflectance>
std::error_code assets::generic_3dtk_loader<Normal, Color, Reflectance>::load(
std::error_code assets::detail::generic_3dtk_loader<Normal, Color, Reflectance>::load(
path_id_lookups& lookups,
data_stores& stores,
bool pedantic
@@ -55,7 +55,7 @@ std::error_code assets::generic_3dtk_loader<Normal, Color, Reflectance>::load(
}
template<bool Normal, bool Color, bool Reflectance>
assets::generic_3dtk_loader<Normal, Color, Reflectance>::parser_context::parser_context(
assets::detail::generic_3dtk_loader<Normal, Color, Reflectance>::parser_context::parser_context(
path_id_lookups& m_id_lookups,
data_stores& m_stores
) :
@@ -63,47 +63,28 @@ assets::generic_3dtk_loader<Normal, Color, Reflectance>::parser_context::parser_
m_stores{ &m_stores }
{
constexpr auto expected_vertex_count = 8192;
m_mesh.positions().reserve(expected_vertex_count);
m_mesh.normals().reserve(expected_vertex_count);
m_mesh.colors().reserve(expected_vertex_count);
m_mesh.reflectances().reserve(expected_vertex_count);
m_mesh.tex_coords().reserve(expected_vertex_count);
m_mesh.triangles().reserve(2 * expected_vertex_count);
m_position_buffer.reserve(expected_vertex_count);
m_normal_buffer.reserve(expected_vertex_count);
m_tex_coord_buffer.reserve(expected_vertex_count);
m_point_cloud.positions().reserve(expected_vertex_count);
m_point_cloud.normals().reserve(expected_vertex_count);
m_point_cloud.colors().reserve(expected_vertex_count);
m_point_cloud.reflectances().reserve(expected_vertex_count);
}
template<bool Normal, bool Color, bool Reflectance>
void assets::generic_3dtk_loader<Normal, Color, Reflectance>::parser_context::reset()
void assets::detail::generic_3dtk_loader<Normal, Color, Reflectance>::parser_context::reset()
{
m_mesh.clear();
m_position_buffer.clear();
m_normal_buffer.clear();
m_tex_coord_buffer.clear();
m_vertex_comp_indices_to_vertex_index.clear();
m_point_cloud.clear();
}
template<bool Normal, bool Color, bool Reflectance>
void assets::generic_3dtk_loader<Normal, Color, Reflectance>::parser_context::operator()(lookup_type::const_pointer entry) noexcept
void assets::detail::generic_3dtk_loader<Normal, Color, Reflectance>::parser_context::operator()(lookup_type::const_pointer entry) noexcept
{
// TODO look up pose
const auto& [ filename, id ] = *entry;
auto in = std::ifstream{ filename };
if (not in.is_open())
std::size_t pose_index;
if (auto res = parse_scan_index(filename))
{
ztu::logger::warn("Cannot open obj file %.", filename);
return;
}
std::size_t scan_index;
if (auto res = parse_index(filename))
{
scan_index = *res;
pose_index = *res;
}
else [[unlikely]]
{
@@ -113,40 +94,77 @@ void assets::generic_3dtk_loader<Normal, Color, Reflectance>::parser_context::op
error.category().name(),
error.message()
);
}
auto pose = asset_lookup.poses.find();
const auto id_it = id_lookup.find(filename);
if (id_it != id_lookup.end()) [[unlikely]]
{
return;
}
in.open(filename);
if (in.is_open())
auto pose_path = entry->first;
pose_path.replace_extension(".pose");
if (pose_path != m_last_pose_path)
{
if ((error = this->read_point_file(filename, buffer))) {
return error;
if (const auto pose_list_id_it = m_id_lookups->pose_lists.find(pose_path); pose_list_id_it != m_id_lookups->pose_lists.end())
{
m_last_pose_path = pose_path;
const auto pose_list_id = pose_list_id_it->second;
// TODO What if the netry is not there we look it pup again and again
// TODO Insert maybe a bool was_valid
if (const auto [ pose_list_it, found ] = m_id_lookups->poses.find(pose_list_id); found)
{
m_last_pose_list = pose_list_it->second;
}
else
{
m_last_pose_list = {};
ztu::logger::error("No matching pose found in store for %.", pose_path);
return;
}
}
else
{
ztu::logger::error("No matching pose registered in lookup %.", pose_path);
return;
}
}
else if (m_last_pose_list.data() == nullptr)
{
ztu::logger::error("No matching pose found in store for %.", pose_path);
return;
}
this->transform_point_cloud(buffer.positions(), pose);
reset();
const auto id = store.add(std::move(buffer));
id_lookup.emplace_hint(id_it, filename, id);
pose_data pose;
if (pose_index < m_last_pose_list.size())
{
pose = m_last_pose_list[pose_index];
}
else
{
ztu::logger::error("Cannot open 3dtk file %", filename);
ztu::logger::error(
"Pose index % of 3dtk file % is out of range for its pose file %. Proceeding with identity pose.",
pose_index,
filename,
m_last_pose_path
);
pose = glm::identity<pose_data>();
}
in.close();
if (const auto e = parse_file(filename, m_point_cloud))
{
ztu::logger::error("Could not parse 3dtk file %: %", filename, e.message());
return;
}
transform_point_cloud(m_point_cloud.positions(), pose);
m_stores->point_clouds.insert(id, m_point_cloud);
}
template<bool Normal, bool Color, bool Reflectance>
ztu::result<pose_prefetch_lookup::index_type> generic_3dtk_loader<Normal, Color, Reflectance>::parse_index(
ztu::result<std::size_t> assets::detail::generic_3dtk_loader<Normal, Color, Reflectance>::parse_scan_index(
const std::string_view filename
) {
static constexpr auto prefix = std::string_view{ "scan" };
@@ -155,164 +173,26 @@ ztu::result<pose_prefetch_lookup::index_type> generic_3dtk_loader<Normal, Color,
if (name_view.length() <= prefix.length()) [[unlikely]]
{
return std::make_error_code(std::errc::invalid_argument);
return std::unexpected{ std::make_error_code(std::errc::invalid_argument) };
}
name_view = name_view.substr(prefix.length());
pose_prefetch_lookup::index_type index;
std::size_t index;
const auto res = std::from_chars(name_view.begin(), name_view.end(), index);
if (res.ec != std::errc{}) [[unlikely]]
{
return std::make_error_code(res.ec);
return std::unexpected{ std::make_error_code(res.ec) };
}
return index;
}
template<bool Normal, bool Color, bool Reflectance>
std::error_code assets::generic_3dtk_loader<Normal, Color, Reflectance>::load(
path_id_lookups& lookups,
data_stores& stores,
bool pedantic = false
) {
namespace fs = std::filesystem;
auto in = std::ifstream{};
auto path_buffer = fs::path{};
auto error = std::error_code{};
const auto load_file = [&](const char* filename)
{
// TODO look up pose
auto scan_index = pose_prefetch_lookup::index_type{};
if (auto res = parse_index(filename))
{
scan_index = *res;
}
else [[unlikely]]
{
error = res.error();
ztu::logger::error(
"Error occurred while parsing scan index in filename %: [%] %",
error.category().name(),
error.message()
);
}
auto pose = asset_lookup.poses.find();
const auto id_it = id_lookup.find(filename);
if (id_it != id_lookup.end()) [[unlikely]]
{
return;
}
in.open(filename);
if (in.is_open())
{
if ((error = this->read_point_file(filename, buffer))) {
return error;
}
this->transform_point_cloud(buffer.positions(), pose);
const auto id = store.add(std::move(buffer));
id_lookup.emplace_hint(id_it, filename, id);
}
else
{
ztu::logger::error("Cannot open 3dtk file %", filename);
}
in.close();
};
for (const auto filename : paths.files)
{
load_file(filename.data());
}
for (const auto directory : paths.directories)
{
directory_buffer.assign(directory.begin(), directory.end());
directory_buffer /= "frames";
const auto directory_exists = not fs::is_directory(directory_buffer, error);
if (error or not directory_exists) [[unlikely]]
{
ztu::logger::error("Could not open point cloud directory %", directory_buffer);
continue;
}
for (const auto& filename : fs::directory_iterator{ directory_buffer }) {
auto point_filename = reinterpret_cast<const fs::path&>(filename);
if (point_filename.extension() != ".3d")
{
continue;
}
load_file(filename.c_str());
}
}
return {};
}
template<bool Normal, bool Color, bool Reflectance>
std::error_code generic_3dtk_loader<Normal, Color, Reflectance>::load_directory(
dynamic_data_loader_ctx& ctx,
dynamic_point_cloud_store& store,
const std::filesystem::path& path,
const bool pedantic
) {
namespace fs = std::filesystem;
std::error_code error;
const auto directory_exists = not fs::is_directory(path, error);
if (error)
{
return error;
}
if (not directory_exists)
{
return make_error_code(std::errc::no_such_file_or_directory);
}
for (const auto& filename : fs::directory_iterator{ path / "frames" }) {
auto point_filename = reinterpret_cast<const fs::path&>(filename);
if (point_filename.extension() != ".3d") {
continue;
}
if ((error = load(ctx, store, point_filename, pedantic)))
{
ztu::logger::error(
"Error while loading point cloud '%': [%] %",
point_filename,
error.category().name(),
error.message()
);
}
}
return {};
}
template<typename T, std::size_t Count>
std::error_code read_vector(std::string_view& line, std::array<T, Count>& vec) {
for (auto& component : vec)
template<int L>
std::error_code read_vector(std::string_view& line, z3d::vec<L, float>& vec) {
for (int i{}; i != L; ++i)
{
auto it = line.begin();
@@ -321,7 +201,7 @@ std::error_code read_vector(std::string_view& line, std::array<T, Count>& vec) {
const auto [ ptr, ec ] = std::from_chars(
it, line.end(),
component,
vec[i],
std::chars_format::hex
);
@@ -330,8 +210,9 @@ std::error_code read_vector(std::string_view& line, std::array<T, Count>& vec) {
return std::make_error_code(ec);
}
if (minus) {
component *= -1.0;
if (minus)
{
vec[i] *= -1.0;
}
line = { ptr + sizeof(' '), line.end() };
@@ -341,15 +222,16 @@ std::error_code read_vector(std::string_view& line, std::array<T, Count>& vec) {
}
template<bool Normal, bool Color, bool Reflectance>
std::error_code generic_3dtk_loader<Normal, Color, Reflectance>::read_point_file(
std::error_code assets::detail::generic_3dtk_loader<Normal, Color, Reflectance>::parse_file(
const std::filesystem::path& filename,
dynamic_point_cloud_data& point_cloud
data_type& point_cloud
) {
std::error_code error;
auto in = std::ifstream(filename);
if (not in.is_open()) {
if (not in.is_open())
{
return std::make_error_code(static_cast<std::errc>(errno));
}
@@ -381,18 +263,6 @@ std::error_code generic_3dtk_loader<Normal, Color, Reflectance>::read_point_file
return count;
}();
ztu::u32 component_count;
std::chars_format float_format;
if ((error = analyze_component_format(line, component_count, float_format)))
{
return error;
}
if (component_count != expected_component_count)
{
return std::make_error_code(std::errc::invalid_argument);
}
auto& positions = point_cloud.positions();
auto& normals = point_cloud.normals();
@@ -432,12 +302,12 @@ std::error_code generic_3dtk_loader<Normal, Color, Reflectance>::read_point_file
if constexpr (Reflectance)
{
point_cloud_vertex_components::reflectance reflectance;
z3d::vec<1, point_cloud_vertex_components::reflectance> reflectance;
if ((error = read_vector(line_view, reflectance)))
{
return error;
}
reflectances.push_back(reflectance);
reflectances.push_back(reflectance[0]);
}
}
while (std::getline(in, line));
@@ -445,110 +315,13 @@ std::error_code generic_3dtk_loader<Normal, Color, Reflectance>::read_point_file
return {};
}
std::error_code base_3dtk_loader::read_pose_file(
const std::filesystem::path& filename,
glm::mat4& pose
template<bool Normal, bool Color, bool Reflectance>
void assets::detail::generic_3dtk_loader<Normal, Color, Reflectance>::transform_point_cloud(
std::span<point_cloud_vertex_components::position> points,
const pose_data& pose
) {
auto in = std::ifstream(filename);
if (not in.is_open()) {
return std::make_error_code(static_cast<std::errc>(errno));
}
std::string line;
std::array<glm::vec3, 2> numbers{};
for (std::size_t row{}; row != 2; ++row) {
std::getline(in, line);
auto it = line.cbegin().base();
auto end = line.cend().base();
for (glm::vec3::length_type col{}; col != 3; ++col) {
const auto [ ptr, ec ] = std::from_chars(
it, end,
numbers[row][col],
std::chars_format::general
);
if (ec != std::errc{}) {
return std::make_error_code(ec);
}
it = ptr + 1; // skip space in between components
}
}
const auto& translation = numbers[0];
auto& angles = numbers[1];
angles *= static_cast<float>(M_PI / 180.0);
pose = (
glm::translate(glm::identity<glm::mat4>(), translation) *
glm::eulerAngleXYZ(angles[0], angles[1], angles[2])
);
return {};
}
std::error_code base_3dtk_loader::analyze_component_format(
const std::string& line,
ztu::u32& component_count,
std::chars_format& format
) {
auto begin = line.cbegin().base();
auto end = line.cend().base();
format = std::chars_format::general;
component_count = 0;
float buffer;
for (auto it = begin; it < end; it += sizeof(' '))
for (auto& point : points)
{
it += *it == '-' or *it == '+';
std::chars_format current_format;
if (*it == '0' and std::next(it) < end and *std::next(it) == 'x')
{
it += 2; // skip '0x'
current_format = std::chars_format::hex;
}
else
{
current_format = std::chars_format::general;
}
if (it == begin and current_format != format)
{
return std::make_error_code(std::errc::invalid_argument);
}
const auto [next_it, err] = std::from_chars(it, end, buffer, current_format);
if (err != std::errc())
{
return std::make_error_code(err);
}
it = next_it;
format = current_format;
++component_count;
}
return {};
}
void base_3dtk_loader::transform_point_cloud(
std::span<point_cloud_vertex_components::position::value_type> points,
const glm::mat4& pose
) {
for (auto& [ x, y, z ] : points) {
auto vec = glm::vec4{ x, y, z, 1.0f };
vec = pose * vec;
x = vec.x;
y = vec.y;
z = vec.z;
point = pose * glm::vec4{ point, 1.0f };
}
}

62
source/assets/file_parsers/kitti_parser.cpp
View File

@@ -14,13 +14,11 @@
assets::kitti_parser::parser_context::parser_context(
const pose_list_id_lookup& pose_list_lookup,
const pose_list_store& pose_list_store,
store_type& m_store
path_id_lookups& pose_list_lookup,
data_stores& stores
) :
m_pose_list_lookup{ &pose_list_lookup },
m_pose_list_store{ &pose_list_store },
m_store{ &m_store }
m_stores{ &stores }
{
constexpr auto expected_vertex_count = 8192;
m_buffer.positions().reserve(expected_vertex_count);
@@ -37,20 +35,28 @@ void assets::kitti_parser::parser_context::operator()(lookup_type::const_pointer
{
const auto& [ filename, id ] = *entry;
pose_data pose;
if (const auto pose_path = get_pose_path(filename); not pose_path)
std::size_t pose_index;
if (const auto res = frame_id_from_filename(filename.c_str()))
{
if (pose_path != m_last_pose_path)
pose_index = res.value;
}
else
{
ztu::logger::error("Could not parse frame id from kitti filename %: %", filename, res.error());
return;
}
if (const auto pose_path = get_pose_path(filename))
{
if (*pose_path != m_last_pose_path)
{
if (const auto pose_list_id_it = m_pose_list_lookup->find(*pose_path); pose_list_id_it != m_pose_list_lookup->end())
if (const auto pose_list_id_it = m_pose_list_lookup->pose_lists.find(*pose_path); pose_list_id_it != m_pose_list_lookup->pose_lists.end())
{
m_last_pose_path = *pose_path;
const auto pose_list_id = pose_list_id_it->second;
if (const auto [ pose_list_it, found ] = m_pose_list_store->find(pose_list_id); found)
if (const auto [ pose_list_it, found ] = m_pose_list_lookup->poses.find(pose_list_id); found)
{
m_last_pose_path = *pose_path;
m_last_pose_list = pose_list_it->second;
}
else
@@ -74,15 +80,32 @@ void assets::kitti_parser::parser_context::operator()(lookup_type::const_pointer
reset();
if (const auto e = load_point_file(filename, m_buffer))
pose_data pose;
if (pose_index < m_last_pose_list.size())
{
ztu::logger::error("Could not load kitti file %: %", filename, e.message());
pose = m_last_pose_list[pose_index];
}
else
{
ztu::logger::error(
"Pose index % of kitti file % is out of range for its pose file %. Proceeding with identity pose.",
pose_index,
filename,
m_last_pose_path
);
pose = glm::identity<pose_data>();
}
if (const auto e = parse_file(filename, m_buffer))
{
ztu::logger::error("Could not parse kitti file %: %", filename, e.message());
return;
}
transform_point_cloud(m_buffer.positions(), pose);
m_store->insert(id, m_buffer);
m_stores->point_clouds.insert(id, m_buffer);
}
ztu::result<std::filesystem::path> assets::kitti_parser::parent_directory(
@@ -138,8 +161,6 @@ std::error_code assets::kitti_parser::load(
data_stores& stores,
bool pedantic
) {
namespace fs = std::filesystem;
m_path_buffer.clear();
lookups.point_clouds.by_extension(".bin", m_path_buffer);
@@ -148,9 +169,8 @@ std::error_code assets::kitti_parser::load(
m_path_buffer.begin(),
m_path_buffer.end(),
parser_context{
lookups.pose_lists,
stores.pose_lists,
stores.point_clouds
lookups,
stores
}
);
@@ -167,7 +187,7 @@ void assets::kitti_parser::transform_point_cloud(
}
}
std::error_code assets::kitti_parser::load_point_file(
std::error_code assets::kitti_parser::parse_file(
const std::filesystem::path& filename,
point_cloud_data& point_cloud
) {

194
source/assets/file_parsers/threedtk_pose_loader.cpp
View File

@@ -1,137 +1,111 @@
#include "assets/file_parsers/threedtk_pose_loader.hpp"
#include <fstream>
#include <execution>
#include <glm/ext/matrix_transform.hpp>
#include <glm/gtx/euler_angles.hpp>
#include "util/logger.hpp"
#include <fstream>
inline std::error_code threedtk_pose_loader::parse_transform_info(
std::ifstream& in,
std::string& line,
std::array<glm::vec3, 2>& transform_info
std::error_code assets::threedtk_pose_loader::prefetch(
path_id_lookups& lookups
) {
return {};
}
std::error_code assets::threedtk_pose_loader::load(
path_id_lookups& lookups,
data_stores& stores,
bool pedantic
) {
m_path_buffer.clear();
lookups.poses.by_extension(".pose", m_path_buffer);
std::for_each(
std::execution::parallel_unsequenced_policy{},
m_path_buffer.begin(),
m_path_buffer.end(),
parser_context{
lookups,
stores
}
);
return {};
}
assets::threedtk_pose_loader::parser_context::parser_context(
path_id_lookups& m_id_lookups,
data_stores& m_stores
) :
m_id_lookups{ &m_id_lookups },
m_stores{ &m_stores }
{
m_line_buffer.reserve(64);
}
void assets::threedtk_pose_loader::parser_context::operator()(lookup_type::const_pointer entry) noexcept
{
const auto& [ filename, id ] = *entry;
if (const auto res = parse_file(filename, m_line_buffer))
{
m_stores->poses.insert(id, res.value());
}
else
{
const auto error = res.error();
ztu::logger::error(
"Error while parsing 3dtk pose file %: [%] %",
filename,
error.category().name(),
error.message()
);
}
}
ztu::result<assets::pose_data> assets::threedtk_pose_loader::parse_file(
const std::filesystem::path& filename,
std::string& line_buffer
) {
auto in = std::ifstream{ filename };
if (not in.is_open())
{
return std::unexpected{ std::make_error_code(std::errc::no_such_file_or_directory) };
}
std::array<glm::vec3, 2> transform_vecs{};
for (std::size_t row{}; row != 2; ++row) {
std::getline(in, line);
std::getline(in, line_buffer);
auto it = line.cbegin().base();
auto end = line.cend().base();
auto it = line_buffer.cbegin().base();
auto end = line_buffer.cend().base();
for (glm::vec3::length_type col{}; col != 3; ++col) {
const auto [ ptr, ec ] = std::from_chars(
it, end,
transform_info[row][col],
transform_vecs[row][col],
std::chars_format::general
);
if (ec != std::errc{}) {
return std::make_error_code(ec);
return std::unexpected{ std::make_error_code(ec) };
}
it = ptr + 1; // skip space in between components
}
}
}
inline ztu::result<pose_prefetch_lookup::index_type> threedtk_pose_loader::parse_index(
const std::string_view filename
) {
static constexpr auto prefix = std::string_view{ "scan" };
auto name_view = filename.substr(0, name_view.find('.'));
if (name_view.length() <= prefix.length()) [[unlikely]]
{
return std::make_error_code(std::errc::invalid_argument);
}
name_view = name_view.substr(prefix.length());
pose_prefetch_lookup::index_type index;
const auto res = std::from_chars(name_view.begin(), name_view.end(), index);
if (res.ec != std::errc{}) [[unlikely]]
{
return std::make_error_code(res.ec);
}
return index;
}
void threedtk_pose_loader::load(
const ztu::string_list& filenames,
dynamic_pose_store& store,
pose_prefetch_lookup& id_lookup
) {
auto filename_buffer = std::string{};
auto in = std::ifstream{};
auto line = std::string{};
auto pose_buffer = dynamic_pose_buffer{};
for (const auto filename : filenames)
{
pose_prefetch_lookup::index_type index;
if (const auto res = parse_index(filename))
{
index = *res;
}
else
{
const auto error = res.error();
ztu::logger::error(
"Error while parsing 3dtk pose file index %: [%] %",
filename,
error.category().name(),
error.message()
);
}
filename_buffer = filename;
in.open(filename_buffer.c_str());
if (not in.is_open()) {
ztu::logger::error("Cannot open 3dtk pose file %", filename);
continue;
}
std::array<glm::vec3, 2> transform_info{};
const auto error = parse_transform_info(in, line, transform_info);
in.close();
if (error)
{
ztu::logger::error(
"Error while parsing 3dtk pose file %: [%] %",
filename,
error.category().name(),
error.message()
);
continue;
}
const auto& translation = transform_info[0];
auto& angles = transform_info[1];
angles *= static_cast<float>(M_PI / 180.0);
pose_buffer = (
glm::translate(glm::identity<glm::mat4>(), translation) *
glm::eulerAngleXYZ(angles[0], angles[1], angles[2])
);
const auto id = store.add(pose_buffer);
id_lookup.emplace(filename, index, id);
}
const auto& translation = transform_vecs[0];
auto& angles = transform_vecs[1];
angles *= static_cast<float>(M_PI / 180.0);
return (
glm::translate(glm::identity<glm::mat4>(), translation) *
glm::eulerAngleXYZ(angles[0], angles[1], angles[2])
);
}