zy4n/Z3D
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

further fixes

Browse Source
This commit is contained in:
ZY4N
2024-12-22 21:52:05 +01:00
parent db8db8f9d7
commit b385b3b1c8
5 changed files with 214 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
source/assets/data_loaders/kitti_loader.cpp
View File

@@ -115,6 +115,7 @@ std::error_code kitti_loader::load(
continue;
}
// TODO find out if the path splitting is consistent
const auto base_directory = parent_directory(file.substr(0, sep_index));
if (not base_directory) [[unlikely]]
{
@@ -162,14 +163,12 @@ std::error_code kitti_loader::load(
{
const auto file_path = std::string_view{ file.path().c_str() };
const auto extension_begin = file_path.rfind('.');
if (extension_begin == std::string_view::npos or file_path.substr(extension_begin) != ".bin")
if (not file_path.ends_with(".bin"))
{
continue;
}
auto filename_begin = file_path.rfind(fs::path::preferred_separator, extension_begin);
auto filename_begin = file_path.rfind(fs::path::preferred_separator);
filename_begin = filename_begin == std::string_view::npos ? 0 : filename_begin + 1;
const auto filename_only = file_path.substr(filename_begin);

75
source/assets/data_loaders/kitti_pose_loader.cpp
View File

@@ -42,9 +42,9 @@ std::error_code kitti_pose_loader::load(
auto path_buffer = fs::path{};
auto in = std::ifstream{}; // TODO disable exceptions (for other loaders as well)
auto pose_buffer = dynamic_pose_buffer{};
pose_buffer = glm::identity<glm::mat4>();
// Needed to initialize 4th row and col
buffer = glm::identity<glm::mat4>();
auto processed_filenames = ztu::string_list{};
@@ -62,13 +62,12 @@ std::error_code kitti_pose_loader::load(
ztu::logger::error("Kitti pose file does not exist: %", path_buffer);
return;
}
processed_filenames.push_back(path_buffer.c_str());
};
for (const auto directory : paths.directories) {
path_buffer.assign(directory.begin(), directory.end());
path_buffer /= "pose.txt";
path_buffer /= pose_filename;
preprocess_file();
}
@@ -79,6 +78,16 @@ std::error_code kitti_pose_loader::load(
for (const auto filename : processed_filenames)
{
// TODO if (not) removing the path separator creates issues.
const auto directory = filename.substr(0, filename.length() - pose_filename.length());
auto [ dir_it, dir_match ] = id_lookup.poses.find_directory(directory);
if (not dir_match) [[unlikely]]
{
dir_it = id_lookup.poses.emplace_dir(dir_it, directory);
}
in.open(filename.data()); // Safe because string list adds null terminator
if (not in.is_open())
{
@@ -90,7 +99,7 @@ std::error_code kitti_pose_loader::load(
for (auto i = pose_prefetch_lookup::index_type{}; in.peek() != std::ifstream::traits_type::eof(); ++i)
{
if (const auto error = parse_pose(in, pose_buffer))
if (const auto error = parse_pose(in, buffer))
{
ztu::logger::error(
"Error occurred while parsing kitti pose % in file %: [%] %",
@@ -102,62 +111,10 @@ std::error_code kitti_pose_loader::load(
continue;
}
const auto id = store.poses.add(pose_buffer);
// TODO if (not) removing the path separator creates issues.
const auto directory = filename.substr(0, filename.length() - pose_filename.length());
id_lookup.poses.emplace(directory, i, id);
const auto id = store.poses.add(buffer);
id_lookup.poses.emplace_hint_dir(dir_it, i, id);
}
in.close();
}
}
void kitti_pose_loader::load(
const ztu::string_list& directories,
dynamic_pose_store& store,
pose_prefetch_lookup& id_lookup
) {
auto filename_buffer = std::filesystem::path{};
auto in = std::ifstream{}; // TODO disable exceptions (for other loaders as well)
auto pose_buffer = dynamic_pose_buffer{};
pose_buffer = glm::identity<glm::mat4>();
for (const auto directory : directories)
{
filename_buffer = directory;
filename_buffer /= "pose.txt";
in.open(filename_buffer);
if (not in.is_open())
{
ztu::logger::error("Cannot open kitti pose file %", filename_buffer);
continue;
}
in >> std::skipws;
for (auto i = pose_prefetch_lookup::index_type{}; in.peek() != std::ifstream::traits_type::eof(); ++i)
{
if (const auto error = parse_pose(in, pose_buffer))
{
ztu::logger::error(
"Error occurred while parsing kitti pose % in file %: [%] %",
i,
filename_buffer,
error.category().name(),
error.message()
);
continue;
}
const auto id = store.add(pose_buffer);
id_lookup.emplace(directory, i, id);
}
in.close();
}
}

227
source/assets/data_loaders/obj_loader.cpp
View File

@@ -61,9 +61,10 @@ inline std::error_code make_error_code(codes e) {
} // namespace mesh_loader_error
using vertex_type = std::array<dynamic_mesh_data::index_type, 3>;
using vertex_type = std::array<dynamic_mesh_buffer::index_type, 3>;
struct indexed_vertex_type {
struct indexed_vertex_type
{
vertex_type vertex;
ztu::u32 buffer_index;
@@ -102,6 +103,121 @@ std::errc parse_numeric_vector(std::string_view param, std::array<T, Count>& val
return {};
};
void find_materials(
std::span<char> buffer,
std::ifstream& in,
ztu::string_list& material_filenames
) {
static constexpr auto keyword = std::string_view{ "\nmtllib " };
const auto buffer_view = std::string_view(buffer);
// Add linebreak to simplify line begin search
buffer.front() = '\n';
auto leftover = std::size_t{ 1 };
do
{
// Keep some old characters to continue matching interrupted sequence
std::copy(buffer.end() - leftover, buffer.end(), buffer.begin());
in.read(buffer.data() + leftover, buffer.size() - leftover);
const auto str = buffer_view.substr(0, leftover + in.gcount());
leftover = keyword.size();
auto pos = std::string_view::size_type{};
while ((pos = str.find(keyword, pos)) != std::string_view::npos)
{
const auto filename_begin = pos + keyword.size();
const auto filename_end = str.find('\n', filename_begin);
if (filename_end != std::string_view::npos)
{
const auto length = filename_end - filename_begin;
const auto material_filename = str.substr(filename_begin, length);
// TODO get base dir from param figure out how to avoid heap
if (material_filename.is_relative())
{
material_filename = directory / material_filename;
}
material_filenames.push_back(material_filename);
pos = filename_end;
}
else // string exceeds the buffer
{
if (pos == 0) [[unlikely]]
{
std::cout << "Ignoring string match, as it exceeds buffer size." << std::endl;
leftover = 0;
}
else
{
leftover = str.size() - pos;
}
break;
}
}
} while (not in.eof());
}
std::error_code obj_loader::prefetch(
const file_dir_list& paths,
prefetch_queue& queue
) {
namespace fs = std::filesystem;
using obj_loader_error::codes;
using obj_loader_error::make_error_code;
auto buffer = std::vector<char>(8 * 1024, '\0');
auto in = std::ifstream{};
const auto parse_file = [&](const char* filename)
{
in.open(filename);
if (not in.is_open()) {
ztu::logger::error("Could not open .obj file '%'", filename);
return;
}
find_materials(buffer, in, queue.mtl_queue.files);
in.close();
};
for (const auto file : paths.files)
{
// `file` is null-terminates by list.
parse_file(file.data());
}
for (const auto directory : paths.directories)
{
for (const auto& file : fs::directory_iterator{ directory }) {
const auto& file_path = std::string_view{ file.path().c_str() };
// TODO remove heap allocation
if (not file_path.ends_with(".obj"))
{
continue;
}
// Null terminated by fs::path.
parse_file(file_path.data());
}
}
return {};
}
std::error_code obj_loader::load_directory(
dynamic_data_loader_ctx& ctx,
dynamic_mesh_store& store,
@@ -144,15 +260,67 @@ std::error_code obj_loader::load_directory(
return {};
}
[[nodiscard]] static std::error_code load(
components::mesh_vertex::flags enabled_components,
dynamic_mesh_buffer& buffer,
const file_dir_list& paths,
prefetch_lookup& id_lookup,
dynamic_data_store& store,
bool pedantic = false
) {
auto in = std::ifstream{};
std::set<indexed_vertex_type> vertex_ids;
const auto parse_file = [&](const char* filename)
{
in.open(filename);
if (not in.is_open()) {
ztu::logger::error("Could not open .obj file '%'", filename);
return;
}
vertex_ids.clear();
in.close();
};
for (const auto file : paths.files)
{
// `file` is null-terminates by list.
parse_file(file.data());
}
for (const auto directory : paths.directories)
{
for (const auto& file : fs::directory_iterator{ directory }) {
const auto& file_path = std::string_view{ file.path().c_str() };
// TODO remove heap allocation
if (not file_path.ends_with(".obj"))
{
continue;
}
// Null terminated by fs::path.
parse_file(file_path.data());
}
}
}
// TODO refactor so there is a function like parse_normals etc.
std::error_code obj_loader::load(
dynamic_data_loader_ctx& ctx,
dynamic_mesh_store& store,
components::mesh_vertex::flags enabled_components,
const std::filesystem::path& filename,
const bool pedantic
dynamic_mesh_buffer& buffer,
const char* filename,
prefetch_lookup& id_lookup,
dynamic_data_store& store,
bool pedantic
) {
using obj_loader_error::codes;
using obj_loader_error::make_error_code;
@@ -173,33 +341,26 @@ std::error_code obj_loader::load(
// and only push unique combinations to the buffers.
std::set<indexed_vertex_type> vertex_ids;
auto mesh = dynamic_mesh_data{};
// Buffers
auto position_buffer = mesh.positions();
auto normal_buffer = mesh.normals();
auto tex_coord_buffer = mesh.tex_coords();
// TODO find out if this is still relevant
auto position_buffer = buffer.positions();
auto normal_buffer = buffer.normals();
auto tex_coord_buffer = buffer.tex_coords();
std::unordered_map<std::string, ztu::u32> material_name_lookup;
constexpr auto mtl_loader_id = *ctx.material_loader.find_loader_static("mtl");
mtl_loader& material_loader = ctx.material_loader.get_loader<mtl_loader_id>();
material_loader.clear_name_lookup();
std::string material_name;
const auto push_mesh = [&](const bool clear_buffers = false) {
if (not mesh.positions().empty())
if (not buffer.positions().empty())
{
// Copy buffers instead of moving to keep capacity for further parsing
// and have the final buffers be shrunk to size.
if (not material_name.empty()) {
if (const auto id = material_loader.find_id(material_name))
{
mesh.material_id() = *id;
buffer.material_id() = *id;
}
else
{
@@ -214,24 +375,24 @@ std::error_code obj_loader::load(
ztu::logger::debug("Parsed % normals.", mesh.normals().size());
ztu::logger::debug("Parsed % tex_coords.", mesh.tex_coords().size());
if (not mesh.positions().empty())
if (not buffer.positions().empty())
{
mesh.components() |= components::mesh_vertex::flags::position;
buffer.components() |= components::mesh_vertex::flags::position;
}
if (not mesh.normals().empty())
if (not buffer.normals().empty())
{
mesh.components() |= components::mesh_vertex::flags::normal;
buffer.components() |= components::mesh_vertex::flags::normal;
}
if (not mesh.tex_coords().empty())
if (not buffer.tex_coords().empty())
{
mesh.components() |= components::mesh_vertex::flags::tex_coord;
buffer.components() |= components::mesh_vertex::flags::tex_coord;
}
ztu::logger::debug("Pushing obj mesh with % triangles.", mesh.triangles().size());
ztu::logger::debug("Pushing obj mesh with % triangles.", buffer.triangles().size());
store.add(std::move(mesh));
store.meshes.add(buffer);
}
if (clear_buffers)
@@ -241,7 +402,7 @@ std::error_code obj_loader::load(
tex_coord_buffer.clear();
}
mesh = dynamic_mesh_data{};
buffer.clear();
vertex_ids.clear();
material_name.clear();
@@ -251,7 +412,7 @@ std::error_code obj_loader::load(
auto indexed_vid = indexed_vertex_type{
.vertex = vertex,
.buffer_index = static_cast<ztu::u32>(mesh.positions().size())
.buffer_index = static_cast<ztu::u32>(buffer.positions().size())
};
// Search through sorted lookup to check if index combination is unique
@@ -263,17 +424,17 @@ std::error_code obj_loader::load(
if (position_index < position_buffer.size())
{
mesh.positions().emplace_back(position_buffer[position_index]);
buffer.positions().emplace_back(position_buffer[position_index]);
}
if (normal_index < normal_buffer.size())
{
mesh.normals().emplace_back(normal_buffer[normal_index]);
buffer.normals().emplace_back(normal_buffer[normal_index]);
}
if (tex_coord_index < tex_coord_buffer.size())
{
mesh.tex_coords().emplace_back(tex_coord_buffer[tex_coord_index]);
buffer.tex_coords().emplace_back(tex_coord_buffer[tex_coord_index]);
}
}
@@ -437,8 +598,6 @@ std::error_code obj_loader::load(
})
);
material_loader.clear_name_lookup();
if (ec != codes::ok)
{
return make_error_code(ec);

18
source/assets/prefetch_lookups/pose_prefetch_lookup.cpp
View File

@@ -70,19 +70,8 @@ void pose_prefetch_lookup::emplace(
directory_it = emplace_dir(directory_it, directory);
}
const auto [ index_it, match ] = find_index(
emplace_hint_dir(
directory_it,
index
);
if (not match)
{
}
emplace_hint_dir_index(
directory_it,
index_it,
index,
id
);
@@ -91,13 +80,10 @@ void pose_prefetch_lookup::emplace(
void pose_prefetch_lookup::emplace_hint_dir(
directory_iterator directory_it,
const std::filesystem::path& directory,
const index_type index,
const dynamic_pose_store::id_type id
) {
directory_it = emplace_dir(directory_it, directory);
const auto [ index_it, match ] = find_index(
directory_it,
index
@@ -105,7 +91,7 @@ void pose_prefetch_lookup::emplace_hint_dir(
if (match) [[unlikely]]
{
const auto dir_index = dir_it->second;
const auto dir_index = directory_it.first->second;
const auto id_index = index_it - m_directory_indices.begin() - dir_index;
m_pose_ids[id_index] = match; // TODO I guess I should warn?!?
}