further fixes

include/assets/prefetch_lookups/pose_prefetch_lookup.hpp

@@ -24,7 +24,6 @@ public:
 
 	void emplace_hint_dir(
 		directory_iterator directory_it,
-		const std::filesystem::path& directory,
 		index_type index,
 		dynamic_pose_store::id_type id
 	);
@@ -45,7 +44,6 @@ public:
 		index_type index
 	);
 
-protected:
 	directory_iterator emplace_dir(
 		directory_iterator directory_it,
 		const std::filesystem::path& directory

source/assets/data_loaders/kitti_loader.cpp

@@ -115,6 +115,7 @@ std::error_code kitti_loader::load(
 			continue;
 		}
 
+		// TODO find out if the path splitting is consistent
 		const auto base_directory = parent_directory(file.substr(0, sep_index));
 		if (not base_directory) [[unlikely]]
 		{
@@ -162,14 +163,12 @@ std::error_code kitti_loader::load(
 		{
 			const auto file_path = std::string_view{ file.path().c_str() };
 
-			const auto extension_begin = file_path.rfind('.');
+			if (not file_path.ends_with(".bin"))
-
-			if (extension_begin == std::string_view::npos or file_path.substr(extension_begin) != ".bin")
 			{
 				continue;
 			}
 
-			auto filename_begin = file_path.rfind(fs::path::preferred_separator, extension_begin);
+			auto filename_begin = file_path.rfind(fs::path::preferred_separator);
 			filename_begin = filename_begin == std::string_view::npos ? 0 : filename_begin + 1;
 
 			const auto filename_only = file_path.substr(filename_begin);

source/assets/data_loaders/kitti_pose_loader.cpp

@@ -42,9 +42,9 @@ std::error_code kitti_pose_loader::load(
 
 	auto path_buffer = fs::path{};
 	auto in = std::ifstream{}; // TODO disable exceptions (for other loaders as well)
-	auto pose_buffer = dynamic_pose_buffer{};
 
-	pose_buffer = glm::identity<glm::mat4>();
+	// Needed to initialize 4th row and col
+	buffer = glm::identity<glm::mat4>();
 
 	auto processed_filenames = ztu::string_list{};
 
@@ -62,13 +62,12 @@ std::error_code kitti_pose_loader::load(
 			ztu::logger::error("Kitti pose file does not exist: %", path_buffer);
 			return;
 		}
-
 		processed_filenames.push_back(path_buffer.c_str());
 	};
 
 	for (const auto directory : paths.directories) {
 		path_buffer.assign(directory.begin(), directory.end());
-		path_buffer /= "pose.txt";
+		path_buffer /= pose_filename;
 		preprocess_file();
 	}
 
@@ -79,6 +78,16 @@ std::error_code kitti_pose_loader::load(
 
 	for (const auto filename : processed_filenames)
 	{
+		// TODO if (not) removing the path separator creates issues.
+		const auto directory = filename.substr(0, filename.length() - pose_filename.length());
+
+		auto [ dir_it, dir_match ] = id_lookup.poses.find_directory(directory);
+
+		if (not dir_match) [[unlikely]]
+		{
+			dir_it = id_lookup.poses.emplace_dir(dir_it, directory);
+		}
+
 		in.open(filename.data()); // Safe because string list adds null terminator
 		if (not in.is_open())
 		{
@@ -90,7 +99,7 @@ std::error_code kitti_pose_loader::load(
 
 		for (auto i = pose_prefetch_lookup::index_type{}; in.peek() != std::ifstream::traits_type::eof(); ++i)
 		{
-			if (const auto error = parse_pose(in, pose_buffer))
+			if (const auto error = parse_pose(in, buffer))
 			{
 				ztu::logger::error(
 					"Error occurred while parsing kitti pose % in file %: [%] %",
@@ -102,60 +111,8 @@ std::error_code kitti_pose_loader::load(
 				continue;
 			}
 
-			const auto id = store.poses.add(pose_buffer);
+			const auto id = store.poses.add(buffer);
-
+			id_lookup.poses.emplace_hint_dir(dir_it, i, id);
-			// TODO if (not) removing the path separator creates issues.
-			const auto directory = filename.substr(0, filename.length() - pose_filename.length());
-
-			id_lookup.poses.emplace(directory, i, id);
-		}
-
-		in.close();
-	}
-}
-
-void kitti_pose_loader::load(
-	const ztu::string_list& directories,
-	dynamic_pose_store& store,
-	pose_prefetch_lookup& id_lookup
-) {
-
-	auto filename_buffer = std::filesystem::path{};
-	auto in = std::ifstream{}; // TODO disable exceptions (for other loaders as well)
-	auto pose_buffer = dynamic_pose_buffer{};
-
-	pose_buffer = glm::identity<glm::mat4>();
-
-	for (const auto directory : directories)
-	{
-		filename_buffer = directory;
-		filename_buffer /= "pose.txt";
-
-		in.open(filename_buffer);
-		if (not in.is_open())
-		{
-			ztu::logger::error("Cannot open kitti pose file %", filename_buffer);
-			continue;
-		}
-
-		in >> std::skipws;
-
-		for (auto i = pose_prefetch_lookup::index_type{}; in.peek() != std::ifstream::traits_type::eof(); ++i)
-		{
-			if (const auto error = parse_pose(in, pose_buffer))
-			{
-				ztu::logger::error(
-					"Error occurred while parsing kitti pose % in file %: [%] %",
-					i,
-					filename_buffer,
-					error.category().name(),
-					error.message()
-				);
-				continue;
-			}
-
-			const auto id = store.add(pose_buffer);
-			id_lookup.emplace(directory, i, id);
 		}
 
 		in.close();

source/assets/data_loaders/obj_loader.cpp

@@ -61,9 +61,10 @@ inline std::error_code make_error_code(codes e) {
 } // namespace mesh_loader_error
 
 
-using vertex_type = std::array<dynamic_mesh_data::index_type, 3>;
+using vertex_type = std::array<dynamic_mesh_buffer::index_type, 3>;
 
-struct indexed_vertex_type {
+struct indexed_vertex_type
+{
 	vertex_type vertex;
 	ztu::u32 buffer_index;
 
@@ -102,6 +103,121 @@ std::errc parse_numeric_vector(std::string_view param, std::array<T, Count>& val
 	return {};
 };
 
+
+void find_materials(
+	std::span<char> buffer,
+	std::ifstream& in,
+	ztu::string_list& material_filenames
+) {
+	static constexpr auto keyword = std::string_view{ "\nmtllib " };
+
+	const auto buffer_view = std::string_view(buffer);
+
+	// Add linebreak to simplify line begin search
+	buffer.front() = '\n';
+	auto leftover = std::size_t{ 1 };
+
+	do
+	{
+		// Keep some old characters to continue matching interrupted sequence
+		std::copy(buffer.end() - leftover, buffer.end(), buffer.begin());
+
+		in.read(buffer.data() + leftover, buffer.size() - leftover);
+
+		const auto str = buffer_view.substr(0, leftover + in.gcount());
+
+		leftover = keyword.size();
+
+		auto pos = std::string_view::size_type{};
+		while ((pos = str.find(keyword, pos)) != std::string_view::npos)
+		{
+			const auto filename_begin = pos + keyword.size();
+			const auto filename_end = str.find('\n', filename_begin);
+
+			if (filename_end != std::string_view::npos)
+			{
+				const auto length = filename_end - filename_begin;
+				const auto material_filename = str.substr(filename_begin, length);
+
+				// TODO get base dir from param figure out how to avoid heap
+				if (material_filename.is_relative())
+				{
+					material_filename = directory / material_filename;
+				}
+
+				material_filenames.push_back(material_filename);
+
+				pos = filename_end;
+			}
+			else // string exceeds the buffer
+			{
+				if (pos == 0) [[unlikely]]
+				{
+					std::cout << "Ignoring string match, as it exceeds buffer size." << std::endl;
+					leftover = 0;
+				}
+				else
+				{
+					leftover = str.size() - pos;
+				}
+				break;
+			}
+		}
+	} while (not in.eof());
+}
+
+
+std::error_code obj_loader::prefetch(
+	const file_dir_list& paths,
+	prefetch_queue& queue
+) {
+	namespace fs = std::filesystem;
+	using obj_loader_error::codes;
+	using obj_loader_error::make_error_code;
+
+	auto buffer = std::vector<char>(8 * 1024, '\0');
+
+	auto in = std::ifstream{};
+
+	const auto parse_file = [&](const char* filename)
+	{
+		in.open(filename);
+		if (not in.is_open()) {
+			ztu::logger::error("Could not open .obj file '%'", filename);
+			return;
+		}
+
+		find_materials(buffer, in, queue.mtl_queue.files);
+
+		in.close();
+	};
+
+	for (const auto file : paths.files)
+	{
+		// `file` is null-terminates by list.
+		parse_file(file.data());
+	}
+
+	for (const auto directory : paths.directories)
+	{
+		for (const auto& file : fs::directory_iterator{ directory }) {
+
+			const auto& file_path = std::string_view{ file.path().c_str() };
+
+			// TODO remove heap allocation
+			if (not file_path.ends_with(".obj"))
+			{
+				continue;
+			}
+
+			// Null terminated by fs::path.
+			parse_file(file_path.data());
+		}
+	}
+
+	return {};
+}
+
 std::error_code obj_loader::load_directory(
 	dynamic_data_loader_ctx& ctx,
 	dynamic_mesh_store& store,
@@ -144,15 +260,67 @@ std::error_code obj_loader::load_directory(
 	return {};
 }
 
+[[nodiscard]] static std::error_code load(
+	components::mesh_vertex::flags enabled_components,
+	dynamic_mesh_buffer& buffer,
+	const file_dir_list& paths,
+	prefetch_lookup& id_lookup,
+	dynamic_data_store& store,
+	bool pedantic = false
+) {
+
+
+	auto in = std::ifstream{};
+	std::set<indexed_vertex_type> vertex_ids;
+
+	const auto parse_file = [&](const char* filename)
+	{
+		in.open(filename);
+		if (not in.is_open()) {
+			ztu::logger::error("Could not open .obj file '%'", filename);
+			return;
+		}
+
+
+		vertex_ids.clear();
+
+		in.close();
+	};
+
+	for (const auto file : paths.files)
+	{
+		// `file` is null-terminates by list.
+		parse_file(file.data());
+	}
+
+	for (const auto directory : paths.directories)
+	{
+		for (const auto& file : fs::directory_iterator{ directory }) {
+
+			const auto& file_path = std::string_view{ file.path().c_str() };
+
+			// TODO remove heap allocation
+			if (not file_path.ends_with(".obj"))
+			{
+				continue;
+			}
+
+			// Null terminated by fs::path.
+			parse_file(file_path.data());
+		}
+	}
+
+}
 
 // TODO refactor so there is a function like parse_normals etc.
 
 std::error_code obj_loader::load(
-	dynamic_data_loader_ctx& ctx,
-	dynamic_mesh_store& store,
 	components::mesh_vertex::flags enabled_components,
-	const std::filesystem::path& filename,
+	dynamic_mesh_buffer& buffer,
-	const bool pedantic
+	const char* filename,
+	prefetch_lookup& id_lookup,
+	dynamic_data_store& store,
+	bool pedantic
 ) {
 	using obj_loader_error::codes;
 	using obj_loader_error::make_error_code;
@@ -173,33 +341,26 @@ std::error_code obj_loader::load(
 	// and only push unique combinations to the buffers.
 	std::set<indexed_vertex_type> vertex_ids;
 
-	auto mesh = dynamic_mesh_data{};
-
 	// Buffers
-	auto position_buffer = mesh.positions();
+	// TODO find out if this is still relevant
-	auto normal_buffer = mesh.normals();
+	auto position_buffer = buffer.positions();
-	auto tex_coord_buffer = mesh.tex_coords();
+	auto normal_buffer = buffer.normals();
+	auto tex_coord_buffer = buffer.tex_coords();
 
 	std::unordered_map<std::string, ztu::u32> material_name_lookup;
 
-
-	constexpr auto mtl_loader_id = *ctx.material_loader.find_loader_static("mtl");
-	mtl_loader& material_loader = ctx.material_loader.get_loader<mtl_loader_id>();
-
-	material_loader.clear_name_lookup();
-
 	std::string material_name;
 
 	const auto push_mesh = [&](const bool clear_buffers = false) {
 
-		if (not mesh.positions().empty())
+		if (not buffer.positions().empty())
 		{
 			// Copy buffers instead of moving to keep capacity for further parsing
 			// and have the final buffers be shrunk to size.
 			if (not material_name.empty()) {
 				if (const auto id = material_loader.find_id(material_name))
 				{
-					mesh.material_id() = *id;
+					buffer.material_id() = *id;
 				}
 				else
 				{
@@ -214,24 +375,24 @@ std::error_code obj_loader::load(
 			ztu::logger::debug("Parsed % normals.", mesh.normals().size());
 			ztu::logger::debug("Parsed % tex_coords.", mesh.tex_coords().size());
 
-			if (not mesh.positions().empty())
+			if (not buffer.positions().empty())
 			{
-				mesh.components() |= components::mesh_vertex::flags::position;
+				buffer.components() |= components::mesh_vertex::flags::position;
 			}
 
-			if (not mesh.normals().empty())
+			if (not buffer.normals().empty())
 			{
-				mesh.components() |= components::mesh_vertex::flags::normal;
+				buffer.components() |= components::mesh_vertex::flags::normal;
 			}
 
-			if (not mesh.tex_coords().empty())
+			if (not buffer.tex_coords().empty())
 			{
-				mesh.components() |= components::mesh_vertex::flags::tex_coord;
+				buffer.components() |= components::mesh_vertex::flags::tex_coord;
 			}
 
-			ztu::logger::debug("Pushing obj mesh with % triangles.", mesh.triangles().size());
+			ztu::logger::debug("Pushing obj mesh with % triangles.", buffer.triangles().size());
 
-			store.add(std::move(mesh));
+			store.meshes.add(buffer);
 		}
 
 		if (clear_buffers)
@@ -241,7 +402,7 @@ std::error_code obj_loader::load(
 			tex_coord_buffer.clear();
 		}
 
-		mesh = dynamic_mesh_data{};
+		buffer.clear();
 
 		vertex_ids.clear();
 		material_name.clear();
@@ -251,7 +412,7 @@ std::error_code obj_loader::load(
 
 		auto indexed_vid = indexed_vertex_type{
 			.vertex = vertex,
-			.buffer_index = static_cast<ztu::u32>(mesh.positions().size())
+			.buffer_index = static_cast<ztu::u32>(buffer.positions().size())
 		};
 
 		// Search through sorted lookup to check if index combination is unique
@@ -263,17 +424,17 @@ std::error_code obj_loader::load(
 
 			if (position_index < position_buffer.size())
 			{
-				mesh.positions().emplace_back(position_buffer[position_index]);
+				buffer.positions().emplace_back(position_buffer[position_index]);
 			}
 
 			if (normal_index < normal_buffer.size())
 			{
-				mesh.normals().emplace_back(normal_buffer[normal_index]);
+				buffer.normals().emplace_back(normal_buffer[normal_index]);
 			}
 
 			if (tex_coord_index < tex_coord_buffer.size())
 			{
-				mesh.tex_coords().emplace_back(tex_coord_buffer[tex_coord_index]);
+				buffer.tex_coords().emplace_back(tex_coord_buffer[tex_coord_index]);
 			}
 		}
 
@@ -437,8 +598,6 @@ std::error_code obj_loader::load(
 		})
 	);
 
-	material_loader.clear_name_lookup();
-
 	if (ec != codes::ok)
 	{
 		return make_error_code(ec);

source/assets/prefetch_lookups/pose_prefetch_lookup.cpp

@@ -70,19 +70,8 @@ void pose_prefetch_lookup::emplace(
 		directory_it = emplace_dir(directory_it, directory);
 	}
 
-	const auto [ index_it, match ] = find_index(
+ 	emplace_hint_dir(
 		directory_it,
-		index
-	);
-
-	if (not match)
-	{
-
-	}
-
-	emplace_hint_dir_index(
-		directory_it,
-		index_it,
 		index,
 		id
 	);
@@ -91,13 +80,10 @@ void pose_prefetch_lookup::emplace(
 
 void pose_prefetch_lookup::emplace_hint_dir(
 	directory_iterator directory_it,
-	const std::filesystem::path& directory,
 	const index_type index,
 	const dynamic_pose_store::id_type id
 ) {
 
-	directory_it = emplace_dir(directory_it, directory);
-
 	const auto [ index_it, match ] = find_index(
 		directory_it,
 		index
@@ -105,7 +91,7 @@ void pose_prefetch_lookup::emplace_hint_dir(
 
 	if (match) [[unlikely]]
 	{
-		const auto dir_index = dir_it->second;
+		const auto dir_index = directory_it.first->second;
 		const auto id_index = index_it - m_directory_indices.begin() - dir_index;
 		m_pose_ids[id_index] = match; // TODO I guess I should warn?!?
 	}