In the middle of multithreading parsers.
This commit is contained in:
zy4n
463
source/assets/file_parsers/glsl_parser.cpp
Normal file
463
source/assets/file_parsers/glsl_parser.cpp
Normal file
@@ -0,0 +1,463 @@
|
||||
#include "assets/file_parsers/glsl_parser.hpp"
|
||||
|
||||
#include <fstream>
|
||||
#include <numeric>
|
||||
|
||||
#include "assets/components/mesh_shader_components.hpp"
|
||||
#include "assets/components/point_cloud_shader_components.hpp"
|
||||
#include "util/logger.hpp"
|
||||
#include <mutex>
|
||||
#include <execution>
|
||||
|
||||
namespace assets::language
|
||||
{
|
||||
enum class declaration_type : std::size_t
|
||||
{
|
||||
stage = 0,
|
||||
geometry = 1,
|
||||
features = 2,
|
||||
static_enable = 3,
|
||||
dynamic_enable = 4,
|
||||
invalid = std::numeric_limits<std::size_t>::max()
|
||||
};
|
||||
|
||||
using stage = shader_components::stage;
|
||||
|
||||
inline constexpr auto declaration_prefix = std::string_view("\n#pragma ");
|
||||
inline constexpr auto title_separator = ':';
|
||||
inline constexpr auto value_separator = ' ';
|
||||
|
||||
inline auto declaration_lookup = ztu::string_lookup<declaration_type>{
|
||||
{ "STAGE", declaration_type::stage },
|
||||
{ "GEOMETRY", declaration_type::geometry },
|
||||
{ "FEATURES", declaration_type::features },
|
||||
{ "STATIC_ENABLE", declaration_type::static_enable },
|
||||
{ "DYNAMIC_ENABLE", declaration_type::dynamic_enable }
|
||||
};
|
||||
|
||||
inline auto stage_lookup = ztu::string_lookup<shader_components::stage>{
|
||||
{ "VERTEX", stage::vertex },
|
||||
{ "TESSELATION_CONTROL", stage::tesselation_control },
|
||||
{ "TESSELATION_EVALUATION", stage::tesselation_evaluation },
|
||||
{ "GEOMETRY", stage::geometry },
|
||||
{ "FRAGMENT", stage::fragment },
|
||||
};
|
||||
|
||||
inline auto geometry_lookup = ztu::string_lookup<model_geometry::types>{
|
||||
{ "MESH", model_geometry::types::mesh },
|
||||
{ "POINT_CLOUD", model_geometry::types::point_cloud }
|
||||
};
|
||||
|
||||
inline auto mesh_feature_lookup = ztu::string_lookup<shader_components::flags>{
|
||||
{ "FACE", static_cast<shader_components::flags>(mesh_shader_components::flags::face) },
|
||||
{ "LINE", static_cast<shader_components::flags>(mesh_shader_components::flags::line) },
|
||||
{ "POINT", static_cast<shader_components::flags>(mesh_shader_components::flags::point) },
|
||||
{ "V_L", static_cast<shader_components::flags>(mesh_shader_components::flags::luminance) },
|
||||
{ "V_RGB", static_cast<shader_components::flags>(mesh_shader_components::flags::color) },
|
||||
{ "V_R", static_cast<shader_components::flags>(mesh_shader_components::flags::alpha) },
|
||||
{ "COLOR_TEXTURE", static_cast<shader_components::flags>(mesh_shader_components::flags::color_texture) },
|
||||
{ "U_LIGHTING", static_cast<shader_components::flags>(mesh_shader_components::flags::uniform_lighting) },
|
||||
{ "T_LIGHTING", static_cast<shader_components::flags>(mesh_shader_components::flags::textured_lighting) },
|
||||
{ "U_RGBA", static_cast<shader_components::flags>(mesh_shader_components::flags::uniform_color) }
|
||||
};
|
||||
|
||||
inline auto point_cloud_feature_lookup = ztu::string_lookup<shader_components::flags>{
|
||||
{ "SQUARE", static_cast<shader_components::flags>(point_cloud_shader_components::flags::square) },
|
||||
{ "LIGHTING", static_cast<shader_components::flags>(point_cloud_shader_components::flags::lighting) },
|
||||
{ "V_L", static_cast<shader_components::flags>(point_cloud_shader_components::flags::luminance) },
|
||||
{ "V_RGB", static_cast<shader_components::flags>(point_cloud_shader_components::flags::color) },
|
||||
{ "V_A", static_cast<shader_components::flags>(point_cloud_shader_components::flags::alpha) },
|
||||
{ "U_RGBA", static_cast<shader_components::flags>(point_cloud_shader_components::flags::uniform_color) },
|
||||
{ "RAINBOW", static_cast<shader_components::flags>(point_cloud_shader_components::flags::rainbow) }
|
||||
};
|
||||
}
|
||||
|
||||
assets::glsl_parser::parser_context::parser_context(
|
||||
store_type& m_store,
|
||||
std::mutex& m_store_mutex
|
||||
) : m_store{ &m_store }, m_store_mutex{ &m_store_mutex }
|
||||
{
|
||||
m_buffer.source.reserve(4096);
|
||||
m_value_buffer.reserve(64);
|
||||
m_declaration_value_count_buffer.reserve(8);
|
||||
}
|
||||
|
||||
void assets::glsl_parser::parser_context::reset()
|
||||
{
|
||||
m_buffer.clear();
|
||||
m_value_buffer.clear();
|
||||
m_declaration_value_count_buffer.clear();
|
||||
std::ranges::fill(
|
||||
m_declaration_type_index_buffer,
|
||||
static_cast<std::size_t>(language::declaration_type::invalid)
|
||||
);
|
||||
}
|
||||
|
||||
void assets::glsl_parser::parser_context::operator()(lookup_type::const_pointer entry) noexcept
|
||||
{
|
||||
const auto& [ filename, id ] = *entry;
|
||||
|
||||
if (const auto e = read_file(filename, m_buffer.source))
|
||||
{
|
||||
ztu::logger::warn("Could not load shader source %: %.", filename, e.message());
|
||||
return;
|
||||
}
|
||||
|
||||
tokenize_declarations();
|
||||
|
||||
if (not parse_metadata_from_tokens())
|
||||
{
|
||||
ztu::logger::warn("Ignoring shader % as it contains malformed metadata.", id);
|
||||
return;
|
||||
}
|
||||
|
||||
remove_metadata_declarations();
|
||||
|
||||
{
|
||||
auto lock = std::lock_guard{ *m_store_mutex };
|
||||
m_store->add(id, m_buffer);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void assets::glsl_parser::parser_context::tokenize_declarations()
|
||||
{
|
||||
auto source_rest = std::string_view( m_buffer.source.data(), m_buffer.source.size() );
|
||||
auto offset = std::string_view::size_type{};
|
||||
|
||||
while ((offset = source_rest.find(language::declaration_prefix)) != std::string_view::npos)
|
||||
{
|
||||
const auto current_token_count = m_value_buffer.size();
|
||||
|
||||
auto line_end = source_rest.find('\n', offset);
|
||||
if (line_end == std::string_view::npos)
|
||||
{
|
||||
line_end = source_rest.length();
|
||||
}
|
||||
|
||||
auto declaration = source_rest.substr(offset, line_end - offset);
|
||||
|
||||
if ((offset = declaration.find(language::title_separator)) == std::string_view::npos)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
const auto title = declaration.substr(0, offset);
|
||||
if (const auto it = language::declaration_lookup.find(title); it != language::declaration_lookup.end())
|
||||
{
|
||||
const auto declaration_type = static_cast<std::size_t>(it->second);
|
||||
m_declaration_type_index_buffer[declaration_type] = m_declaration_value_count_buffer.size();
|
||||
}
|
||||
else
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
declaration = declaration.substr(offset);
|
||||
|
||||
if (not declaration.empty() and declaration.front() == language::value_separator)
|
||||
{
|
||||
declaration = declaration.substr(sizeof(language::value_separator), declaration.length());
|
||||
}
|
||||
|
||||
while ((offset = declaration.find(language::value_separator)) != std::string_view::npos)
|
||||
{
|
||||
m_value_buffer.emplace_back(declaration.substr(0, offset));
|
||||
declaration = declaration.substr(offset + sizeof(language::value_separator));
|
||||
}
|
||||
|
||||
if (not declaration.empty())
|
||||
{
|
||||
m_value_buffer.emplace_back(declaration);
|
||||
}
|
||||
|
||||
m_declaration_value_count_buffer.emplace_back(
|
||||
m_value_buffer.size() - current_token_count
|
||||
);
|
||||
|
||||
// Preserve line break
|
||||
source_rest = source_rest.substr(line_end);
|
||||
}
|
||||
}
|
||||
|
||||
bool assets::glsl_parser::parser_context::parse_metadata_from_tokens()
|
||||
{
|
||||
using namespace std::string_view_literals;
|
||||
|
||||
for (const auto [ type, name, parser ] :
|
||||
{
|
||||
std::make_tuple(language::declaration_type::stage, "stage"sv, &parse_stage_declaration),
|
||||
std::make_tuple(language::declaration_type::geometry, "geometry"sv, &parse_geometry_declaration),
|
||||
std::make_tuple(language::declaration_type::features, "features"sv, &parse_components_declaration),
|
||||
std::make_tuple(language::declaration_type::static_enable, "static_enable"sv, &parse_static_enable_declaration),
|
||||
std::make_tuple(language::declaration_type::dynamic_enable, "dynamic_enable"sv, &parse_dynamic_enable_declaration)
|
||||
}
|
||||
) {
|
||||
const auto index = m_declaration_type_index_buffer[static_cast<std::size_t>(type)];
|
||||
|
||||
if (index == static_cast<std::size_t>(language::declaration_type::invalid))
|
||||
{
|
||||
ztu::logger::warn("Shader metadata error: Missing % declaration.", name);
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto value_token_offset = std::accumulate(
|
||||
m_declaration_value_count_buffer.begin(),
|
||||
m_declaration_value_count_buffer.begin() + index,
|
||||
std::size_t{}
|
||||
);
|
||||
const auto value_token_count = m_declaration_value_count_buffer[index];
|
||||
|
||||
if (not parser(std::span(m_value_buffer).subspan(value_token_offset, value_token_count), m_buffer))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void assets::glsl_parser::parser_context::remove_metadata_declarations()
|
||||
{
|
||||
|
||||
if (m_declaration_value_count_buffer.empty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// 1. For every declaration expand the first value view to the size of the line.
|
||||
// 2. Delete all other value views as they are not needed anymore.
|
||||
// 3. Delete all declaration lines by overwriting them with the following source code.
|
||||
// 4. Resize the source vector to the new character count.
|
||||
|
||||
const auto source_view = std::string_view{ m_buffer.source.data(), m_buffer.source.size() };
|
||||
const auto value_span = std::span(m_value_buffer);
|
||||
|
||||
auto offset = std::size_t{};
|
||||
for (auto [ index, count ] : std::ranges::views::enumerate(m_declaration_value_count_buffer))
|
||||
{
|
||||
const auto values = value_span.subspan(offset, count);
|
||||
const auto value = values.front();
|
||||
|
||||
std::size_t begin_pos = value.data() - source_view.data();
|
||||
std::size_t end_pos = begin_pos + value.size();
|
||||
|
||||
begin_pos = source_view.rfind('\n', begin_pos);
|
||||
if (begin_pos == std::string_view::npos)
|
||||
{
|
||||
begin_pos = 0;
|
||||
}
|
||||
|
||||
end_pos = source_view.find('\n', end_pos);
|
||||
if (end_pos == std::string_view::npos)
|
||||
{
|
||||
end_pos = source_view.length();
|
||||
}
|
||||
else
|
||||
{
|
||||
end_pos += sizeof('\n');
|
||||
}
|
||||
|
||||
m_value_buffer[index] = source_view.substr(begin_pos, end_pos - begin_pos);
|
||||
|
||||
offset += count;
|
||||
}
|
||||
|
||||
const auto lines = std::span(m_value_buffer.data(), m_declaration_value_count_buffer.size());
|
||||
|
||||
const auto& first_line = lines.front();
|
||||
auto source_end = first_line.begin();
|
||||
auto line_end = first_line.end();
|
||||
|
||||
for (const auto& line : lines.subspan(1))
|
||||
{
|
||||
std::copy(line_end, line.begin(), source_end);
|
||||
source_end += line.size();
|
||||
line_end = line.end();
|
||||
}
|
||||
|
||||
const auto source_rest_size = m_buffer.source.end().base() - line_end;
|
||||
std::copy_n(line_end, source_rest_size, source_end);
|
||||
|
||||
source_end += source_rest_size;
|
||||
|
||||
m_buffer.source.resize(source_end - m_buffer.source.data());
|
||||
}
|
||||
|
||||
|
||||
|
||||
std::error_code assets::glsl_parser::read_file(
|
||||
const std::filesystem::path& filename,
|
||||
std::vector<char>& source
|
||||
) {
|
||||
auto file = std::ifstream{ filename };
|
||||
|
||||
if (not file.is_open())
|
||||
{
|
||||
return std::make_error_code(std::errc::no_such_file_or_directory);
|
||||
}
|
||||
|
||||
file.seekg(0, std::ios::end);
|
||||
const std::streampos size = file.tellg();
|
||||
|
||||
if (size == 0 or size == std::numeric_limits<std::streampos>::max())
|
||||
{
|
||||
return std::make_error_code(std::errc::invalid_seek);
|
||||
}
|
||||
|
||||
file.seekg(0, std::ios::beg);
|
||||
|
||||
source.clear();
|
||||
source.reserve(sizeof('\n') + size);
|
||||
source.push_back('\n');
|
||||
|
||||
std::copy(
|
||||
std::istreambuf_iterator<char>(file),
|
||||
std::istreambuf_iterator<char>(),
|
||||
std::back_inserter(source)
|
||||
);
|
||||
|
||||
file.close();
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
std::error_code assets::glsl_parser::prefetch(
|
||||
path_id_lookups&
|
||||
) {
|
||||
// Nothing to prefetch
|
||||
return {};
|
||||
}
|
||||
|
||||
std::error_code assets::glsl_parser::load(
|
||||
const path_id_lookups& lookups,
|
||||
shader_source_store& store,
|
||||
bool
|
||||
) {
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
m_path_buffer.clear();
|
||||
lookups.shader_sources.by_extension(".glsl", m_path_buffer);
|
||||
|
||||
auto store_mutex = std::mutex{};
|
||||
|
||||
std::for_each(
|
||||
std::execution::parallel_unsequenced_policy{},
|
||||
m_path_buffer.begin(),
|
||||
m_path_buffer.end(),
|
||||
parser_context{ store, store_mutex }
|
||||
);
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
bool assets::glsl_parser::parse_geometry_declaration(
|
||||
const std::span<const std::string_view> values,
|
||||
model_geometry::types& geometry_type
|
||||
) {
|
||||
|
||||
if (values.size() != 1)
|
||||
{
|
||||
ztu::logger::warn("Invalid geometry declaration: Expected exactly one token but got %.", values.size());
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto value = values.front();
|
||||
|
||||
if (const auto it = language::geometry_lookup.find(value); it != language::geometry_lookup.end())
|
||||
{
|
||||
geometry_type = it->second;
|
||||
}
|
||||
else
|
||||
{
|
||||
ztu::logger::warn("Invalid geometry declaration: Unknown geometry %.", value);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool assets::glsl_parser::parse_stage_declaration(
|
||||
const std::span<const std::string_view> values,
|
||||
shader_components::stage& stage
|
||||
) {
|
||||
|
||||
if (values.size() != 1)
|
||||
{
|
||||
ztu::logger::warn("Invalid stage declaration: Expected exactly one token but got %.", values.size());
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto value = values.front();
|
||||
|
||||
if (const auto it = language::stage_lookup.find(value); it != language::stage_lookup.end())
|
||||
{
|
||||
stage = it->second;
|
||||
}
|
||||
else
|
||||
{
|
||||
ztu::logger::warn("Invalid stage declaration: Unknown stage %.", value);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool assets::glsl_parser::parse_components_declaration(
|
||||
const std::span<const std::string_view> values,
|
||||
shader_source_data& buffer
|
||||
) {
|
||||
return parse_component_tokens(values, buffer.geometry_type, buffer.components);
|
||||
}
|
||||
|
||||
bool assets::glsl_parser::parse_static_enable_declaration(
|
||||
const std::span<const std::string_view> values,
|
||||
shader_source_data& buffer
|
||||
) {
|
||||
return parse_component_tokens(values, buffer.geometry_type, buffer.static_enable);
|
||||
}
|
||||
|
||||
bool assets::glsl_parser::parse_dynamic_enable_declaration(
|
||||
const std::span<const std::string_view> values,
|
||||
shader_source_data& buffer
|
||||
) {
|
||||
return parse_component_tokens(values, buffer.geometry_type, buffer.dynamic_enable);
|
||||
}
|
||||
|
||||
bool assets::glsl_parser::parse_component_tokens(
|
||||
const std::span<const std::string_view> values,
|
||||
const model_geometry::types geometry_type,
|
||||
shader_components::flags& components
|
||||
) {
|
||||
components = {};
|
||||
|
||||
const ztu::string_lookup<shader_components::flags>* component_lookup{};
|
||||
|
||||
switch (geometry_type)
|
||||
{
|
||||
case model_geometry::types::mesh:
|
||||
component_lookup = &language::mesh_feature_lookup;
|
||||
break;
|
||||
case model_geometry::types::point_cloud:
|
||||
component_lookup = &language::point_cloud_feature_lookup;
|
||||
break;
|
||||
default:
|
||||
ztu::logger::warn("Internal error: Unknown geometry index %.", static_cast<int>(geometry_type));
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
for (const auto value : values)
|
||||
{
|
||||
if (const auto it = component_lookup->find(value); it != component_lookup->end())
|
||||
{
|
||||
components |= it->second;
|
||||
}
|
||||
else
|
||||
{
|
||||
ztu::logger::warn("Ignoring unknown feature token %.", value);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
Reference in New Issue
Block a user