Further shader compilation development.
This commit is contained in:
zy4n
@@ -4,431 +4,239 @@
|
||||
#include <bits/ranges_algobase.h>
|
||||
#include "util/logger.hpp"
|
||||
|
||||
void zgl::shader_program_compiler::tokenize_declarations(
|
||||
std::string_view source_rest,
|
||||
std::vector<std::string_view> tokens,
|
||||
std::vector<std::size_t> declaration_token_counts,
|
||||
std::span<std::size_t> declaration_type_indices
|
||||
) {
|
||||
tokens.clear();
|
||||
declaration_token_counts.clear();
|
||||
std::ranges::fill(declaration_type_indices, static_cast<std::size_t>(metadata_declaration_type::invalid));
|
||||
|
||||
constexpr auto pragma_prefix = std::string_view("\n#pragma ");
|
||||
constexpr auto title_separator = ':';
|
||||
constexpr auto token_separator = ' ';
|
||||
struct prioritized_metadata_comparator
|
||||
{
|
||||
using type = zgl::shader_metadata;
|
||||
|
||||
auto offset = std::string_view::size_type{};
|
||||
|
||||
auto keyword = pragma_prefix;
|
||||
|
||||
while ((offset = source_rest.find(keyword)) != std::string_view::npos)
|
||||
bool operator()(const type& a, const type& b) const noexcept
|
||||
{
|
||||
const auto current_token_count = tokens.size();
|
||||
|
||||
auto line_end = source_rest.find('\n', offset);
|
||||
if (line_end == std::string_view::npos)
|
||||
if (a.geometry != b.geometry)
|
||||
{
|
||||
line_end = source_rest.length();
|
||||
return a.geometry > b.geometry;
|
||||
}
|
||||
|
||||
auto declaration = source_rest.substr(offset, line_end - offset);
|
||||
|
||||
if ((offset = declaration.find(title_separator)) == std::string_view::npos)
|
||||
if (a.stage != b.stage)
|
||||
{
|
||||
continue;
|
||||
return a.stage > b.stage;
|
||||
}
|
||||
|
||||
const auto title = declaration.substr(0, offset);
|
||||
if (const auto it = declaration_lookup.find(title); it != declaration_lookup.end())
|
||||
{
|
||||
const auto declaration_type = static_cast<std::size_t>(it->second);
|
||||
declaration_type_indices[declaration_type] = declaration_token_counts.size();
|
||||
}
|
||||
else
|
||||
{
|
||||
continue;
|
||||
}
|
||||
static constexpr auto more_features = std::popcount<zgl::shading::features::generic::type>;
|
||||
|
||||
declaration = declaration.substr(offset);
|
||||
|
||||
if (not declaration.empty() and declaration.front() == token_separator)
|
||||
{
|
||||
declaration = declaration.substr(sizeof(token_separator), declaration.length());
|
||||
}
|
||||
|
||||
while ((offset = declaration.find(token_separator)) != std::string_view::npos)
|
||||
{
|
||||
tokens.emplace_back(declaration.substr(0, offset));
|
||||
declaration = declaration.substr(offset + sizeof(token_separator));
|
||||
}
|
||||
|
||||
if (not declaration.empty())
|
||||
{
|
||||
tokens.emplace_back(declaration);
|
||||
}
|
||||
|
||||
declaration_token_counts.emplace_back(
|
||||
tokens.size() - current_token_count
|
||||
return std::ranges::lexicographical_compare(
|
||||
std::array{ a.dynamic_enable, a.static_enabled },
|
||||
std::array{ b.dynamic_enable, b.static_enabled },
|
||||
std::greater{},
|
||||
more_features,
|
||||
more_features
|
||||
);
|
||||
source_rest = source_rest.substr(line_end + sizeof('\n'));
|
||||
keyword = pragma_prefix.substr(sizeof('\n'));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
bool zgl::shader_program_compiler::parse_stage_declaration(
|
||||
std::span<const std::string_view> tokens,
|
||||
shader_program::metadata_type& metadata
|
||||
|
||||
zgl::shader_handle zgl::shader_program_compiler::find_shader(
|
||||
const shading::shader_requirements& requirements
|
||||
) {
|
||||
if (tokens.size() != 1)
|
||||
{
|
||||
ztu::logger::warn("Invalid stage declaration: Expected exactly one token but got %.", tokens.size());
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto token = tokens.front();
|
||||
auto handle = shader_handle{};
|
||||
|
||||
if (const auto it = stage_lookup.find(token); it != stage_lookup.end())
|
||||
{
|
||||
metadata.stage = it->second;
|
||||
}
|
||||
else
|
||||
{
|
||||
ztu::logger::warn("Invalid stage declaration: Unknown stage %.", token);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool zgl::shader_program_compiler::parse_geometry_declaration(
|
||||
std::span<const std::string_view> tokens,
|
||||
shader_program::metadata_type& metadata
|
||||
) {
|
||||
if (tokens.size() != 1)
|
||||
{
|
||||
ztu::logger::warn("Invalid geometry declaration: Expected exactly one token but got %.", tokens.size());
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto token = tokens.front();
|
||||
|
||||
if (const auto it = geometry_lookup.find(token); it != geometry_lookup.end())
|
||||
{
|
||||
metadata.geometry = it->second;
|
||||
}
|
||||
else
|
||||
{
|
||||
ztu::logger::warn("Invalid geometry declaration: Unknown geometry %.", token);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void zgl::shader_program_compiler::parse_feature_tokens(
|
||||
std::span<const std::string_view> tokens,
|
||||
const ztu::string_lookup<T>& feature_lookup,
|
||||
T& features
|
||||
) {
|
||||
features = {};
|
||||
|
||||
for (const auto token : tokens)
|
||||
{
|
||||
if (const auto it = feature_lookup.find(token); it != feature_lookup.end())
|
||||
auto shader_it = std::ranges::lower_bound(
|
||||
m_shader_lookup,
|
||||
std::pair{ requirements.geometry, requirements.stage },
|
||||
std::greater{},
|
||||
[](const shader_lookup_entry_type& entry)
|
||||
{
|
||||
features |= it->second;
|
||||
const auto& meta = entry.first;
|
||||
return std::pair{ meta.geometry, meta.stage };
|
||||
}
|
||||
else
|
||||
{
|
||||
ztu::logger::warn("Ignoring unknown feature token %.", token);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool zgl::shader_program_compiler::parse_features_declaration(
|
||||
std::span<const std::string_view> tokens,
|
||||
shader_program::metadata_type& metadata
|
||||
) {
|
||||
switch (metadata.geometry)
|
||||
{
|
||||
case shader_program::geometry::types::mesh:
|
||||
parse_feature_tokens(tokens, mesh_feature_lookup, metadata.feature_set.mesh.features);
|
||||
break;
|
||||
case shader_program::geometry::types::point_cloud:
|
||||
parse_feature_tokens(tokens, point_cloud_feature_lookup, metadata.feature_set.point_cloud.features);
|
||||
break;
|
||||
default:
|
||||
ztu::logger::warn("Internal error: Unknown geometry index %.", static_cast<int>(metadata.geometry));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool zgl::shader_program_compiler::parse_static_enable_declaration(
|
||||
std::span<const std::string_view> tokens,
|
||||
shader_program::metadata_type& metadata
|
||||
) {
|
||||
switch (metadata.geometry)
|
||||
{
|
||||
case shader_program::geometry::types::mesh:
|
||||
parse_feature_tokens(tokens, mesh_feature_lookup, metadata.feature_set.mesh.static_enable);
|
||||
break;
|
||||
case shader_program::geometry::types::point_cloud:
|
||||
parse_feature_tokens(tokens, point_cloud_feature_lookup, metadata.feature_set.point_cloud.static_enable);
|
||||
break;
|
||||
default:
|
||||
ztu::logger::warn("Internal error: Unknown geometry index %.", static_cast<int>(metadata.geometry));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool zgl::shader_program_compiler::parse_dynamic_enable_declaration(
|
||||
std::span<const std::string_view> tokens,
|
||||
shader_program::metadata_type& metadata
|
||||
) {
|
||||
switch (metadata.geometry)
|
||||
{
|
||||
case shader_program::geometry::types::mesh:
|
||||
parse_feature_tokens(tokens, mesh_feature_lookup, metadata.feature_set.mesh.dynamic_enable);
|
||||
break;
|
||||
case shader_program::geometry::types::point_cloud:
|
||||
parse_feature_tokens(tokens, point_cloud_feature_lookup, metadata.feature_set.point_cloud.dynamic_enable);
|
||||
break;
|
||||
default:
|
||||
ztu::logger::warn("Internal error: Unknown geometry index %.", static_cast<int>(metadata.geometry));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
std::optional<shader_program::metadata_type> zgl::shader_program_compiler::parse_metadata_from_tokens(
|
||||
std::span<const std::string_view> tokens,
|
||||
std::span<const std::size_t> declaration_token_counts,
|
||||
std::span<const std::size_t> declaration_type_indices
|
||||
) {
|
||||
using enum metadata_declaration_type;
|
||||
using namespace std::string_view_literals;
|
||||
|
||||
shader_program::metadata_type data;
|
||||
|
||||
for (const auto [ type, name, parser ] : {
|
||||
std::make_tuple(stage, "stage"sv, &parse_stage_declaration),
|
||||
std::make_tuple(geometry, "geometry"sv, &parse_geometry_declaration),
|
||||
std::make_tuple(features, "features"sv, &parse_features_declaration),
|
||||
std::make_tuple(static_enable, "static_enable"sv, &parse_static_enable_declaration),
|
||||
std::make_tuple(dynamic_enable, "dynamic_enable"sv, &parse_dynamic_enable_declaration)
|
||||
}) {
|
||||
const auto index = declaration_type_indices[static_cast<std::size_t>(type)];
|
||||
|
||||
if (index == static_cast<std::size_t>(invalid))
|
||||
{
|
||||
ztu::logger::warn("Shader metadata error: Missing % declaration.", name);
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const auto token_offset = std::accumulate(
|
||||
declaration_token_counts.begin(),
|
||||
declaration_token_counts.begin() + index,
|
||||
std::size_t{}
|
||||
);
|
||||
const auto token_count = declaration_token_counts[index];
|
||||
|
||||
if (not parser(tokens.subspan(token_offset, token_count), data))
|
||||
{
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
auto zgl::shader_program_compiler::find_compatible_shader_source(
|
||||
shader_program::metadata_type& requirements
|
||||
) {
|
||||
|
||||
const auto lower_it = std::ranges::lower_bound(
|
||||
shader_lookup,
|
||||
requirements,
|
||||
shader_program::metadata_type::feature_ignorant_less{},
|
||||
&std::pair<shader_program::metadata_type, dynamic_shader_source_store::id_type>::first
|
||||
);
|
||||
|
||||
auto generic_requirement_feature_set = requirements.generic_feature_set();
|
||||
const auto& required_features = generic_requirement_feature_set.features;
|
||||
dynamic_shader_source_store::id_type source_id{};
|
||||
shading::features::generic::type to_be_enabled{};
|
||||
|
||||
while (
|
||||
lower_it != shader_lookup.end() and
|
||||
lower_it->first.geometry == requirements.geometry and
|
||||
lower_it->first.stage == requirements.stage
|
||||
shader_it != m_shader_lookup.end() and
|
||||
shader_it->first.geometry == requirements.geometry and
|
||||
shader_it->first.stage == requirements.stage
|
||||
) {
|
||||
const auto& data = lower_it->first;
|
||||
const auto& [ features, static_enable, dynamic_enable ] = data.generic_feature_set();
|
||||
const auto& [ meta, data ] = *shader_it;
|
||||
|
||||
const auto missing_features = required_features & ~features;
|
||||
const auto unwanted_features = ~required_features & features;
|
||||
const auto fixed_unwanted_features = unwanted_features & ~static_enable & ~dynamic_enable;
|
||||
const auto unwanted_static_features = meta.static_enabled & ~requirements.features;
|
||||
const auto required_dynamic_features = requirements.features & ~meta.static_enabled;
|
||||
const auto missing_dynamic_features = required_dynamic_features & ~meta.dynamic_enable;
|
||||
|
||||
if (missing_features == 0 and fixed_unwanted_features == 0)
|
||||
if (unwanted_static_features == 0 and missing_dynamic_features == 0)
|
||||
{
|
||||
// Tell caller which features need to be toggled before compilation
|
||||
// and which features can still be dynamically enabled after compilation.
|
||||
generic_requirement_feature_set.static_enable = required_features & static_enable;
|
||||
generic_requirement_feature_set.dynamic_enable = dynamic_enable;
|
||||
requirements.from_generic_feature_set(generic_requirement_feature_set);
|
||||
to_be_enabled = req.features & static_enable;
|
||||
source_id = id;
|
||||
|
||||
return lower_it->second;
|
||||
res.static_enabled = features & ~dynamic_enable & ~unwanted_features;
|
||||
res.dynamic_enable = dynamic_enable;
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
++shader_it;
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void add_required_feature_defines(
|
||||
T toggle_flags,
|
||||
std::span<const std::string> defines,
|
||||
std::vector<const char*>& shader_strings
|
||||
) {
|
||||
auto index = std::size_t{};
|
||||
while (toggle_flags != T{})
|
||||
if (source_id)
|
||||
{
|
||||
if ((toggle_flags & T{ 1 }) != T{})
|
||||
const auto [ shader_source_it, source_found ] = shader_sources.find(source_id);
|
||||
if (source_found)
|
||||
{
|
||||
shader_strings.push_back(defines[index].c_str());
|
||||
get_define_strings(
|
||||
req.geometry,
|
||||
to_be_enabled,
|
||||
res.string_count,
|
||||
shader_strings
|
||||
);
|
||||
}
|
||||
toggle_flags >>= 1;
|
||||
++index;
|
||||
}
|
||||
|
||||
return handle;
|
||||
}
|
||||
|
||||
void zgl::shader_program_compiler::compile_shaders(
|
||||
const dynamic_shader_source_store& shader_sources,
|
||||
std::span<const shader_program::metadata_type> requirements_list,
|
||||
std::vector<shader_handle>& shader_handles
|
||||
std::span<const shading::shader_set_requirements> requirements,
|
||||
std::span<shader_set_metadata> metadata,
|
||||
std::span<shader_handle_set> shader_sets
|
||||
) {
|
||||
for (auto [ req, shader_set_meta, shader_set ] : std::ranges::views::zip(
|
||||
requirements,
|
||||
metadata,
|
||||
shader_sets
|
||||
)) {
|
||||
shader_set = {};
|
||||
shader_set_meta = {};
|
||||
|
||||
static constexpr auto max_shader_strings = std::max(
|
||||
mesh_feature_defines.size(),
|
||||
point_cloud_feature_defines.size()
|
||||
) + 1;
|
||||
auto shader_req = shading::shader_requirements{
|
||||
.geometry = req.geometry,
|
||||
.stage = {},
|
||||
.features = req.features
|
||||
};
|
||||
|
||||
std::vector<const char*> shader_strings;
|
||||
shader_strings.reserve(max_shader_strings);
|
||||
|
||||
|
||||
for (auto requirements : requirements_list)
|
||||
{
|
||||
auto shader_id = GLuint{};
|
||||
|
||||
const auto source_id = find_compatible_shader_source(requirements);
|
||||
if (not source_id)
|
||||
for (auto [ stage_index, handle ] : std::ranges::views::enumerate(shader_set.stages))
|
||||
{
|
||||
continue;
|
||||
shader_req.stage = static_cast<shading::stage::types>(stage_index);
|
||||
handle = find_shader(shader_req);
|
||||
if (not handle.valid())
|
||||
{
|
||||
m_source_requirement_buffer.push_back(shader_req);
|
||||
}
|
||||
}
|
||||
|
||||
const auto [ shader_source_it, source_found ] = shader_sources.find(*source_id);
|
||||
if (not source_found)
|
||||
{
|
||||
ztu::logger::warn("Missing shader source with id %.", *source_id);
|
||||
continue;
|
||||
}
|
||||
|
||||
const auto& shader_source = *shader_source_it;
|
||||
|
||||
if (shader_source.source.empty() and requirements.stage == shader_program::stage::types::geometry)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
shader_strings.clear();
|
||||
|
||||
switch (requirements.geometry) {
|
||||
case shader_program::geometry::types::mesh:
|
||||
add_required_feature_defines(requirements.feature_set.mesh.static_enable, mesh_feature_defines, shader_strings);
|
||||
break;
|
||||
case shader_program::geometry::types::point_cloud:
|
||||
add_required_feature_defines(requirements.feature_set.point_cloud.static_enable, point_cloud_feature_defines, shader_strings);
|
||||
break;
|
||||
default:
|
||||
std::unreachable();
|
||||
}
|
||||
|
||||
shader_strings.push_back(shader_source.source.data());
|
||||
|
||||
shader_id = glCreateShader(static_cast<GLenum>(requirements.stage));
|
||||
|
||||
glShaderSource(shader_id, static_cast<GLsizei>(shader_strings.size()), shader_strings.data(), nullptr);
|
||||
glCompileShader(shader_id);
|
||||
|
||||
GLint success;
|
||||
glGetShaderiv(shader_id, GL_COMPILE_STATUS, &success);
|
||||
|
||||
if (not success)
|
||||
{
|
||||
GLint log_length{};
|
||||
glGetShaderiv(shader_id, GL_INFO_LOG_LENGTH, &log_length);
|
||||
|
||||
auto log = std::string(log_length, ' ');
|
||||
glGetShaderInfoLog(shader_id, log_length, nullptr, log.data());
|
||||
|
||||
ztu::logger::error("Error while compiling shader:\n%", log);
|
||||
|
||||
glDeleteShader(shader_id);
|
||||
shader_id = GLuint{};
|
||||
}
|
||||
|
||||
shader_handles.emplace_back(shader_id);
|
||||
}
|
||||
|
||||
|
||||
m_preprocessed_shader_source_metadata_buffer.clear();
|
||||
m_preprocessed_shader_source_metadata_buffer.resize(m_source_requirement_buffer.size());
|
||||
m_source_strings_buffer.clear();
|
||||
|
||||
m_preprocessor.fetch_shader_sources(
|
||||
shader_sources,
|
||||
m_source_requirement_buffer,
|
||||
m_preprocessed_shader_source_metadata_buffer,
|
||||
m_source_strings_buffer
|
||||
);
|
||||
|
||||
auto source_strings_it = m_source_strings_buffer.begin();
|
||||
auto source_req_it = m_source_requirement_buffer.begin();
|
||||
auto source_meta_it = m_preprocessed_shader_source_metadata_buffer.begin();
|
||||
|
||||
const auto prev_shader_count = m_shader_lookup.size();
|
||||
|
||||
for (auto [ shader_set_meta, shader_set ] : std::ranges::views::zip( metadata, shader_sets))
|
||||
{
|
||||
for (auto [ stage_index, handle ] : std::ranges::views::enumerate(shader_set.stages))
|
||||
{
|
||||
if (not handle.valid())
|
||||
{
|
||||
if (source_meta_it->string_count > 0)
|
||||
{
|
||||
shader_data shader{};
|
||||
|
||||
if (compile_shader(
|
||||
gl_shader_types[stage_index],
|
||||
std::span(source_strings_it.base(), source_meta_it->string_count),
|
||||
shader
|
||||
)) {
|
||||
handle = shader.handle;
|
||||
auto shader_meta = shader_metadata{
|
||||
.geometry = source_req_it->geometry,
|
||||
.stage = source_req_it->stage,
|
||||
.static_enabled = source_meta_it->static_enabled,
|
||||
.dynamic_enable = source_meta_it->dynamic_enable
|
||||
};
|
||||
|
||||
shader_set_meta.static_enabled |= shader_meta.static_enabled;
|
||||
shader_set_meta.dynamic_enable |= shader_meta.dynamic_enable;
|
||||
|
||||
m_shader_lookup.emplace_back(shader_meta, std::move(shader));
|
||||
}
|
||||
}
|
||||
|
||||
source_strings_it += source_meta_it->string_count;
|
||||
++source_meta_it;
|
||||
++source_req_it;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const auto new_shaders = std::span(m_shader_lookup).subspan(prev_shader_count);
|
||||
|
||||
std::ranges::sort(
|
||||
new_shaders,
|
||||
prioritized_metadata_comparator{},
|
||||
&shader_lookup_entry_type::first
|
||||
);
|
||||
|
||||
std::ranges::inplace_merge(
|
||||
m_shader_lookup,
|
||||
m_shader_lookup.begin() + prev_shader_count,
|
||||
prioritized_metadata_comparator{},
|
||||
&shader_lookup_entry_type::first
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
void zgl::shader_program_compiler::register_shader_sources(
|
||||
bool zgl::shader_program_compiler::compile_shader(
|
||||
GLenum shader_type,
|
||||
std::span<const char*> source_strings,
|
||||
shader_data& shader
|
||||
) {
|
||||
|
||||
shader = shader_data(glCreateShader(shader_type));
|
||||
|
||||
glShaderSource(
|
||||
shader.handle.id,
|
||||
static_cast<GLsizei>(source_strings.size()),
|
||||
source_strings.data(),
|
||||
nullptr
|
||||
);
|
||||
|
||||
glCompileShader(shader.handle.id);
|
||||
|
||||
GLint success;
|
||||
glGetShaderiv(shader.handle.id, GL_COMPILE_STATUS, &success);
|
||||
|
||||
if (not success)
|
||||
{
|
||||
GLint log_length{};
|
||||
glGetShaderiv(shader.handle.id, GL_INFO_LOG_LENGTH, &log_length);
|
||||
|
||||
auto log = std::string(log_length, ' ');
|
||||
glGetShaderInfoLog(shader.handle.id, log_length, nullptr, log.data());
|
||||
|
||||
ztu::logger::error("Error while compiling shader:\n%", log);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void zgl::shader_program_compiler::preprocess(
|
||||
const dynamic_shader_source_store& shader_sources
|
||||
) {
|
||||
std::vector<std::string_view> tokens;
|
||||
std::vector<std::size_t> declaration_token_counts;
|
||||
std::array<std::size_t, 4> declaration_type_indices;
|
||||
|
||||
tokens.reserve(32);
|
||||
declaration_token_counts.reserve(4);
|
||||
|
||||
for (const auto& [ id, shader_source ] : shader_sources)
|
||||
{
|
||||
tokenize_declarations(
|
||||
shader_source,
|
||||
tokens,
|
||||
declaration_token_counts,
|
||||
declaration_type_indices
|
||||
);
|
||||
|
||||
const auto metadata = parse_metadata_from_tokens(
|
||||
tokens,
|
||||
declaration_token_counts,
|
||||
declaration_type_indices
|
||||
);
|
||||
|
||||
if (not metadata)
|
||||
{
|
||||
ztu::logger::warn("Ignoring shader % as it contains malformed metadata.", id);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Sorted insert should be faster than std::sort and std::unique
|
||||
// for small numbers of elements and high numbers of duplicates.
|
||||
const auto it = std::ranges::upper_bound(
|
||||
shader_lookup,
|
||||
*metadata,
|
||||
std::less{},
|
||||
&std::pair<shader_program::metadata_type, dynamic_shader_source_store::id_type>::first
|
||||
);
|
||||
|
||||
if (it != shader_lookup.end() and it->first == *metadata)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
shader_lookup.emplace(it, *metadata, id);
|
||||
}
|
||||
m_preprocessor.preprocess(shader_sources);
|
||||
}
|
||||
453
source/opengl/data_uploaders/shader_preprocessor.cpp
Normal file
453
source/opengl/data_uploaders/shader_preprocessor.cpp
Normal file
@@ -0,0 +1,453 @@
|
||||
#include "opengl/data_uploaders/shader_preprocessor.hpp"
|
||||
|
||||
#include "util/logger.hpp"
|
||||
#include <algorithm>
|
||||
#include <cassert>
|
||||
#include <numeric>
|
||||
|
||||
#include "opengl/shading/shader_metadata_language.hpp"
|
||||
|
||||
struct prioritized_feature_set_comparator
|
||||
{
|
||||
using type = zgl::shader_features_set<zgl::shading::features::generic::type>;
|
||||
|
||||
bool operator()(const type& a, const type& b) const noexcept
|
||||
{
|
||||
static constexpr auto more_features = std::popcount<zgl::shading::features::generic::type>;
|
||||
|
||||
return std::ranges::lexicographical_compare(
|
||||
std::array{ a.dynamic_enable, a.features, a.static_enable },
|
||||
std::array{ b.dynamic_enable, b.features, b.static_enable },
|
||||
std::greater{},
|
||||
more_features,
|
||||
more_features
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
struct prioritized_metadata_comparator
|
||||
{
|
||||
using type = zgl::shader_source_metadata;
|
||||
|
||||
bool operator()(const type& a, const type& b) const noexcept
|
||||
{
|
||||
if (a.geometry != b.geometry)
|
||||
{
|
||||
return a.geometry > b.geometry;
|
||||
}
|
||||
|
||||
if (a.stage != b.stage)
|
||||
{
|
||||
return a.stage > b.stage;
|
||||
}
|
||||
|
||||
const auto features_a = a.generic_feature_set();
|
||||
const auto features_b = b.generic_feature_set();
|
||||
|
||||
return feature_set_comparator(features_a, features_b);
|
||||
}
|
||||
|
||||
private:
|
||||
prioritized_feature_set_comparator feature_set_comparator{};
|
||||
};
|
||||
|
||||
|
||||
|
||||
void zgl::shader_preprocessor::preprocess(
|
||||
const dynamic_shader_source_store& shader_sources
|
||||
) {
|
||||
namespace language = shading::shader_metadata_language;
|
||||
|
||||
for (const auto& [ id, shader_source ] : shader_sources)
|
||||
{
|
||||
m_value_token_buffer.clear();
|
||||
m_declaration_token_count_buffer.clear();
|
||||
std::ranges::fill(
|
||||
m_declaration_type_index_buffer,
|
||||
static_cast<std::size_t>(language::declaration_type::invalid)
|
||||
);
|
||||
|
||||
tokenize_declarations(shader_source);
|
||||
|
||||
const auto metadata = parse_metadata_from_tokens();
|
||||
|
||||
if (not metadata)
|
||||
{
|
||||
ztu::logger::warn("Ignoring shader % as it contains malformed metadata.", id);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Sorted insert should be faster than std::sort and std::unique
|
||||
// for small numbers of elements and high numbers of duplicates.
|
||||
const auto it = std::ranges::upper_bound(
|
||||
m_shader_source_lookup,
|
||||
*metadata,
|
||||
prioritized_metadata_comparator{},
|
||||
&std::pair<shader_source_metadata, dynamic_shader_source_store::id_type>::first
|
||||
);
|
||||
|
||||
if (it != m_shader_source_lookup.end() and it->first == *metadata)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
m_shader_source_lookup.emplace(it, *metadata, id);
|
||||
}
|
||||
}
|
||||
|
||||
void zgl::shader_preprocessor::fetch_shader_sources(
|
||||
const dynamic_shader_source_store& shader_sources,
|
||||
std::span<const shading::shader_source_requirements> requirements,
|
||||
std::span<preprocessed_shader_source_metadata> metadata,
|
||||
std::vector<const char*>& shader_strings
|
||||
) {
|
||||
|
||||
assert(requirements.size() == metadata.size());
|
||||
|
||||
static constexpr auto max_shader_strings = std::max(
|
||||
mesh_feature_defines.size(),
|
||||
point_cloud_feature_defines.size()
|
||||
) + 1;
|
||||
|
||||
shader_strings.reserve(max_shader_strings);
|
||||
|
||||
std::ranges::transform(
|
||||
requirements,
|
||||
metadata.begin(),
|
||||
[&](const shading::shader_source_requirements& req)
|
||||
{
|
||||
auto res = preprocessed_shader_source_metadata{};
|
||||
|
||||
auto source_it = std::ranges::lower_bound(
|
||||
m_shader_source_lookup,
|
||||
std::pair{ req.geometry, req.stage },
|
||||
std::greater{},
|
||||
[](const source_lookup_entry_type& entry)
|
||||
{
|
||||
const auto& meta = entry.first;
|
||||
return std::pair{ meta.geometry, meta.stage };
|
||||
}
|
||||
);
|
||||
|
||||
dynamic_shader_source_store::id_type source_id{};
|
||||
shading::features::generic::type to_be_enabled{};
|
||||
|
||||
while (
|
||||
source_it != m_shader_source_lookup.end() and
|
||||
source_it->first.geometry == req.geometry and
|
||||
source_it->first.stage == req.stage
|
||||
) {
|
||||
const auto& [ meta, id ] = *source_it;
|
||||
const auto& [ features, static_enable, dynamic_enable ] = meta.generic_feature_set();
|
||||
|
||||
const auto missing_features = req.features & ~features;
|
||||
const auto unwanted_features = ~req.features & features;
|
||||
const auto fixed_unwanted_features = unwanted_features & ~static_enable & ~dynamic_enable;
|
||||
|
||||
if (missing_features == 0 and fixed_unwanted_features == 0)
|
||||
{
|
||||
to_be_enabled = req.features & static_enable;
|
||||
source_id = id;
|
||||
|
||||
res.static_enabled = features & ~dynamic_enable & ~unwanted_features;
|
||||
res.dynamic_enable = dynamic_enable;
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
++source_it;
|
||||
}
|
||||
|
||||
if (source_id)
|
||||
{
|
||||
const auto [ shader_source_it, source_found ] = shader_sources.find(source_id);
|
||||
if (source_found)
|
||||
{
|
||||
get_define_strings(
|
||||
req.geometry,
|
||||
to_be_enabled,
|
||||
res.string_count,
|
||||
shader_strings
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
void zgl::shader_preprocessor::get_define_strings(
|
||||
const shading::model_geometry::types geometry,
|
||||
shading::features::generic::type features,
|
||||
shading::features::generic::type& feature_count,
|
||||
std::vector<const char*>& defines
|
||||
) {
|
||||
std::span<const char*> all_defines;
|
||||
|
||||
switch (geometry)
|
||||
{
|
||||
case shading::model_geometry::types::mesh:
|
||||
all_defines = mesh_feature_defines;
|
||||
break;
|
||||
case shading::model_geometry::types::point_cloud:
|
||||
all_defines = point_cloud_feature_defines;
|
||||
break;
|
||||
default:
|
||||
std::unreachable();
|
||||
}
|
||||
|
||||
auto index = std::size_t{};
|
||||
while (features != 0)
|
||||
{
|
||||
if ((features & 1) != 0)
|
||||
{
|
||||
defines.push_back(all_defines[index]);
|
||||
++feature_count;
|
||||
}
|
||||
features >>= 1;
|
||||
++index;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void zgl::shader_preprocessor::tokenize_declarations(
|
||||
std::string_view source_rest
|
||||
) {
|
||||
namespace language = shading::shader_metadata_language;
|
||||
|
||||
auto offset = std::string_view::size_type{};
|
||||
|
||||
auto keyword = language::declaration_prefix;
|
||||
|
||||
while ((offset = source_rest.find(keyword)) != std::string_view::npos)
|
||||
{
|
||||
const auto current_token_count = m_value_token_buffer.size();
|
||||
|
||||
auto line_end = source_rest.find('\n', offset);
|
||||
if (line_end == std::string_view::npos)
|
||||
{
|
||||
line_end = source_rest.length();
|
||||
}
|
||||
|
||||
auto declaration = source_rest.substr(offset, line_end - offset);
|
||||
|
||||
if ((offset = declaration.find(language::title_separator)) == std::string_view::npos)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
const auto title = declaration.substr(0, offset);
|
||||
if (const auto it = language::declaration_lookup.find(title); it != language::declaration_lookup.end())
|
||||
{
|
||||
const auto declaration_type = static_cast<std::size_t>(it->second);
|
||||
m_declaration_type_index_buffer[declaration_type] = m_declaration_token_count_buffer.size();
|
||||
}
|
||||
else
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
declaration = declaration.substr(offset);
|
||||
|
||||
if (not declaration.empty() and declaration.front() == language::value_separator)
|
||||
{
|
||||
declaration = declaration.substr(sizeof(language::value_separator), declaration.length());
|
||||
}
|
||||
|
||||
while ((offset = declaration.find(language::value_separator)) != std::string_view::npos)
|
||||
{
|
||||
m_value_token_buffer.emplace_back(declaration.substr(0, offset));
|
||||
declaration = declaration.substr(offset + sizeof(language::value_separator));
|
||||
}
|
||||
|
||||
if (not declaration.empty())
|
||||
{
|
||||
m_value_token_buffer.emplace_back(declaration);
|
||||
}
|
||||
|
||||
m_declaration_token_count_buffer.emplace_back(
|
||||
m_value_token_buffer.size() - current_token_count
|
||||
);
|
||||
source_rest = source_rest.substr(line_end + sizeof('\n'));
|
||||
keyword = language::declaration_prefix.substr(sizeof('\n'));
|
||||
}
|
||||
}
|
||||
|
||||
bool zgl::shader_preprocessor::parse_stage_declaration(
|
||||
std::span<const std::string_view> values,
|
||||
shader_source_metadata& metadata
|
||||
) {
|
||||
namespace language = shading::shader_metadata_language;
|
||||
|
||||
if (values.size() != 1)
|
||||
{
|
||||
ztu::logger::warn("Invalid stage declaration: Expected exactly one token but got %.", values.size());
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto value = values.front();
|
||||
|
||||
if (const auto it = language::stage_lookup.find(value); it != language::stage_lookup.end())
|
||||
{
|
||||
metadata.stage = it->second;
|
||||
}
|
||||
else
|
||||
{
|
||||
ztu::logger::warn("Invalid stage declaration: Unknown stage %.", value);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool zgl::shader_preprocessor::parse_geometry_declaration(
|
||||
std::span<const std::string_view> values,
|
||||
shader_source_metadata& metadata
|
||||
) {
|
||||
namespace language = shading::shader_metadata_language;
|
||||
|
||||
if (values.size() != 1)
|
||||
{
|
||||
ztu::logger::warn("Invalid geometry declaration: Expected exactly one token but got %.", values.size());
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto value = values.front();
|
||||
|
||||
if (const auto it = language::geometry_lookup.find(value); it != language::geometry_lookup.end())
|
||||
{
|
||||
metadata.geometry = it->second;
|
||||
}
|
||||
else
|
||||
{
|
||||
ztu::logger::warn("Invalid geometry declaration: Unknown geometry %.", value);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void zgl::shader_preprocessor::parse_feature_tokens(
|
||||
std::span<const std::string_view> values,
|
||||
const ztu::string_lookup<T>& feature_lookup,
|
||||
T& features
|
||||
) {
|
||||
features = {};
|
||||
|
||||
for (const auto value : values)
|
||||
{
|
||||
if (const auto it = feature_lookup.find(value); it != feature_lookup.end())
|
||||
{
|
||||
features |= it->second;
|
||||
}
|
||||
else
|
||||
{
|
||||
ztu::logger::warn("Ignoring unknown feature token %.", value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool zgl::shader_preprocessor::parse_features_declaration(
|
||||
std::span<const std::string_view> values,
|
||||
shader_source_metadata& metadata
|
||||
) {
|
||||
namespace language = shading::shader_metadata_language;
|
||||
|
||||
switch (metadata.geometry)
|
||||
{
|
||||
case shading::model_geometry::types::mesh:
|
||||
parse_feature_tokens(values, language::mesh_feature_lookup, metadata.feature_set.mesh.features);
|
||||
break;
|
||||
case shading::model_geometry::types::point_cloud:
|
||||
parse_feature_tokens(values, language::point_cloud_feature_lookup, metadata.feature_set.point_cloud.features);
|
||||
break;
|
||||
default:
|
||||
ztu::logger::warn("Internal error: Unknown geometry index %.", static_cast<int>(metadata.geometry));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool zgl::shader_preprocessor::parse_static_enable_declaration(
|
||||
std::span<const std::string_view> values,
|
||||
shader_source_metadata& metadata
|
||||
) {
|
||||
namespace language = shading::shader_metadata_language;
|
||||
|
||||
switch (metadata.geometry)
|
||||
{
|
||||
case shading::model_geometry::types::mesh:
|
||||
parse_feature_tokens(values, language::mesh_feature_lookup, metadata.feature_set.mesh.static_enable);
|
||||
break;
|
||||
case shading::model_geometry::types::point_cloud:
|
||||
parse_feature_tokens(values, language::point_cloud_feature_lookup, metadata.feature_set.point_cloud.static_enable);
|
||||
break;
|
||||
default:
|
||||
ztu::logger::warn("Internal error: Unknown geometry index %.", static_cast<int>(metadata.geometry));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool zgl::shader_preprocessor::parse_dynamic_enable_declaration(
|
||||
std::span<const std::string_view> values,
|
||||
shader_source_metadata& metadata
|
||||
) {
|
||||
namespace language = shading::shader_metadata_language;
|
||||
|
||||
switch (metadata.geometry)
|
||||
{
|
||||
case shading::model_geometry::types::mesh:
|
||||
parse_feature_tokens(values, language::mesh_feature_lookup, metadata.feature_set.mesh.dynamic_enable);
|
||||
break;
|
||||
case shading::model_geometry::types::point_cloud:
|
||||
parse_feature_tokens(values, language::point_cloud_feature_lookup, metadata.feature_set.point_cloud.dynamic_enable);
|
||||
break;
|
||||
default:
|
||||
ztu::logger::warn("Internal error: Unknown geometry index %.", static_cast<int>(metadata.geometry));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
std::optional<zgl::shader_source_metadata> zgl::shader_preprocessor::parse_metadata_from_tokens()
|
||||
{
|
||||
namespace language = shading::shader_metadata_language;
|
||||
using namespace std::string_view_literals;
|
||||
|
||||
shader_source_metadata data;
|
||||
|
||||
for (const auto [ type, name, parser ] :
|
||||
{
|
||||
std::make_tuple(language::declaration_type::stage, "stage"sv, &parse_stage_declaration),
|
||||
std::make_tuple(language::declaration_type::geometry, "geometry"sv, &parse_geometry_declaration),
|
||||
std::make_tuple(language::declaration_type::features, "features"sv, &parse_features_declaration),
|
||||
std::make_tuple(language::declaration_type::static_enable, "static_enable"sv, &parse_static_enable_declaration),
|
||||
std::make_tuple(language::declaration_type::dynamic_enable, "dynamic_enable"sv, &parse_dynamic_enable_declaration)
|
||||
}
|
||||
) {
|
||||
const auto index = m_declaration_type_index_buffer[static_cast<std::size_t>(type)];
|
||||
|
||||
if (index == static_cast<std::size_t>(language::declaration_type::invalid))
|
||||
{
|
||||
ztu::logger::warn("Shader metadata error: Missing % declaration.", name);
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const auto value_token_offset = std::accumulate(
|
||||
m_declaration_token_count_buffer.begin(),
|
||||
m_declaration_token_count_buffer.begin() + index,
|
||||
std::size_t{}
|
||||
);
|
||||
const auto value_token_count = m_declaration_token_count_buffer[index];
|
||||
|
||||
if (not parser(std::span(m_value_token_buffer).subspan(value_token_offset, value_token_count), data))
|
||||
{
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
Reference in New Issue
Block a user