zy4n/Z3D
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

In the middle of multithreading parsers.

Browse Source
This commit is contained in:
zy4n
2025-03-30 22:38:06 +02:00
parent d18b40a7fc
commit 144126ee7a
80 changed files with 2904 additions and 1450 deletions
Download Patch File Download Diff File Expand all files Collapse all files

375
source/opengl/data_managers/shader_source_manager.cpp
View File

@@ -4,10 +4,11 @@
#include <algorithm>
#include <cassert>
#include <numeric>
#include <utility>
#include "opengl/shading/shader_metadata_language.hpp"
static auto mesh_feature_defines = std::array{
static auto mesh_component_defines = std::array{
"#define FACE\n",
"#define LINE\n",
"#define POINT\n",
@@ -19,7 +20,7 @@ static auto mesh_feature_defines = std::array{
"#define U_RGBA\n",
};
static auto point_cloud_feature_defines = std::array{
static auto point_cloud_component_defines = std::array{
"#define SQUARE\n",
"#define LIGHTING\n",
"#define V_L\n",
@@ -29,34 +30,15 @@ static auto point_cloud_feature_defines = std::array{
"#define RAINBOW\n"
};
struct prioritized_feature_set_comparator
{
using type = zgl::shader_features_set<zgl::shading::features::generic::type>;
bool operator()(const type& a, const type& b) const noexcept
{
static constexpr auto more_features = std::popcount<zgl::shading::features::generic::type>;
return std::ranges::lexicographical_compare(
std::array{ a.dynamic_enable, a.features, a.static_enable },
std::array{ b.dynamic_enable, b.features, b.static_enable },
std::greater{},
more_features,
more_features
);
}
};
struct prioritized_metadata_comparator
{
using type = zgl::shader_source_metadata;
bool operator()(const type& a, const type& b) const noexcept
{
if (a.geometry != b.geometry)
if (a.geometry_type != b.geometry_type)
{
return a.geometry > b.geometry;
return a.geometry_type > b.geometry_type;
}
if (a.stage != b.stage)
@@ -64,62 +46,56 @@ struct prioritized_metadata_comparator
return a.stage > b.stage;
}
const auto features_a = a.generic_feature_set();
const auto features_b = b.generic_feature_set();
static constexpr auto more_components = std::popcount<assets::shader_components::flags>;
return feature_set_comparator(features_a, features_b);
// Sort by dynamic components first to make sure when compatible components are found
// the compiled shader will have maximum dynamic compatibility.
return std::ranges::lexicographical_compare(
std::array{ a.dynamic_enable, a.components, a.static_enable },
std::array{ b.dynamic_enable, b.components, b.static_enable },
std::greater{},
more_components,
more_components
);
}
private:
prioritized_feature_set_comparator feature_set_comparator{};
};
void zgl::shader_source_manager::process(const store_type& shader_sources)
{
namespace language = shading::shader_metadata_language;
for (const auto& [ id, shader_source ] : shader_sources)
{
m_value_token_buffer.clear();
m_declaration_token_count_buffer.clear();
std::ranges::fill(
m_declaration_type_index_buffer,
static_cast<std::size_t>(language::declaration_type::invalid)
);
tokenize_declarations(shader_source);
const auto metadata = parse_metadata_from_tokens();
if (not metadata)
{
ztu::logger::warn("Ignoring shader % as it contains malformed metadata.", id);
continue;
}
const auto meta = metadata_type{
.stage = shader_source.stage,
.components = shader_source.components,
.static_enable = shader_source.static_enable,
.dynamic_enable = shader_source.dynamic_enable
};
// Sorted insert should be faster than std::sort and std::unique
// for small numbers of elements and high numbers of duplicates.
const auto it = std::ranges::upper_bound(
m_shader_source_lookup,
*metadata,
meta,
prioritized_metadata_comparator{},
&std::pair<shader_source_metadata, dynamic_shader_source_store::id_type>::first
&entry_type::first
);
if (it != m_shader_source_lookup.end() and it->first == *metadata)
if (it != m_shader_source_lookup.end() and it->first == meta)
{
continue;
}
m_shader_source_lookup.emplace(it, *metadata, id);
m_shader_source_lookup.emplace(it, meta, id);
}
}
void zgl::shader_source_manager::get_shader_sources(
void zgl::shader_source_manager::fetch(
const assets::shader_source_store& shader_sources,
std::span<const shading::shader_source_requirements> requirements,
std::span<const shading::shader_requirements> requirements,
std::span<preprocessed_shader_source_metadata> metadata,
std::vector<const char*>& shader_strings
) {
@@ -127,8 +103,8 @@ void zgl::shader_source_manager::get_shader_sources(
assert(requirements.size() == metadata.size());
static constexpr auto max_shader_strings = std::max(
mesh_feature_defines.size(),
point_cloud_feature_defines.size()
mesh_component_defines.size(),
point_cloud_component_defines.size()
) + 1;
shader_strings.reserve(max_shader_strings);
@@ -136,43 +112,42 @@ void zgl::shader_source_manager::get_shader_sources(
std::ranges::transform(
requirements,
metadata.begin(),
[&](const shading::shader_source_requirements& req)
[&](const shading::shader_requirements& req)
{
auto res = preprocessed_shader_source_metadata{};
auto source_it = std::ranges::lower_bound(
m_shader_source_lookup,
std::pair{ req.geometry, req.stage },
std::pair{ req.geometry_type, req.stage },
std::greater{},
[](const source_lookup_entry_type& entry)
[](const entry_type& entry)
{
const auto& meta = entry.first;
return std::pair{ meta.geometry, meta.stage };
return std::pair{ meta.geometry_type, meta.stage };
}
);
assets::shader_source_store::id_type source_id{};
shading::features::generic::type to_be_enabled{};
assets::shader_components::flags to_be_enabled{};
while (
source_it != m_shader_source_lookup.end() and
source_it->first.geometry == req.geometry and
source_it->first.geometry == req.geometry_type and
source_it->first.stage == req.stage
) {
const auto& [ meta, id ] = *source_it;
const auto& [ features, static_enable, dynamic_enable ] = meta.generic_feature_set();
const auto missing_features = req.features & ~features;
const auto unwanted_features = ~req.features & features;
const auto fixed_unwanted_features = unwanted_features & ~static_enable & ~dynamic_enable;
const auto missing_components = req.components & ~meta.components;
const auto unwanted_components = ~req.components & meta.components;
const auto fixed_unwanted_components = unwanted_components & ~meta.static_enable & ~meta.dynamic_enable;
if (missing_features == 0 and fixed_unwanted_features == 0)
if (missing_components == 0 and fixed_unwanted_components == 0)
{
to_be_enabled = req.features & static_enable;
to_be_enabled = req.components & meta.static_enable;
source_id = id;
res.static_enabled = features & ~dynamic_enable & ~unwanted_features;
res.dynamic_enable = dynamic_enable;
res.static_enabled = meta.commponents & ~meta.dynamic_enable & ~unwanted_components;
res.dynamic_enable = meta.dynamic_enable;
break;
}
@@ -186,7 +161,7 @@ void zgl::shader_source_manager::get_shader_sources(
if (source_found)
{
get_define_strings(
req.geometry,
req.geometry_type,
to_be_enabled,
res.string_count,
shader_strings
@@ -200,276 +175,34 @@ void zgl::shader_source_manager::get_shader_sources(
}
void zgl::shader_source_manager::get_define_strings(
const shading::model_geometry::types geometry,
shading::features::generic::type features,
shading::features::generic::type& feature_count,
const assets::model_geometry::types geometry_type,
assets::shader_components::flags components,
assets::shader_components::flags& component_count,
std::vector<const char*>& defines
) {
std::span<const char*> all_defines;
switch (geometry)
switch (geometry_type)
{
case shading::model_geometry::types::mesh:
all_defines = mesh_feature_defines;
all_defines = mesh_component_defines;
break;
case shading::model_geometry::types::point_cloud:
all_defines = point_cloud_feature_defines;
all_defines = point_cloud_component_defines;
break;
default:
std::unreachable();
}
auto index = std::size_t{};
while (features != 0)
while (components != 0)
{
if ((features & 1) != 0)
if ((components & 1) != 0)
{
defines.push_back(all_defines[index]);
++feature_count;
++component_count;
}
features >>= 1;
components >>= 1;
++index;
}
}
void zgl::shader_source_manager::tokenize_declarations(
std::string_view source_rest
) {
namespace language = shading::shader_metadata_language;
auto offset = std::string_view::size_type{};
auto keyword = language::declaration_prefix;
while ((offset = source_rest.find(keyword)) != std::string_view::npos)
{
const auto current_token_count = m_value_token_buffer.size();
auto line_end = source_rest.find('\n', offset);
if (line_end == std::string_view::npos)
{
line_end = source_rest.length();
}
auto declaration = source_rest.substr(offset, line_end - offset);
if ((offset = declaration.find(language::title_separator)) == std::string_view::npos)
{
continue;
}
const auto title = declaration.substr(0, offset);
if (const auto it = language::declaration_lookup.find(title); it != language::declaration_lookup.end())
{
const auto declaration_type = static_cast<std::size_t>(it->second);
m_declaration_type_index_buffer[declaration_type] = m_declaration_token_count_buffer.size();
}
else
{
continue;
}
declaration = declaration.substr(offset);
if (not declaration.empty() and declaration.front() == language::value_separator)
{
declaration = declaration.substr(sizeof(language::value_separator), declaration.length());
}
while ((offset = declaration.find(language::value_separator)) != std::string_view::npos)
{
m_value_token_buffer.emplace_back(declaration.substr(0, offset));
declaration = declaration.substr(offset + sizeof(language::value_separator));
}
if (not declaration.empty())
{
m_value_token_buffer.emplace_back(declaration);
}
m_declaration_token_count_buffer.emplace_back(
m_value_token_buffer.size() - current_token_count
);
source_rest = source_rest.substr(line_end + sizeof('\n'));
keyword = language::declaration_prefix.substr(sizeof('\n'));
}
}
bool zgl::shader_source_manager::parse_stage_declaration(
std::span<const std::string_view> values,
shader_source_metadata& metadata
) {
namespace language = shading::shader_metadata_language;
if (values.size() != 1)
{
ztu::logger::warn("Invalid stage declaration: Expected exactly one token but got %.", values.size());
return false;
}
const auto value = values.front();
if (const auto it = language::stage_lookup.find(value); it != language::stage_lookup.end())
{
metadata.stage = it->second;
}
else
{
ztu::logger::warn("Invalid stage declaration: Unknown stage %.", value);
return false;
}
return true;
}
bool zgl::shader_source_manager::parse_geometry_declaration(
std::span<const std::string_view> values,
shader_source_metadata& metadata
) {
namespace language = shading::shader_metadata_language;
if (values.size() != 1)
{
ztu::logger::warn("Invalid geometry declaration: Expected exactly one token but got %.", values.size());
return false;
}
const auto value = values.front();
if (const auto it = language::geometry_lookup.find(value); it != language::geometry_lookup.end())
{
metadata.geometry = it->second;
}
else
{
ztu::logger::warn("Invalid geometry declaration: Unknown geometry %.", value);
return false;
}
return true;
}
template<typename T>
void zgl::shader_source_manager::parse_feature_tokens(
std::span<const std::string_view> values,
const ztu::string_lookup<T>& feature_lookup,
T& features
) {
features = {};
for (const auto value : values)
{
if (const auto it = feature_lookup.find(value); it != feature_lookup.end())
{
features |= it->second;
}
else
{
ztu::logger::warn("Ignoring unknown feature token %.", value);
}
}
}
bool zgl::shader_source_manager::parse_features_declaration(
std::span<const std::string_view> values,
shader_source_metadata& metadata
) {
namespace language = shading::shader_metadata_language;
switch (metadata.geometry)
{
case shading::model_geometry::types::mesh:
parse_feature_tokens(values, language::mesh_feature_lookup, metadata.feature_set.mesh.features);
break;
case shading::model_geometry::types::point_cloud:
parse_feature_tokens(values, language::point_cloud_feature_lookup, metadata.feature_set.point_cloud.features);
break;
default:
ztu::logger::warn("Internal error: Unknown geometry index %.", static_cast<int>(metadata.geometry));
return false;
}
return true;
}
bool zgl::shader_source_manager::parse_static_enable_declaration(
std::span<const std::string_view> values,
shader_source_metadata& metadata
) {
namespace language = shading::shader_metadata_language;
switch (metadata.geometry)
{
case shading::model_geometry::types::mesh:
parse_feature_tokens(values, language::mesh_feature_lookup, metadata.feature_set.mesh.static_enable);
break;
case shading::model_geometry::types::point_cloud:
parse_feature_tokens(values, language::point_cloud_feature_lookup, metadata.feature_set.point_cloud.static_enable);
break;
default:
ztu::logger::warn("Internal error: Unknown geometry index %.", static_cast<int>(metadata.geometry));
return false;
}
return true;
}
bool zgl::shader_source_manager::parse_dynamic_enable_declaration(
std::span<const std::string_view> values,
shader_source_metadata& metadata
) {
namespace language = shading::shader_metadata_language;
switch (metadata.geometry)
{
case shading::model_geometry::types::mesh:
parse_feature_tokens(values, language::mesh_feature_lookup, metadata.feature_set.mesh.dynamic_enable);
break;
case shading::model_geometry::types::point_cloud:
parse_feature_tokens(values, language::point_cloud_feature_lookup, metadata.feature_set.point_cloud.dynamic_enable);
break;
default:
ztu::logger::warn("Internal error: Unknown geometry index %.", static_cast<int>(metadata.geometry));
return false;
}
return true;
}
std::optional<zgl::shader_source_metadata> zgl::shader_source_manager::parse_metadata_from_tokens()
{
namespace language = shading::shader_metadata_language;
using namespace std::string_view_literals;
shader_source_metadata data;
for (const auto [ type, name, parser ] :
{
std::make_tuple(language::declaration_type::stage, "stage"sv, &parse_stage_declaration),
std::make_tuple(language::declaration_type::geometry, "geometry"sv, &parse_geometry_declaration),
std::make_tuple(language::declaration_type::features, "features"sv, &parse_features_declaration),
std::make_tuple(language::declaration_type::static_enable, "static_enable"sv, &parse_static_enable_declaration),
std::make_tuple(language::declaration_type::dynamic_enable, "dynamic_enable"sv, &parse_dynamic_enable_declaration)
}
) {
const auto index = m_declaration_type_index_buffer[static_cast<std::size_t>(type)];
if (index == static_cast<std::size_t>(language::declaration_type::invalid))
{
ztu::logger::warn("Shader metadata error: Missing % declaration.", name);
return std::nullopt;
}
const auto value_token_offset = std::accumulate(
m_declaration_token_count_buffer.begin(),
m_declaration_token_count_buffer.begin() + index,
std::size_t{}
);
const auto value_token_count = m_declaration_token_count_buffer[index];
if (not parser(std::span(m_value_token_buffer).subspan(value_token_offset, value_token_count), data))
{
return std::nullopt;
}
}
return data;
}