Flutter Impeller
compiler.cc
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
6 
7 #include <cstdint>
8 #include <filesystem>
9 #include <memory>
10 #include <optional>
11 #include <sstream>
12 #include <string>
13 #include <utility>
14 
15 #include "flutter/fml/paths.h"
25 
26 namespace impeller {
27 namespace compiler {
28 
29 namespace {
30 constexpr const char* kEGLImageExternalExtension = "GL_OES_EGL_image_external";
31 constexpr const char* kEGLImageExternalExtension300 =
32  "GL_OES_EGL_image_external_essl3";
33 } // namespace
34 
35 // This value should be <= 7372. UBOs can be larger on some devices but a
36 // performance cost will be paid.
37 // https://docs.qualcomm.com/bundle/publicresource/topics/80-78185-2/best_practices.html?product=1601111740035277#buffer-best-practices
38 static const uint32_t kMaxUniformBufferSize = 6208;
39 
40 static uint32_t ParseMSLVersion(const std::string& msl_version) {
41  std::stringstream sstream(msl_version);
42  std::string version_part;
43  uint32_t major = 1;
44  uint32_t minor = 2;
45  uint32_t patch = 0;
46  if (std::getline(sstream, version_part, '.')) {
47  major = std::stoi(version_part);
48  if (std::getline(sstream, version_part, '.')) {
49  minor = std::stoi(version_part);
50  if (std::getline(sstream, version_part, '.')) {
51  patch = std::stoi(version_part);
52  }
53  }
54  }
55  if (major < 1 || (major == 1 && minor < 2)) {
56  std::cerr << "--metal-version version must be at least 1.2. Have "
57  << msl_version << std::endl;
58  }
59  return spirv_cross::CompilerMSL::Options::make_msl_version(major, minor,
60  patch);
61 }
62 
64  const spirv_cross::ParsedIR& ir,
65  const SourceOptions& source_options,
66  std::optional<uint32_t> msl_version_override = {}) {
67  auto sl_compiler = std::make_shared<spirv_cross::CompilerMSL>(ir);
68  spirv_cross::CompilerMSL::Options sl_options;
69  sl_options.platform =
71  sl_options.msl_version = msl_version_override.value_or(
72  ParseMSLVersion(source_options.metal_version));
73  sl_options.ios_use_simdgroup_functions =
74  sl_options.is_ios() &&
75  sl_options.msl_version >=
76  spirv_cross::CompilerMSL::Options::make_msl_version(2, 4, 0);
77  sl_options.use_framebuffer_fetch_subpasses = true;
78  sl_compiler->set_msl_options(sl_options);
79 
80  // Sort the float and sampler uniforms according to their declared/decorated
81  // order. For user authored fragment shaders, the API for setting uniform
82  // values uses the index of the uniform in the declared order. By default, the
83  // metal backend of spirv-cross will order uniforms according to usage. To fix
84  // this, we use the sorted order and the add_msl_resource_binding API to force
85  // the ordering to match the declared order. Note that while this code runs
86  // for all compiled shaders, it will only affect vertex and fragment shaders
87  // due to the specified stage.
88  auto floats =
89  SortUniforms(&ir, sl_compiler.get(), spirv_cross::SPIRType::Float);
90  auto images =
91  SortUniforms(&ir, sl_compiler.get(), spirv_cross::SPIRType::SampledImage);
92 
93  spv::ExecutionModel execution_model =
94  spv::ExecutionModel::ExecutionModelFragment;
95  if (source_options.type == SourceType::kVertexShader) {
96  execution_model = spv::ExecutionModel::ExecutionModelVertex;
97  }
98 
99  uint32_t buffer_offset = 0;
100  uint32_t sampler_offset = 0;
101  for (auto& float_id : floats) {
102  sl_compiler->add_msl_resource_binding(
103  {.stage = execution_model,
104  .basetype = spirv_cross::SPIRType::BaseType::Float,
105  .desc_set = sl_compiler->get_decoration(float_id,
106  spv::DecorationDescriptorSet),
107  .binding =
108  sl_compiler->get_decoration(float_id, spv::DecorationBinding),
109  .count = 1u,
110  .msl_buffer = buffer_offset});
111  buffer_offset++;
112  }
113  for (auto& image_id : images) {
114  sl_compiler->add_msl_resource_binding({
115  .stage = execution_model,
116  .basetype = spirv_cross::SPIRType::BaseType::SampledImage,
117  .desc_set =
118  sl_compiler->get_decoration(image_id, spv::DecorationDescriptorSet),
119  .binding =
120  sl_compiler->get_decoration(image_id, spv::DecorationBinding),
121  .count = 1u,
122  // A sampled image is both an image and a sampler, so both
123  // offsets need to be set or depending on the partiular shader
124  // the bindings may be incorrect.
125  .msl_texture = sampler_offset,
126  .msl_sampler = sampler_offset,
127  });
128  sampler_offset++;
129  }
130 
131  return CompilerBackend(sl_compiler);
132 }
133 
135  const spirv_cross::ParsedIR& ir,
136  const SourceOptions& source_options) {
137  auto gl_compiler = std::make_shared<spirv_cross::CompilerGLSL>(ir);
138  spirv_cross::CompilerGLSL::Options sl_options;
139  sl_options.force_zero_initialized_variables = true;
140  sl_options.vertex.fixup_clipspace = true;
141  sl_options.vulkan_semantics = true;
142  gl_compiler->set_common_options(sl_options);
143  return CompilerBackend(gl_compiler);
144 }
145 
146 static CompilerBackend CreateGLSLCompiler(const spirv_cross::ParsedIR& ir,
147  const SourceOptions& source_options) {
148  auto gl_compiler = std::make_shared<spirv_cross::CompilerGLSL>(ir);
149 
150  // Walk the variables and insert the external image extension if any of them
151  // begins with the external texture prefix. Unfortunately, we can't walk
152  // `gl_compiler->get_shader_resources().separate_samplers` until the compiler
153  // is further along.
154  //
155  // Unfortunately, we can't just let the shader author add this extension and
156  // use `samplerExternalOES` directly because compiling to spirv requires the
157  // source language profile to be at least 310 ES, but this extension is
158  // incompatible with ES 310+.
159  for (auto& id : ir.ids_for_constant_or_variable) {
160  if (StringStartsWith(ir.get_name(id), kExternalTexturePrefix)) {
161  if (source_options.gles_language_version >= 300) {
162  gl_compiler->require_extension(kEGLImageExternalExtension300);
163  } else {
164  gl_compiler->require_extension(kEGLImageExternalExtension);
165  }
166  break;
167  }
168  }
169 
170  spirv_cross::CompilerGLSL::Options sl_options;
171  sl_options.force_zero_initialized_variables = true;
172  sl_options.vertex.fixup_clipspace = true;
173  if (source_options.target_platform == TargetPlatform::kOpenGLES ||
176  sl_options.version = source_options.gles_language_version > 0
177  ? source_options.gles_language_version
178  : 100;
179  sl_options.es = true;
180  if (source_options.target_platform == TargetPlatform::kRuntimeStageGLES3) {
181  sl_options.version = 300;
182  }
183  if (source_options.require_framebuffer_fetch &&
184  source_options.type == SourceType::kFragmentShader) {
185  gl_compiler->remap_ext_framebuffer_fetch(0, 0, true);
186  }
187  gl_compiler->set_variable_type_remap_callback(
188  [&](const spirv_cross::SPIRType& type, const std::string& var_name,
189  std::string& name_of_type) {
190  if (StringStartsWith(var_name, kExternalTexturePrefix)) {
191  name_of_type = "samplerExternalOES";
192  }
193  });
194  } else {
195  sl_options.version = source_options.gles_language_version > 0
196  ? source_options.gles_language_version
197  : 120;
198  sl_options.es = false;
199  }
200  gl_compiler->set_common_options(sl_options);
201  return CompilerBackend(gl_compiler);
202 }
203 
204 static CompilerBackend CreateSkSLCompiler(const spirv_cross::ParsedIR& ir,
205  const SourceOptions& source_options) {
206  auto sksl_compiler = std::make_shared<CompilerSkSL>(ir);
207  return CompilerBackend(sksl_compiler);
208 }
209 
211  switch (platform) {
213  FML_UNREACHABLE();
219  return false;
225  return true;
226  }
227  FML_UNREACHABLE();
228 }
229 
230 static CompilerBackend CreateCompiler(const spirv_cross::ParsedIR& ir,
231  const SourceOptions& source_options) {
232  CompilerBackend compiler;
233  switch (source_options.target_platform) {
237  compiler = CreateMSLCompiler(ir, source_options);
238  break;
241  compiler = CreateVulkanCompiler(ir, source_options);
242  break;
248  compiler = CreateGLSLCompiler(ir, source_options);
249  break;
251  compiler = CreateSkSLCompiler(ir, source_options);
252  }
253  if (!compiler) {
254  return {};
255  }
256  auto* backend = compiler.GetCompiler();
257  if (!EntryPointMustBeNamedMain(source_options.target_platform) &&
258  source_options.source_language == SourceLanguage::kGLSL) {
259  backend->rename_entry_point("main", source_options.entry_point_name,
260  ToExecutionModel(source_options.type));
261  }
262  return compiler;
263 }
264 
265 namespace {
266 uint32_t CalculateUBOSize(const spirv_cross::Compiler* compiler) {
267  spirv_cross::ShaderResources resources = compiler->get_shader_resources();
268  uint32_t result = 0;
269  for (const spirv_cross::Resource& ubo : resources.uniform_buffers) {
270  const spirv_cross::SPIRType& ubo_type =
271  compiler->get_type(ubo.base_type_id);
272  uint32_t size = compiler->get_declared_struct_size(ubo_type);
273  result += size;
274  }
275  return result;
276 }
277 
278 } // namespace
279 
280 Compiler::Compiler(const std::shared_ptr<const fml::Mapping>& source_mapping,
281  const SourceOptions& source_options,
282  Reflector::Options reflector_options)
283  : options_(source_options) {
284  if (!source_mapping || source_mapping->GetMapping() == nullptr) {
285  COMPILER_ERROR(error_stream_)
286  << "Could not read shader source or shader source was empty.";
287  return;
288  }
289 
290  if (source_options.target_platform == TargetPlatform::kUnknown) {
291  COMPILER_ERROR(error_stream_) << "Target platform not specified.";
292  return;
293  }
294 
295  SPIRVCompilerOptions spirv_options;
296 
297  // Make sure reflection is as effective as possible. The generated shaders
298  // will be processed later by backend specific compilers.
299  spirv_options.generate_debug_info = true;
300 
301  switch (options_.source_language) {
303  // Expects GLSL 4.60 (Core Profile).
304  // https://www.khronos.org/registry/OpenGL/specs/gl/GLSLangSpec.4.60.pdf
305  spirv_options.source_langauge =
306  shaderc_source_language::shaderc_source_language_glsl;
308  shaderc_profile::shaderc_profile_core, //
309  460, //
310  };
311  break;
313  spirv_options.source_langauge =
314  shaderc_source_language::shaderc_source_language_hlsl;
315  break;
317  COMPILER_ERROR(error_stream_) << "Source language invalid.";
318  return;
319  }
320 
321  switch (source_options.target_platform) {
324  SPIRVCompilerTargetEnv target;
325 
326  if (source_options.use_half_textures) {
327  target.env = shaderc_target_env::shaderc_target_env_opengl;
328  target.version = shaderc_env_version::shaderc_env_version_opengl_4_5;
329  target.spirv_version = shaderc_spirv_version::shaderc_spirv_version_1_0;
330  } else {
331  target.env = shaderc_target_env::shaderc_target_env_vulkan;
332  target.version = shaderc_env_version::shaderc_env_version_vulkan_1_1;
333  target.spirv_version = shaderc_spirv_version::shaderc_spirv_version_1_3;
334  }
335 
336  spirv_options.target = target;
337  } break;
342  SPIRVCompilerTargetEnv target;
343 
344  target.env = shaderc_target_env::shaderc_target_env_vulkan;
345  target.version = shaderc_env_version::shaderc_env_version_vulkan_1_1;
346  target.spirv_version = shaderc_spirv_version::shaderc_spirv_version_1_3;
347 
348  if (source_options.target_platform ==
350  spirv_options.macro_definitions.push_back("IMPELLER_GRAPHICS_BACKEND");
351  spirv_options.relaxed_vulkan_rules = true;
352  }
353  spirv_options.target = target;
354  } break;
358  SPIRVCompilerTargetEnv target;
359 
360  target.env = shaderc_target_env::shaderc_target_env_opengl;
361  target.version = shaderc_env_version::shaderc_env_version_opengl_4_5;
362  target.spirv_version = shaderc_spirv_version::shaderc_spirv_version_1_0;
363 
364  spirv_options.target = target;
365  spirv_options.macro_definitions.push_back("IMPELLER_GRAPHICS_BACKEND");
366  if (source_options.target_platform == TargetPlatform::kRuntimeStageGLES ||
367  source_options.target_platform ==
369  spirv_options.macro_definitions.push_back("IMPELLER_TARGET_OPENGLES");
370  }
371  } break;
372  case TargetPlatform::kSkSL: {
373  SPIRVCompilerTargetEnv target;
374 
375  target.env = shaderc_target_env::shaderc_target_env_opengl;
376  target.version = shaderc_env_version::shaderc_env_version_opengl_4_5;
377  target.spirv_version = shaderc_spirv_version::shaderc_spirv_version_1_0;
378 
379  // When any optimization level above 'zero' is enabled, the phi merges at
380  // loop continue blocks are rendered using syntax that is supported in
381  // GLSL, but not in SkSL.
382  // https://bugs.chromium.org/p/skia/issues/detail?id=13518.
383  spirv_options.optimization_level =
384  shaderc_optimization_level::shaderc_optimization_level_zero;
385  spirv_options.target = target;
386  spirv_options.macro_definitions.push_back("SKIA_GRAPHICS_BACKEND");
387  } break;
389  COMPILER_ERROR(error_stream_) << "Target platform invalid.";
390  return;
391  }
392 
393  // Implicit definition that indicates that this compilation is for the device
394  // (instead of the host).
395  spirv_options.macro_definitions.push_back("IMPELLER_DEVICE");
396  for (const auto& define : source_options.defines) {
397  spirv_options.macro_definitions.push_back(define);
398  }
399 
400  std::vector<std::string> included_file_names;
401  spirv_options.includer = std::make_shared<Includer>(
402  options_.working_directory, options_.include_dirs,
403  [&included_file_names](auto included_name) {
404  included_file_names.emplace_back(std::move(included_name));
405  });
406 
407  // SPIRV Generation.
408  SPIRVCompiler spv_compiler(source_options, source_mapping);
409 
410  spirv_assembly_ = spv_compiler.CompileToSPV(
411  error_stream_, spirv_options.BuildShadercOptions());
412 
413  if (!spirv_assembly_) {
414  return;
415  } else {
416  included_file_names_ = std::move(included_file_names);
417  }
418 
419  // SL Generation.
420  spirv_cross::Parser parser(
421  reinterpret_cast<const uint32_t*>(spirv_assembly_->GetMapping()),
422  spirv_assembly_->GetSize() / sizeof(uint32_t));
423  // The parser and compiler must be run separately because the parser contains
424  // meta information (like type member names) that are useful for reflection.
425  parser.parse();
426 
427  const auto parsed_ir =
428  std::make_shared<spirv_cross::ParsedIR>(parser.get_parsed_ir());
429 
430  auto sl_compiler = CreateCompiler(*parsed_ir, options_);
431 
432  if (!sl_compiler) {
433  COMPILER_ERROR(error_stream_)
434  << "Could not create compiler for target platform.";
435  return;
436  }
437 
438  uint32_t ubo_size = CalculateUBOSize(sl_compiler.GetCompiler());
439  if (ubo_size > kMaxUniformBufferSize) {
440  COMPILER_ERROR(error_stream_) << "Uniform buffer size exceeds max ("
441  << kMaxUniformBufferSize << "): " << ubo_size;
442  return;
443  }
444 
445  // We need to invoke the compiler even if we don't use the SL mapping later
446  // for Vulkan. The reflector needs information that is only valid after a
447  // successful compilation call.
448  auto sl_compilation_result =
449  CreateMappingWithString(sl_compiler.GetCompiler()->compile());
450 
451  // If the target is Vulkan, our shading language is SPIRV which we already
452  // have. We just need to strip it of debug information. If it isn't, we need
453  // to invoke the appropriate compiler to compile the SPIRV to the target SL.
454  if (source_options.target_platform == TargetPlatform::kVulkan ||
456  auto stripped_spirv_options = spirv_options;
457  stripped_spirv_options.generate_debug_info = false;
458  sl_mapping_ = spv_compiler.CompileToSPV(
459  error_stream_, stripped_spirv_options.BuildShadercOptions());
460  } else {
461  sl_mapping_ = sl_compilation_result;
462  }
463 
464  if (!sl_mapping_) {
465  COMPILER_ERROR(error_stream_) << "Could not generate SL from SPIRV";
466  return;
467  }
468 
469  reflector_ = std::make_unique<Reflector>(std::move(reflector_options), //
470  parsed_ir, //
471  GetSLShaderSource(), //
472  sl_compiler //
473  );
474 
475  if (!reflector_->IsValid()) {
476  COMPILER_ERROR(error_stream_)
477  << "Could not complete reflection on generated shader.";
478  return;
479  }
480 
481  is_valid_ = true;
482 }
483 
484 Compiler::~Compiler() = default;
485 
486 std::shared_ptr<fml::Mapping> Compiler::GetSPIRVAssembly() const {
487  return spirv_assembly_;
488 }
489 
490 std::shared_ptr<fml::Mapping> Compiler::GetSLShaderSource() const {
491  return sl_mapping_;
492 }
493 
494 bool Compiler::IsValid() const {
495  return is_valid_;
496 }
497 
498 std::string Compiler::GetSourcePrefix() const {
499  std::stringstream stream;
500  stream << options_.file_name << ": ";
501  return stream.str();
502 }
503 
504 std::string Compiler::GetErrorMessages() const {
505  return error_stream_.str();
506 }
507 
508 const std::vector<std::string>& Compiler::GetIncludedFileNames() const {
509  return included_file_names_;
510 }
511 
512 static std::string JoinStrings(std::vector<std::string> items,
513  const std::string& separator) {
514  std::stringstream stream;
515  for (size_t i = 0, count = items.size(); i < count; i++) {
516  const auto is_last = (i == count - 1);
517 
518  stream << items[i];
519  if (!is_last) {
520  stream << separator;
521  }
522  }
523  return stream.str();
524 }
525 
526 std::string Compiler::GetDependencyNames(const std::string& separator) const {
527  std::vector<std::string> dependencies = included_file_names_;
528  dependencies.push_back(options_.file_name);
529  return JoinStrings(dependencies, separator);
530 }
531 
532 std::unique_ptr<fml::Mapping> Compiler::CreateDepfileContents(
533  std::initializer_list<std::string> targets_names) const {
534  // https://github.com/ninja-build/ninja/blob/master/src/depfile_parser.cc#L28
535  const auto targets = JoinStrings(targets_names, " ");
536  const auto dependencies = GetDependencyNames(" ");
537 
538  std::stringstream stream;
539  stream << targets << ": " << dependencies << "\n";
540 
541  auto contents = std::make_shared<std::string>(stream.str());
542  return std::make_unique<fml::NonOwnedMapping>(
543  reinterpret_cast<const uint8_t*>(contents->data()), contents->size(),
544  [contents](auto, auto) {});
545 }
546 
548  return reflector_.get();
549 }
550 
551 } // namespace compiler
552 } // namespace impeller
GLenum type
std::shared_ptr< fml::Mapping > GetSPIRVAssembly() const
Definition: compiler.cc:486
const Reflector * GetReflector() const
Definition: compiler.cc:547
Compiler(const std::shared_ptr< const fml::Mapping > &source_mapping, const SourceOptions &options, Reflector::Options reflector_options)
Definition: compiler.cc:280
const std::vector< std::string > & GetIncludedFileNames() const
Definition: compiler.cc:508
std::unique_ptr< fml::Mapping > CreateDepfileContents(std::initializer_list< std::string > targets) const
Definition: compiler.cc:532
std::shared_ptr< fml::Mapping > GetSLShaderSource() const
Definition: compiler.cc:490
std::string GetErrorMessages() const
Definition: compiler.cc:504
std::shared_ptr< fml::Mapping > CompileToSPV(std::stringstream &error_stream, const shaderc::CompileOptions &spirv_options) const
#define COMPILER_ERROR(stream)
Definition: logger.h:39
static CompilerBackend CreateSkSLCompiler(const spirv_cross::ParsedIR &ir, const SourceOptions &source_options)
Definition: compiler.cc:204
static const uint32_t kMaxUniformBufferSize
Definition: compiler.cc:38
static CompilerBackend CreateVulkanCompiler(const spirv_cross::ParsedIR &ir, const SourceOptions &source_options)
Definition: compiler.cc:134
constexpr char kExternalTexturePrefix[]
Definition: constants.h:11
static bool EntryPointMustBeNamedMain(TargetPlatform platform)
Definition: compiler.cc:210
static CompilerBackend CreateMSLCompiler(const spirv_cross::ParsedIR &ir, const SourceOptions &source_options, std::optional< uint32_t > msl_version_override={})
Definition: compiler.cc:63
static CompilerBackend CreateCompiler(const spirv_cross::ParsedIR &ir, const SourceOptions &source_options)
Definition: compiler.cc:230
static std::string JoinStrings(std::vector< std::string > items, const std::string &separator)
Definition: compiler.cc:512
spirv_cross::CompilerMSL::Options::Platform TargetPlatformToMSLPlatform(TargetPlatform platform)
Definition: types.cc:215
static uint32_t ParseMSLVersion(const std::string &msl_version)
Definition: compiler.cc:40
bool StringStartsWith(const std::string &target, const std::string &prefix)
Definition: utilities.cc:87
static CompilerBackend CreateGLSLCompiler(const spirv_cross::ParsedIR &ir, const SourceOptions &source_options)
Definition: compiler.cc:146
spv::ExecutionModel ToExecutionModel(SourceType type)
Definition: types.cc:201
std::vector< spirv_cross::ID > SortUniforms(const spirv_cross::ParsedIR *ir, const spirv_cross::Compiler *compiler, std::optional< spirv_cross::SPIRType::BaseType > type_filter, bool include)
Sorts uniform declarations in an IR according to decoration order.
std::shared_ptr< fml::Mapping > CreateMappingWithString(std::string string)
Creates a mapping with string data.
Definition: allocation.cc:111
spirv_cross::Compiler * GetCompiler()
std::optional< shaderc_source_language > source_langauge
std::vector< std::string > macro_definitions
shaderc_optimization_level optimization_level
std::optional< SPIRVCompilerSourceProfile > source_profile
std::shared_ptr< Includer > includer
shaderc::CompileOptions BuildShadercOptions() const
std::optional< SPIRVCompilerTargetEnv > target
bool use_half_textures
Whether half-precision textures should be supported, requiring opengl semantics. Only used on metal t...
std::vector< IncludeDir > include_dirs
bool require_framebuffer_fetch
Whether the GLSL framebuffer fetch extension will be required.
std::shared_ptr< fml::UniqueFD > working_directory
std::vector< std::string > defines