Flutter Impeller
ahb_texture_source_vk.cc
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
6 
11 
12 namespace impeller {
13 
14 using AHBProperties = vk::StructureChain<
15  // For VK_ANDROID_external_memory_android_hardware_buffer
16  vk::AndroidHardwareBufferPropertiesANDROID,
17  // For VK_ANDROID_external_memory_android_hardware_buffer
18  vk::AndroidHardwareBufferFormatPropertiesANDROID>;
19 
21  const vk::Device& device,
22  const AHBProperties& ahb_props,
23  const AHardwareBuffer_Desc& ahb_desc) {
24  const auto& ahb_format =
25  ahb_props.get<vk::AndroidHardwareBufferFormatPropertiesANDROID>();
26 
27  vk::StructureChain<vk::ImageCreateInfo,
28  // For VK_KHR_external_memory
29  vk::ExternalMemoryImageCreateInfo,
30  // For VK_ANDROID_external_memory_android_hardware_buffer
31  vk::ExternalFormatANDROID>
32  image_chain;
33 
34  auto& image_info = image_chain.get<vk::ImageCreateInfo>();
35 
36  vk::ImageUsageFlags image_usage_flags;
37  if (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE) {
38  image_usage_flags |= vk::ImageUsageFlagBits::eSampled;
39  }
40  if (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_FRAMEBUFFER) {
41  image_usage_flags |= vk::ImageUsageFlagBits::eColorAttachment;
42  }
43 
44  vk::ImageCreateFlags image_create_flags;
45  if (ahb_desc.usage & AHARDWAREBUFFER_USAGE_PROTECTED_CONTENT) {
46  image_create_flags |= vk::ImageCreateFlagBits::eProtected;
47  }
48  if (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_CUBE_MAP) {
49  image_create_flags |= vk::ImageCreateFlagBits::eCubeCompatible;
50  }
51 
52  image_info.imageType = vk::ImageType::e2D;
53  image_info.format = ahb_format.format;
54  image_info.extent.width = ahb_desc.width;
55  image_info.extent.height = ahb_desc.height;
56  image_info.extent.depth = 1;
57  image_info.mipLevels =
58  (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_MIPMAP_COMPLETE)
59  ? ISize{ahb_desc.width, ahb_desc.height}.MipCount()
60  : 1u;
61  image_info.arrayLayers = ahb_desc.layers;
62  image_info.samples = vk::SampleCountFlagBits::e1;
63  image_info.tiling = vk::ImageTiling::eOptimal;
64  image_info.usage = image_usage_flags;
65  image_info.flags = image_create_flags;
66  image_info.sharingMode = vk::SharingMode::eExclusive;
67  image_info.initialLayout = vk::ImageLayout::eUndefined;
68 
69  image_chain.get<vk::ExternalMemoryImageCreateInfo>().handleTypes =
70  vk::ExternalMemoryHandleTypeFlagBits::eAndroidHardwareBufferANDROID;
71 
72  // If the format isn't natively supported by Vulkan (i.e, be a part of the
73  // base vkFormat enum), an untyped "external format" must be specified when
74  // creating the image and the image views. Usually includes YUV formats.
75  if (ahb_format.format == vk::Format::eUndefined) {
76  image_chain.get<vk::ExternalFormatANDROID>().externalFormat =
77  ahb_format.externalFormat;
78  } else {
79  image_chain.unlink<vk::ExternalFormatANDROID>();
80  }
81 
82  auto image = device.createImageUnique(image_chain.get());
83  if (image.result != vk::Result::eSuccess) {
84  VALIDATION_LOG << "Could not create image for external buffer: "
85  << vk::to_string(image.result);
86  return {};
87  }
88 
89  return std::move(image.value);
90 }
91 
92 static vk::UniqueDeviceMemory ImportVKDeviceMemoryFromAndroidHarwareBuffer(
93  const vk::Device& device,
94  const vk::PhysicalDevice& physical_device,
95  const vk::Image& image,
96  struct AHardwareBuffer* hardware_buffer,
97  const AHBProperties& ahb_props) {
98  vk::PhysicalDeviceMemoryProperties memory_properties;
99  physical_device.getMemoryProperties(&memory_properties);
100  int memory_type_index = AllocatorVK::FindMemoryTypeIndex(
101  ahb_props.get().memoryTypeBits, memory_properties);
102  if (memory_type_index < 0) {
103  VALIDATION_LOG << "Could not find memory type of external image.";
104  return {};
105  }
106 
107  vk::StructureChain<vk::MemoryAllocateInfo,
108  // Core in 1.1
109  vk::MemoryDedicatedAllocateInfo,
110  // For VK_ANDROID_external_memory_android_hardware_buffer
111  vk::ImportAndroidHardwareBufferInfoANDROID>
112  memory_chain;
113 
114  auto& mem_alloc_info = memory_chain.get<vk::MemoryAllocateInfo>();
115  mem_alloc_info.allocationSize = ahb_props.get().allocationSize;
116  mem_alloc_info.memoryTypeIndex = memory_type_index;
117 
118  auto& dedicated_alloc_info =
119  memory_chain.get<vk::MemoryDedicatedAllocateInfo>();
120  dedicated_alloc_info.image = image;
121 
122  auto& ahb_import_info =
123  memory_chain.get<vk::ImportAndroidHardwareBufferInfoANDROID>();
124  ahb_import_info.buffer = hardware_buffer;
125 
126  auto device_memory = device.allocateMemoryUnique(memory_chain.get());
127  if (device_memory.result != vk::Result::eSuccess) {
128  VALIDATION_LOG << "Could not allocate device memory for external image : "
129  << vk::to_string(device_memory.result);
130  return {};
131  }
132 
133  return std::move(device_memory.value);
134 }
135 
136 static std::shared_ptr<YUVConversionVK> CreateYUVConversion(
137  const ContextVK& context,
138  const AHBProperties& ahb_props) {
139  YUVConversionDescriptorVK conversion_chain;
140 
141  const auto& ahb_format =
142  ahb_props.get<vk::AndroidHardwareBufferFormatPropertiesANDROID>();
143 
144  auto& conversion_info = conversion_chain.get();
145 
146  conversion_info.format = ahb_format.format;
147  conversion_info.ycbcrModel = ahb_format.suggestedYcbcrModel;
148  conversion_info.ycbcrRange = ahb_format.suggestedYcbcrRange;
149  conversion_info.components = ahb_format.samplerYcbcrConversionComponents;
150  conversion_info.xChromaOffset = ahb_format.suggestedXChromaOffset;
151  conversion_info.yChromaOffset = ahb_format.suggestedYChromaOffset;
152  // If the potential format features of the sampler Y′CBCR conversion do not
153  // support VK_FORMAT_FEATURE_SAMPLED_IMAGE_YCBCR_CONVERSION_LINEAR_FILTER_BIT,
154  // chromaFilter must not be VK_FILTER_LINEAR.
155  //
156  // Since we are not checking, let's just default to a safe value.
157  conversion_info.chromaFilter = vk::Filter::eNearest;
158  conversion_info.forceExplicitReconstruction = false;
159 
160  if (conversion_info.format == vk::Format::eUndefined) {
161  auto& external_format = conversion_chain.get<vk::ExternalFormatANDROID>();
162  external_format.externalFormat = ahb_format.externalFormat;
163  } else {
164  conversion_chain.unlink<vk::ExternalFormatANDROID>();
165  }
166 
167  return context.GetYUVConversionLibrary()->GetConversion(conversion_chain);
168 }
169 
170 static vk::UniqueImageView CreateVKImageView(
171  const vk::Device& device,
172  const vk::Image& image,
173  const vk::SamplerYcbcrConversion& yuv_conversion,
174  const AHBProperties& ahb_props,
175  const AHardwareBuffer_Desc& ahb_desc) {
176  const auto& ahb_format =
177  ahb_props.get<vk::AndroidHardwareBufferFormatPropertiesANDROID>();
178 
179  vk::StructureChain<vk::ImageViewCreateInfo,
180  // Core in 1.1
181  vk::SamplerYcbcrConversionInfo>
182  view_chain;
183 
184  auto& view_info = view_chain.get();
185 
186  view_info.image = image;
187  view_info.viewType = vk::ImageViewType::e2D;
188  view_info.format = ahb_format.format;
189  view_info.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor;
190  view_info.subresourceRange.baseMipLevel = 0u;
191  view_info.subresourceRange.baseArrayLayer = 0u;
192  view_info.subresourceRange.levelCount =
193  (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_MIPMAP_COMPLETE)
194  ? ISize{ahb_desc.width, ahb_desc.height}.MipCount()
195  : 1u;
196  view_info.subresourceRange.layerCount = ahb_desc.layers;
197 
198  // We need a custom YUV conversion only if we don't recognize the format.
199  if (view_info.format == vk::Format::eUndefined) {
200  view_chain.get<vk::SamplerYcbcrConversionInfo>().conversion =
201  yuv_conversion;
202  } else {
203  view_chain.unlink<vk::SamplerYcbcrConversionInfo>();
204  }
205 
206  auto image_view = device.createImageViewUnique(view_info);
207  if (image_view.result != vk::Result::eSuccess) {
208  VALIDATION_LOG << "Could not create external image view: "
209  << vk::to_string(image_view.result);
210  return {};
211  }
212 
213  return std::move(image_view.value);
214 }
215 
216 static PixelFormat ToPixelFormat(AHardwareBuffer_Format format) {
217  switch (format) {
218  case AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM:
220  case AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT:
222  case AHARDWAREBUFFER_FORMAT_D24_UNORM_S8_UINT:
224  case AHARDWAREBUFFER_FORMAT_D32_FLOAT_S8_UINT:
226  case AHARDWAREBUFFER_FORMAT_S8_UINT:
227  return PixelFormat::kS8UInt;
228  case AHARDWAREBUFFER_FORMAT_R8_UNORM:
230  case AHARDWAREBUFFER_FORMAT_R10G10B10A10_UNORM:
231  case AHARDWAREBUFFER_FORMAT_R16G16_UINT:
232  case AHARDWAREBUFFER_FORMAT_D32_FLOAT:
233  case AHARDWAREBUFFER_FORMAT_R16_UINT:
234  case AHARDWAREBUFFER_FORMAT_D24_UNORM:
235  case AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420:
236  case AHARDWAREBUFFER_FORMAT_YCbCr_P010:
237  case AHARDWAREBUFFER_FORMAT_BLOB:
238  case AHARDWAREBUFFER_FORMAT_R10G10B10A2_UNORM:
239  case AHARDWAREBUFFER_FORMAT_R5G6B5_UNORM:
240  case AHARDWAREBUFFER_FORMAT_R8G8B8_UNORM:
241  case AHARDWAREBUFFER_FORMAT_D16_UNORM:
242  case AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM:
243  // Not understood by the rest of Impeller. Use a placeholder but create
244  // the native image and image views using the right external format.
245  break;
246  }
248 }
249 
250 static TextureType ToTextureType(const AHardwareBuffer_Desc& ahb_desc) {
251  if (ahb_desc.layers == 1u) {
253  }
254  if (ahb_desc.layers % 6u == 0 &&
255  (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_CUBE_MAP)) {
257  }
258  // Our texture types seem to understand external OES textures. Should these be
259  // wired up instead?
261 }
262 
264  const AHardwareBuffer_Desc& ahb_desc) {
265  const auto ahb_size = ISize{ahb_desc.width, ahb_desc.height};
266  TextureDescriptor desc;
267  // We are not going to touch hardware buffers on the CPU or use them as
268  // transient attachments. Just treat them as device private.
270  desc.format =
271  ToPixelFormat(static_cast<AHardwareBuffer_Format>(ahb_desc.format));
272  desc.size = ahb_size;
273  desc.type = ToTextureType(ahb_desc);
276  desc.mip_count = (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_MIPMAP_COMPLETE)
277  ? ahb_size.MipCount()
278  : 1u;
279  return desc;
280 }
281 
283  const std::shared_ptr<ContextVK>& context,
284  struct AHardwareBuffer* ahb,
285  const AHardwareBuffer_Desc& ahb_desc)
286  : TextureSourceVK(ToTextureDescriptor(ahb_desc)) {
287  if (!context) {
288  VALIDATION_LOG << "Invalid context.";
289  return;
290  }
291 
292  const auto& device = context->GetDevice();
293  const auto& physical_device = context->GetPhysicalDevice();
294 
295  AHBProperties ahb_props;
296 
297  if (device.getAndroidHardwareBufferPropertiesANDROID(ahb, &ahb_props.get()) !=
298  vk::Result::eSuccess) {
299  VALIDATION_LOG << "Could not determine properties of the Android hardware "
300  "buffer.";
301  return;
302  }
303 
304  const auto& ahb_format =
305  ahb_props.get<vk::AndroidHardwareBufferFormatPropertiesANDROID>();
306 
307  // Create an image to refer to our external image.
308  auto image =
309  CreateVKImageWrapperForAndroidHarwareBuffer(device, ahb_props, ahb_desc);
310  if (!image) {
311  return;
312  }
313 
314  // Create a device memory allocation to refer to our external image.
316  device, physical_device, image.get(), ahb, ahb_props);
317  if (!device_memory) {
318  return;
319  }
320 
321  // Bind the image to the image memory.
322  if (auto result = device.bindImageMemory(image.get(), device_memory.get(), 0);
323  result != vk::Result::eSuccess) {
324  VALIDATION_LOG << "Could not bind external device memory to image : "
325  << vk::to_string(result);
326  return;
327  }
328 
329  // Figure out how to perform YUV conversions.
330  auto yuv_conversion = CreateYUVConversion(*context, ahb_props);
331  if (!yuv_conversion || !yuv_conversion->IsValid()) {
332  return;
333  }
334 
335  // Create image view for the newly created image.
336  auto image_view = CreateVKImageView(device, //
337  image.get(), //
338  yuv_conversion->GetConversion(), //
339  ahb_props, //
340  ahb_desc //
341  );
342  if (!image_view) {
343  return;
344  }
345 
346  needs_yuv_conversion_ = ahb_format.format == vk::Format::eUndefined;
347  device_memory_ = std::move(device_memory);
348  image_ = std::move(image);
349  yuv_conversion_ = std::move(yuv_conversion);
350  image_view_ = std::move(image_view);
351 
352 #ifdef IMPELLER_DEBUG
353  context->SetDebugName(device_memory_.get(), "AHB Device Memory");
354  context->SetDebugName(image_.get(), "AHB Image");
355  context->SetDebugName(yuv_conversion_->GetConversion(), "AHB YUV Conversion");
356  context->SetDebugName(image_view_.get(), "AHB ImageView");
357 #endif // IMPELLER_DEBUG
358 
359  is_valid_ = true;
360 }
361 
362 // |TextureSourceVK|
364 
366  return is_valid_;
367 }
368 
369 // |TextureSourceVK|
370 vk::Image AHBTextureSourceVK::GetImage() const {
371  return image_.get();
372 }
373 
374 // |TextureSourceVK|
375 vk::ImageView AHBTextureSourceVK::GetImageView() const {
376  return image_view_.get();
377 }
378 
379 // |TextureSourceVK|
381  return image_view_.get();
382 }
383 
384 // |TextureSourceVK|
386  return false;
387 }
388 
389 // |TextureSourceVK|
390 std::shared_ptr<YUVConversionVK> AHBTextureSourceVK::GetYUVConversion() const {
391  return needs_yuv_conversion_ ? yuv_conversion_ : nullptr;
392 }
393 
394 } // namespace impeller
impeller::PixelFormat::kS8UInt
@ kS8UInt
impeller::AllocatorVK::FindMemoryTypeIndex
static int32_t FindMemoryTypeIndex(uint32_t memory_type_bits_requirement, vk::PhysicalDeviceMemoryProperties &memory_properties)
Select a matching memory type for the given [memory_type_bits_requirement], or -1 if none is found.
Definition: allocator_vk.cc:173
impeller::TextureSourceVK
Abstract base class that represents a vkImage and an vkImageView.
Definition: texture_source_vk.h:28
impeller::AHBTextureSourceVK::IsValid
bool IsValid() const
Definition: ahb_texture_source_vk.cc:365
allocator_vk.h
impeller::AHBTextureSourceVK::~AHBTextureSourceVK
~AHBTextureSourceVK() override
impeller::AHBTextureSourceVK::AHBTextureSourceVK
AHBTextureSourceVK(const std::shared_ptr< ContextVK > &context, struct AHardwareBuffer *hardware_buffer, const AHardwareBuffer_Desc &hardware_buffer_desc)
Definition: ahb_texture_source_vk.cc:282
impeller::PixelFormat::kR8UNormInt
@ kR8UNormInt
impeller::AHBTextureSourceVK::GetImageView
vk::ImageView GetImageView() const override
Retrieve the image view used for sampling/blitting/compute with this texture source.
Definition: ahb_texture_source_vk.cc:375
texture_source_vk.h
impeller::TextureDescriptor::format
PixelFormat format
Definition: texture_descriptor.h:40
impeller::PixelFormat::kR8G8B8A8UNormInt
@ kR8G8B8A8UNormInt
impeller::TextureType
TextureType
Definition: formats.h:263
impeller::TextureDescriptor::mip_count
size_t mip_count
Definition: texture_descriptor.h:42
yuv_conversion_library_vk.h
impeller::CreateVKImageWrapperForAndroidHarwareBuffer
static vk::UniqueImage CreateVKImageWrapperForAndroidHarwareBuffer(const vk::Device &device, const AHBProperties &ahb_props, const AHardwareBuffer_Desc &ahb_desc)
Definition: ahb_texture_source_vk.cc:20
impeller::TextureDescriptor::sample_count
SampleCount sample_count
Definition: texture_descriptor.h:44
impeller::ImportVKDeviceMemoryFromAndroidHarwareBuffer
static vk::UniqueDeviceMemory ImportVKDeviceMemoryFromAndroidHarwareBuffer(const vk::Device &device, const vk::PhysicalDevice &physical_device, const vk::Image &image, struct AHardwareBuffer *hardware_buffer, const AHBProperties &ahb_props)
Definition: ahb_texture_source_vk.cc:92
impeller::PixelFormat::kD32FloatS8UInt
@ kD32FloatS8UInt
impeller::PixelFormat
PixelFormat
The Pixel formats supported by Impeller. The naming convention denotes the usage of the component,...
Definition: formats.h:100
impeller::TextureDescriptor::type
TextureType type
Definition: texture_descriptor.h:39
impeller::TSize< int64_t >
impeller::StorageMode::kDevicePrivate
@ kDevicePrivate
impeller::CompressionType::kLossless
@ kLossless
impeller::PixelFormat::kD24UnormS8Uint
@ kD24UnormS8Uint
impeller::TextureType::kTextureCube
@ kTextureCube
impeller::PixelFormat::kR16G16B16A16Float
@ kR16G16B16A16Float
impeller::ToTextureType
constexpr GLenum ToTextureType(TextureType type)
Definition: formats_gles.h:170
impeller::AHBTextureSourceVK::GetImage
vk::Image GetImage() const override
Get the image handle for this texture source.
Definition: ahb_texture_source_vk.cc:370
impeller::CreateYUVConversion
static std::shared_ptr< YUVConversionVK > CreateYUVConversion(const ContextVK &context, const AHBProperties &ahb_props)
Definition: ahb_texture_source_vk.cc:136
impeller::TextureType::kTexture2D
@ kTexture2D
impeller::TSize::width
Type width
Definition: size.h:22
impeller::ContextVK
Definition: context_vk.h:42
impeller::TextureDescriptor::size
ISize size
Definition: texture_descriptor.h:41
impeller::ToTextureDescriptor
static TextureDescriptor ToTextureDescriptor(const AHardwareBuffer_Desc &ahb_desc)
Definition: ahb_texture_source_vk.cc:263
VALIDATION_LOG
#define VALIDATION_LOG
Definition: validation.h:73
ahb_texture_source_vk.h
impeller::SampleCount::kCount1
@ kCount1
impeller::ContextVK::GetYUVConversionLibrary
const std::shared_ptr< YUVConversionLibraryVK > & GetYUVConversionLibrary() const
Definition: context_vk.cc:622
impeller::TextureDescriptor::storage_mode
StorageMode storage_mode
Definition: texture_descriptor.h:38
impeller::AHBProperties
vk::StructureChain< vk::AndroidHardwareBufferPropertiesANDROID, vk::AndroidHardwareBufferFormatPropertiesANDROID > AHBProperties
Definition: ahb_texture_source_vk.cc:18
impeller::TextureDescriptor
A lightweight object that describes the attributes of a texture that can then used an allocator to cr...
Definition: texture_descriptor.h:37
context_vk.h
impeller::YUVConversionDescriptorVK
vk::StructureChain< vk::SamplerYcbcrConversionCreateInfo > YUVConversionDescriptorVK
Definition: yuv_conversion_vk.h:30
impeller::AHBTextureSourceVK::GetRenderTargetView
vk::ImageView GetRenderTargetView() const override
Retrieve the image view used for render target attachments with this texture source.
Definition: ahb_texture_source_vk.cc:380
impeller::TextureDescriptor::compression_type
CompressionType compression_type
Definition: texture_descriptor.h:45
impeller
Definition: aiks_blur_unittests.cc:20
impeller::AHBTextureSourceVK::IsSwapchainImage
bool IsSwapchainImage() const override
Determines if swapchain image. That is, an image used as the root render target.
Definition: ahb_texture_source_vk.cc:385
impeller::CreateVKImageView
static vk::UniqueImageView CreateVKImageView(const vk::Device &device, const vk::Image &image, const vk::SamplerYcbcrConversion &yuv_conversion, const AHBProperties &ahb_props, const AHardwareBuffer_Desc &ahb_desc)
Definition: ahb_texture_source_vk.cc:170
impeller::ToPixelFormat
static PixelFormat ToPixelFormat(AHardwareBuffer_Format format)
Definition: ahb_texture_source_vk.cc:216
impeller::AHBTextureSourceVK::GetYUVConversion
std::shared_ptr< YUVConversionVK > GetYUVConversion() const override
When sampling from textures whose formats are not known to Vulkan, a custom conversion is necessary t...
Definition: ahb_texture_source_vk.cc:390