Skip to content

Fix iOS Camera Texture memory leak on dispose #1135

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Sep 14, 2022
Merged

Fix iOS Camera Texture memory leak on dispose #1135

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
693f3ed
Fix issue where camera texture delegate was stopping reset from being…
Alex-MSFT Sep 13, 2022
e733c32
Remove implData reference from CameraTextureDelegate, and just pass d…
Alex-MSFT Sep 13, 2022
36ef82e
PR Feedback
Alex-MSFT Sep 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Plugins/NativeCamera/Source/Apple/NativeCameraImpl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ namespace Babylon::Plugins
Graphics::DeviceContext* m_deviceContext;
Napi::Env m_env;

std::shared_ptr<ImplData> m_implData;
std::unique_ptr<ImplData> m_implData;

bool m_overrideCameraTexture{};

Expand Down
46 changes: 32 additions & 14 deletions Plugins/NativeCamera/Source/Apple/NativeCameraImpl.mm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,14 @@ @interface CameraTextureDelegate : NSObject <AVCaptureVideoDataOutputSampleBuffe
{
@public AVCaptureVideoOrientation videoOrientation;

std::shared_ptr<Babylon::Plugins::Camera::Impl::ImplData> implData;
CVMetalTextureCacheRef textureCache;
bool orientationUpdated;
}

- (id)init:(std::shared_ptr<Babylon::Plugins::Camera::Impl::ImplData>)implData;
- (id)init:(CVMetalTextureCacheRef)textureCache;
- (id<MTLTexture>)getCameraTextureY;
- (id<MTLTexture>)getCameraTextureCbCr;
- (void)reset;

@end

Expand Down Expand Up @@ -140,7 +141,9 @@ fragment float4 fragmentShader(RasterizerData in [[stage_in]],
[currentCommandBuffer waitUntilCompleted];
}

[avCaptureSession stopRunning];
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
[avCaptureSession stopRunning];
});

if (textureCache)
{
Expand All @@ -152,7 +155,7 @@ fragment float4 fragmentShader(RasterizerData in [[stage_in]],
CameraTextureDelegate* cameraTextureDelegate{};
AVCaptureSession* avCaptureSession{};
CVMetalTextureCacheRef textureCache{};
id <MTLTexture> textureRGBA{};
id<MTLTexture> textureRGBA{};
id<MTLRenderPipelineState> cameraPipelineState{};
id<MTLDevice> metalDevice{};
id<MTLCommandQueue> commandQueue{};
Expand Down Expand Up @@ -191,7 +194,7 @@ fragment float4 fragmentShader(RasterizerData in [[stage_in]],

dispatch_async(dispatch_get_main_queue(), ^{
CVMetalTextureCacheCreate(nullptr, nullptr, m_implData->metalDevice, nullptr, &m_implData->textureCache);
m_implData->cameraTextureDelegate = [[CameraTextureDelegate alloc]init:m_implData];
m_implData->cameraTextureDelegate = [[CameraTextureDelegate alloc]init:m_implData->textureCache];
m_implData->avCaptureSession = [[AVCaptureSession alloc] init];
m_implData->textureRGBA = nil;

Expand Down Expand Up @@ -347,7 +350,9 @@ fragment float4 fragmentShader(RasterizerData in [[stage_in]],
// Actually start the camera session.
[m_implData->avCaptureSession addOutput:dataOutput];
[m_implData->avCaptureSession commitConfiguration];
[m_implData->avCaptureSession startRunning];
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
[m_implData->avCaptureSession startRunning];
});

// Create a pipeline state for converting the camera output to RGBA.
id<MTLLibrary> lib = CompileShader(m_implData->metalDevice, shaderSource);
Expand Down Expand Up @@ -463,7 +468,9 @@ fragment float4 fragmentShader(RasterizerData in [[stage_in]],

void Camera::Impl::Close()
{
m_implData.reset(new ImplData);
[m_implData->cameraTextureDelegate reset];
m_implData.reset();
m_implData = std::make_unique<ImplData>();
}
}

Expand All @@ -472,10 +479,10 @@ @implementation CameraTextureDelegate {
CVMetalTextureRef cameraTextureCbCr;
}

- (id)init:(std::shared_ptr<Babylon::Plugins::Camera::Impl::ImplData>)implData
- (id)init:(CVMetalTextureCacheRef)textureCache
{
self = [super init];
self->implData = implData;
self->textureCache = textureCache;
#if (TARGET_OS_IPHONE)
[[NSNotificationCenter defaultCenter]addObserver:self selector:@selector(OrientationDidChange:) name:UIDeviceOrientationDidChangeNotification object:nil];
[self updateOrientation];
Expand Down Expand Up @@ -513,6 +520,12 @@ - (id)init:(std::shared_ptr<Babylon::Plugins::Camera::Impl::ImplData>)implData
return nil;
}

- (void) reset {
@synchronized (self) {
self->textureCache = nil;
}
}

#if (TARGET_OS_IPHONE)
/**
Updates target video orientation.
Expand Down Expand Up @@ -603,11 +616,16 @@ - (CVMetalTextureRef)getCameraTexture:(CVPixelBufferRef)pixelBuffer plane:(int)p
auto pixelFormat = planeIndex ? MTLPixelFormatRG8Unorm : MTLPixelFormatR8Unorm;
CVMetalTextureRef textureRef;

// Create a texture from the corresponding plane.
auto status = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, implData->textureCache, pixelBuffer, nil, pixelFormat, planeWidth, planeHeight, planeIndex, &textureRef);
if (status != kCVReturnSuccess) {
CVBufferRelease(textureRef);
textureRef = nil;
@synchronized (self) {
if (self->textureCache == nil) {
return{};
}

auto status = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormat, planeWidth, planeHeight, planeIndex, &textureRef);
if (status != kCVReturnSuccess) {
CVBufferRelease(textureRef);
textureRef = nil;
}
}

return textureRef;
Expand Down