fix memory leak in local video track

This commit is contained in:
Daniel Gultsch 2023-02-24 09:53:57 +01:00
parent 63df518c19
commit 1be1334794
No known key found for this signature in database
GPG key ID: F43D18AD2A0982C2
9 changed files with 133 additions and 20 deletions

View file

@ -44,6 +44,7 @@ import im.conversations.android.xmpp.manager.JingleConnectionManager;
import im.conversations.android.xmpp.model.disco.external.Service; import im.conversations.android.xmpp.model.disco.external.Service;
import im.conversations.android.xmpp.model.error.Condition; import im.conversations.android.xmpp.model.error.Condition;
import im.conversations.android.xmpp.model.error.Error; import im.conversations.android.xmpp.model.error.Error;
import im.conversations.android.xmpp.model.jingle.error.JingleCondition;
import im.conversations.android.xmpp.model.jmi.Accept; import im.conversations.android.xmpp.model.jmi.Accept;
import im.conversations.android.xmpp.model.jmi.JingleMessage; import im.conversations.android.xmpp.model.jmi.JingleMessage;
import im.conversations.android.xmpp.model.jmi.Proceed; import im.conversations.android.xmpp.model.jmi.Proceed;
@ -587,9 +588,11 @@ public class JingleRtpConnection extends AbstractJingleConnection
final Set<ContentAddition.Summary> removeSummary = final Set<ContentAddition.Summary> removeSummary =
ContentAddition.summary(receivedContentRemove); ContentAddition.summary(receivedContentRemove);
if (contentAddSummary.equals(removeSummary)) { if (contentAddSummary.equals(removeSummary)) {
LOGGER.info("Retracting content {}", removeSummary);
this.incomingContentAdd = null; this.incomingContentAdd = null;
updateEndUserState(); updateEndUserState();
} else { } else {
LOGGER.info("content add summary {} did not match remove summary {}", contentAddSummary, removeSummary);
webRTCWrapper.close(); webRTCWrapper.close();
sendSessionTerminate( sendSessionTerminate(
Reason.FAILED_APPLICATION, Reason.FAILED_APPLICATION,
@ -1767,7 +1770,6 @@ public class JingleRtpConnection extends AbstractJingleConnection
private void send(final JinglePacket jinglePacket) { private void send(final JinglePacket jinglePacket) {
jinglePacket.setTo(id.with); jinglePacket.setTo(id.with);
connection.sendIqPacket(jinglePacket, this::handleIqResponse); connection.sendIqPacket(jinglePacket, this::handleIqResponse);
connection.sendIqPacket(jinglePacket, this::handleIqResponse);
} }
private synchronized void handleIqResponse(final Iq response) { private synchronized void handleIqResponse(final Iq response) {
@ -1840,18 +1842,26 @@ public class JingleRtpConnection extends AbstractJingleConnection
private void respondWithTieBreak(final Iq jinglePacket) { private void respondWithTieBreak(final Iq jinglePacket) {
respondWithJingleError( respondWithJingleError(
jinglePacket, "tie-break", Error.Type.CANCEL, new Condition.Conflict()); jinglePacket,
new JingleCondition.TieBreak(),
Error.Type.CANCEL,
new Condition.Conflict());
} }
private void respondWithOutOfOrder(final Iq jinglePacket) { private void respondWithOutOfOrder(final Iq jinglePacket) {
respondWithJingleError( respondWithJingleError(
jinglePacket, "out-of-order", Error.Type.WAIT, new Condition.UnexpectedRequest()); jinglePacket,
new JingleCondition.OutOfOrder(),
Error.Type.WAIT,
new Condition.UnexpectedRequest());
} }
private void respondWithJingleError( private void respondWithJingleError(
final Iq original, String jingleCondition, final Error.Type type, Condition condition) { final Iq original,
// TODO add jingle condition JingleCondition jingleCondition,
connection.sendErrorFor(original, type, condition); final Error.Type type,
Condition condition) {
connection.sendErrorFor(original, type, condition, jingleCondition);
} }
private void respondOk(final Iq jinglePacket) { private void respondOk(final Iq jinglePacket) {

View file

@ -38,7 +38,7 @@ class TrackWrapper<T extends MediaStreamTrack> {
final RtpTransceiver transceiver = final RtpTransceiver transceiver =
peerConnection == null ? null : getTransceiver(peerConnection, trackWrapper); peerConnection == null ? null : getTransceiver(peerConnection, trackWrapper);
if (transceiver == null) { if (transceiver == null) {
Log.w(Config.LOGTAG, "unable to detect transceiver for " + trackWrapper.rtpSender.id()); Log.w(Config.LOGTAG, "unable to detect transceiver for " + trackWrapper.getRtpSenderId());
return Optional.of(trackWrapper.track); return Optional.of(trackWrapper.track);
} }
final RtpTransceiver.RtpTransceiverDirection direction = transceiver.getDirection(); final RtpTransceiver.RtpTransceiverDirection direction = transceiver.getDirection();
@ -51,13 +51,21 @@ class TrackWrapper<T extends MediaStreamTrack> {
} }
} }
public String getRtpSenderId() {
try {
return track.id();
} catch (final IllegalStateException e) {
return null;
}
}
public static <T extends MediaStreamTrack> RtpTransceiver getTransceiver( public static <T extends MediaStreamTrack> RtpTransceiver getTransceiver(
@Nonnull final PeerConnection peerConnection, final TrackWrapper<T> trackWrapper) { @Nonnull final PeerConnection peerConnection, final TrackWrapper<T> trackWrapper) {
final RtpSender rtpSender = trackWrapper.rtpSender; final String rtpSenderId = trackWrapper.getRtpSenderId();
for (final RtpTransceiver transceiver : peerConnection.getTransceivers()) { for (final RtpTransceiver transceiver : peerConnection.getTransceivers()) {
if (transceiver.getSender().id().equals(rtpSender.id())) { if (transceiver.getSender().id().equals(rtpSenderId)) {
return transceiver; return transceiver;
} }
} }
return null; return null;
} }

View file

@ -216,6 +216,14 @@ public class WebRTCWrapper {
} }
} }
private static void dispose(final VideoTrack videoTrack) {
try {
videoTrack.dispose();
} catch (final IllegalStateException e) {
Log.e(Config.LOGTAG, "unable to dispose of video track", e);
}
}
public void setup( public void setup(
final Context service, final Context service,
@Nonnull final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference) @Nonnull final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference)
@ -441,6 +449,7 @@ public class WebRTCWrapper {
final VideoSourceWrapper videoSourceWrapper = this.videoSourceWrapper; final VideoSourceWrapper videoSourceWrapper = this.videoSourceWrapper;
final AppRTCAudioManager audioManager = this.appRTCAudioManager; final AppRTCAudioManager audioManager = this.appRTCAudioManager;
final EglBase eglBase = this.eglBase; final EglBase eglBase = this.eglBase;
final var localVideoTrack = this.localVideoTrack;
if (peerConnection != null) { if (peerConnection != null) {
this.peerConnection = null; this.peerConnection = null;
dispose(peerConnection); dispose(peerConnection);
@ -449,7 +458,10 @@ public class WebRTCWrapper {
ToneManager.getInstance(context).setAppRtcAudioManagerHasControl(false); ToneManager.getInstance(context).setAppRtcAudioManagerHasControl(false);
mainHandler.post(audioManager::stop); mainHandler.post(audioManager::stop);
} }
this.localVideoTrack = null; if (localVideoTrack != null) {
this.localVideoTrack = null;
dispose(localVideoTrack.track);
}
this.remoteVideoTrack = null; this.remoteVideoTrack = null;
if (videoSourceWrapper != null) { if (videoSourceWrapper != null) {
this.videoSourceWrapper = null; this.videoSourceWrapper = null;
@ -461,8 +473,8 @@ public class WebRTCWrapper {
videoSourceWrapper.dispose(); videoSourceWrapper.dispose();
} }
if (eglBase != null) { if (eglBase != null) {
eglBase.release();
this.eglBase = null; this.eglBase = null;
eglBase.release();
} }
if (peerConnectionFactory != null) { if (peerConnectionFactory != null) {
this.peerConnectionFactory = null; this.peerConnectionFactory = null;

View file

@ -610,9 +610,13 @@ public class RtpSessionActivity extends BaseActivity
final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference; final WeakReference<JingleRtpConnection> weakReference = this.rtpConnectionReference;
final JingleRtpConnection jingleRtpConnection = final JingleRtpConnection jingleRtpConnection =
weakReference == null ? null : weakReference.get(); weakReference == null ? null : weakReference.get();
final var jmc = this.jingleConnectionManager;
if (jingleRtpConnection != null) { if (jingleRtpConnection != null) {
releaseVideoTracks(jingleRtpConnection); releaseVideoTracks(jingleRtpConnection);
} }
if (jmc != null) {
jmc.removeOnJingleRtpConnectionUpdate(this);
}
releaseProximityWakeLock(); releaseProximityWakeLock();
super.onStop(); super.onStop();
} }

View file

@ -20,6 +20,7 @@ public class SurfaceViewRenderer extends org.webrtc.SurfaceViewRenderer {
super(context, attrs); super(context, attrs);
} }
@Override
public void onFrameResolutionChanged(int videoWidth, int videoHeight, int rotation) { public void onFrameResolutionChanged(int videoWidth, int videoHeight, int rotation) {
super.onFrameResolutionChanged(videoWidth, videoHeight, rotation); super.onFrameResolutionChanged(videoWidth, videoHeight, rotation);
final int rotatedWidth = rotation != 0 && rotation != 180 ? videoHeight : videoWidth; final int rotatedWidth = rotation != 0 && rotation != 180 ? videoHeight : videoWidth;

View file

@ -1791,7 +1791,11 @@ public class XmppConnection extends AbstractAccountService implements Runnable {
this.sendPacket(response); this.sendPacket(response);
} }
public void sendErrorFor(final Iq request, final Error.Type type, final Condition condition) { public void sendErrorFor(
final Iq request,
final Error.Type type,
final Condition condition,
final Error.Extension... extensions) {
final var from = request.getFrom(); final var from = request.getFrom();
final var id = request.getId(); final var id = request.getId();
final var response = new Iq(Iq.Type.ERROR); final var response = new Iq(Iq.Type.ERROR);
@ -1800,6 +1804,7 @@ public class XmppConnection extends AbstractAccountService implements Runnable {
final Error error = response.addExtension(new Error()); final Error error = response.addExtension(new Error());
error.setType(type); error.setType(type);
error.setCondition(condition); error.setCondition(condition);
error.addExtensions(extensions);
this.sendPacket(response); this.sendPacket(response);
} }

View file

@ -32,6 +32,7 @@ import im.conversations.android.xml.Namespace;
import im.conversations.android.xmpp.XmppConnection; import im.conversations.android.xmpp.XmppConnection;
import im.conversations.android.xmpp.model.error.Condition; import im.conversations.android.xmpp.model.error.Condition;
import im.conversations.android.xmpp.model.error.Error; import im.conversations.android.xmpp.model.error.Error;
import im.conversations.android.xmpp.model.jingle.error.JingleCondition;
import im.conversations.android.xmpp.model.jmi.Accept; import im.conversations.android.xmpp.model.jmi.Accept;
import im.conversations.android.xmpp.model.jmi.JingleMessage; import im.conversations.android.xmpp.model.jmi.JingleMessage;
import im.conversations.android.xmpp.model.jmi.Proceed; import im.conversations.android.xmpp.model.jmi.Proceed;
@ -87,7 +88,10 @@ public class JingleConnectionManager extends AbstractManager {
final String sessionId = packet.getSessionId(); final String sessionId = packet.getSessionId();
if (sessionId == null) { if (sessionId == null) {
respondWithJingleError( respondWithJingleError(
iq, "unknown-session", Error.Type.CANCEL, new Condition.ItemNotFound()); iq,
new JingleCondition.UnknownSession(),
Error.Type.CANCEL,
new Condition.ItemNotFound());
return; return;
} }
final AbstractJingleConnection.Id id = AbstractJingleConnection.Id.of(packet); final AbstractJingleConnection.Id id = AbstractJingleConnection.Id.of(packet);
@ -123,9 +127,10 @@ public class JingleConnectionManager extends AbstractManager {
} }
connection = new JingleRtpConnection(context, this.connection, id, from); connection = new JingleRtpConnection(context, this.connection, id, from);
} else { } else {
// TODO this is probably the wrong jingle error condition
respondWithJingleError( respondWithJingleError(
packet, packet,
"unsupported-info", new JingleCondition.UnsupportedInfo(),
Error.Type.CANCEL, Error.Type.CANCEL,
new Condition.FeatureNotImplemented()); new Condition.FeatureNotImplemented());
return; return;
@ -135,7 +140,10 @@ public class JingleConnectionManager extends AbstractManager {
} else { } else {
Log.d(Config.LOGTAG, "unable to route jingle packet: " + packet); Log.d(Config.LOGTAG, "unable to route jingle packet: " + packet);
respondWithJingleError( respondWithJingleError(
packet, "unknown-session", Error.Type.CANCEL, new Condition.ItemNotFound()); packet,
new JingleCondition.UnknownSession(),
Error.Type.CANCEL,
new Condition.ItemNotFound());
} }
} }
@ -225,9 +233,11 @@ public class JingleConnectionManager extends AbstractManager {
} }
private void respondWithJingleError( private void respondWithJingleError(
final Iq original, String jingleCondition, final Error.Type type, Condition condition) { final Iq original,
// TODO add jingle condition final JingleCondition jingleCondition,
connection.sendErrorFor(original, type, condition); final Error.Type type,
Condition condition) {
connection.sendErrorFor(original, type, condition, jingleCondition);
} }
public void handle(final Message message) { public void handle(final Message message) {
@ -771,6 +781,12 @@ public class JingleConnectionManager extends AbstractManager {
this.onJingleRtpConnectionUpdate = listener; this.onJingleRtpConnectionUpdate = listener;
} }
public void removeOnJingleRtpConnectionUpdate(final OnJingleRtpConnectionUpdate listener) {
if (this.onJingleRtpConnectionUpdate == listener) {
this.onJingleRtpConnectionUpdate = null;
}
}
public RtpSessionNotification getNotificationService() { public RtpSessionNotification getNotificationService() {
return this.rtpSessionNotification; return this.rtpSessionNotification;
} }

View file

@ -28,10 +28,23 @@ public class Error extends Extension {
this.setAttribute("type", type.toString().toLowerCase(Locale.ROOT)); this.setAttribute("type", type.toString().toLowerCase(Locale.ROOT));
} }
public void addExtensions(final Extension[] extensions) {
for (final Extension extension : extensions) {
this.addExtension(extension);
}
}
public enum Type { public enum Type {
MODIFY, MODIFY,
CANCEL, CANCEL,
AUTH, AUTH,
WAIT WAIT
} }
public static class Extension extends im.conversations.android.xmpp.model.Extension {
public Extension(Class<? extends im.conversations.android.xmpp.model.Extension> clazz) {
super(clazz);
}
}
} }

View file

@ -0,0 +1,44 @@
package im.conversations.android.xmpp.model.jingle.error;
import im.conversations.android.annotation.XmlElement;
import im.conversations.android.xml.Namespace;
import im.conversations.android.xmpp.model.error.Error;
public abstract class JingleCondition extends Error.Extension {
private JingleCondition(Class<? extends JingleCondition> clazz) {
super(clazz);
}
@XmlElement(namespace = Namespace.JINGLE_ERRORS)
public static class OutOfOrder extends JingleCondition {
public OutOfOrder() {
super(OutOfOrder.class);
}
}
@XmlElement(namespace = Namespace.JINGLE_ERRORS)
public static class TieBreak extends JingleCondition {
public TieBreak() {
super(TieBreak.class);
}
}
@XmlElement(namespace = Namespace.JINGLE_ERRORS)
public static class UnknownSession extends JingleCondition {
public UnknownSession() {
super(UnknownSession.class);
}
}
@XmlElement(namespace = Namespace.JINGLE_ERRORS)
public static class UnsupportedInfo extends JingleCondition {
public UnsupportedInfo() {
super(UnsupportedInfo.class);
}
}
}