Skip to content

Commit 057575f

Browse files
sbordetsbordet
committedFeb 28, 2019
Issue #3373 - OutOfMemoryError: Java heap space in GZIPContentDecoder.
Modified jetty-client content decoding to be fully non-blocking; this allows for a better backpressure and less usage of the buffer pool. Modified GZIPContentDecoder to aggregate decoded ByteBuffers in a smarter way that avoids too many data copies and pollution of the buffer pool with intermediate size buffers. Removed duplicate test GZIPContentDecoderTest. Improved javadocs and improved AsyncMiddleManServlet to release buffers used by the GZIPContentDecoder. Signed-off-by: Simone Bordet <simone.bordet@gmail.com>
·
jetty-12.1.0.beta0jetty-9.4.16.v20190411
1 parent 6196ff1 commit 057575f

File tree

9 files changed

+368
-566
lines changed

9 files changed

+368
-566
lines changed
 

‎jetty-client/src/main/java/org/eclipse/jetty/client/ContentDecoder.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,15 @@ public interface ContentDecoder
3535
*/
3636
public abstract ByteBuffer decode(ByteBuffer buffer);
3737

38+
/**
39+
* <p>Releases the ByteBuffer returned by {@link #decode(ByteBuffer)}.</p>
40+
*
41+
* @param decoded the ByteBuffer returned by {@link #decode(ByteBuffer)}
42+
*/
43+
public default void release(ByteBuffer decoded)
44+
{
45+
}
46+
3847
/**
3948
* Factory for {@link ContentDecoder}s; subclasses must implement {@link #newContentDecoder()}.
4049
* <p>

‎jetty-client/src/main/java/org/eclipse/jetty/client/GZIPContentDecoder.java

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,16 @@
1818

1919
package org.eclipse.jetty.client;
2020

21+
import java.nio.ByteBuffer;
22+
2123
import org.eclipse.jetty.io.ByteBufferPool;
2224

2325
/**
2426
* {@link ContentDecoder} for the "gzip" encoding.
2527
*/
2628
public class GZIPContentDecoder extends org.eclipse.jetty.http.GZIPContentDecoder implements ContentDecoder
2729
{
28-
private static final int DEFAULT_BUFFER_SIZE = 2048;
30+
public static final int DEFAULT_BUFFER_SIZE = 8192;
2931

3032
public GZIPContentDecoder()
3133
{
@@ -42,6 +44,13 @@ public GZIPContentDecoder(ByteBufferPool byteBufferPool, int bufferSize)
4244
super(byteBufferPool, bufferSize);
4345
}
4446

47+
@Override
48+
protected boolean decodedChunk(ByteBuffer chunk)
49+
{
50+
super.decodedChunk(chunk);
51+
return true;
52+
}
53+
4554
/**
4655
* Specialized {@link ContentDecoder.Factory} for the "gzip" encoding.
4756
*/

‎jetty-client/src/main/java/org/eclipse/jetty/client/HttpReceiver.java

Lines changed: 45 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import java.io.IOException;
2222
import java.net.URI;
2323
import java.nio.ByteBuffer;
24-
import java.util.ArrayList;
2524
import java.util.Collections;
2625
import java.util.Enumeration;
2726
import java.util.HashMap;
@@ -37,7 +36,7 @@
3736
import org.eclipse.jetty.http.HttpStatus;
3837
import org.eclipse.jetty.util.BufferUtil;
3938
import org.eclipse.jetty.util.Callback;
40-
import org.eclipse.jetty.util.CountingCallback;
39+
import org.eclipse.jetty.util.IteratingNestedCallback;
4140
import org.eclipse.jetty.util.component.Destroyable;
4241
import org.eclipse.jetty.util.log.Log;
4342
import org.eclipse.jetty.util.log.Logger;
@@ -339,35 +338,7 @@ protected boolean responseContent(HttpExchange exchange, ByteBuffer buffer, Call
339338
}
340339
else
341340
{
342-
try
343-
{
344-
List<ByteBuffer> decodeds = new ArrayList<>(2);
345-
while (buffer.hasRemaining())
346-
{
347-
ByteBuffer decoded = decoder.decode(buffer);
348-
if (!decoded.hasRemaining())
349-
continue;
350-
decodeds.add(decoded);
351-
if (LOG.isDebugEnabled())
352-
LOG.debug("Response content decoded ({}) {}{}{}", decoder, response, System.lineSeparator(), BufferUtil.toDetailString(decoded));
353-
}
354-
355-
if (decodeds.isEmpty())
356-
{
357-
callback.succeeded();
358-
}
359-
else
360-
{
361-
int size = decodeds.size();
362-
CountingCallback counter = new CountingCallback(callback, size);
363-
for (ByteBuffer decoded : decodeds)
364-
notifier.notifyContent(response, decoded, counter, contentListeners);
365-
}
366-
}
367-
catch (Throwable x)
368-
{
369-
callback.failed(x);
370-
}
341+
new Decoder(notifier, response, decoder, buffer, callback).iterate();
371342
}
372343

373344
if (updateResponseState(ResponseState.TRANSIENT, ResponseState.CONTENT))
@@ -615,4 +586,47 @@ private enum ResponseState
615586
*/
616587
FAILURE
617588
}
589+
590+
private class Decoder extends IteratingNestedCallback
591+
{
592+
private final ResponseNotifier notifier;
593+
private final HttpResponse response;
594+
private final ContentDecoder decoder;
595+
private final ByteBuffer buffer;
596+
private ByteBuffer decoded;
597+
598+
public Decoder(ResponseNotifier notifier, HttpResponse response, ContentDecoder decoder, ByteBuffer buffer, Callback callback)
599+
{
600+
super(callback);
601+
this.notifier = notifier;
602+
this.response = response;
603+
this.decoder = decoder;
604+
this.buffer = buffer;
605+
}
606+
607+
@Override
608+
protected Action process() throws Throwable
609+
{
610+
while (true)
611+
{
612+
decoded = decoder.decode(buffer);
613+
if (decoded.hasRemaining())
614+
break;
615+
if (!buffer.hasRemaining())
616+
return Action.SUCCEEDED;
617+
}
618+
if (LOG.isDebugEnabled())
619+
LOG.debug("Response content decoded ({}) {}{}{}", decoder, response, System.lineSeparator(), BufferUtil.toDetailString(decoded));
620+
621+
notifier.notifyContent(response, decoded, this, contentListeners);
622+
return Action.SCHEDULED;
623+
}
624+
625+
@Override
626+
public void succeeded()
627+
{
628+
decoder.release(decoded);
629+
super.succeeded();
630+
}
631+
}
618632
}

0 commit comments

Comments
 (0)
Please sign in to comment.