Limit heap memory usage when compressing high number of concurrent responses

Previously each concurrent compression could allocate huge compression state
with the size up to 1Mb each. So 10K concurrent connections could result in
10Gb of compression state in the heap.

This CL limits the number of compression states among concurrent requests
when {Append,Write}{Gzip,Deflate}* functions are called to O(GOMAXPROCS).
These functions are used by CompressHandler* for non-streaming responses,
i.e. it should cover the majority of use cases.

Memory usage for 10K concurrent connections that compress responses drops
from 10Gb to 200Mb after this CL.

1 files changed, 3 insertions, 8 deletions

diff --git a/fs.go b/fs.go
index c36073d..0d83b60 100644
--- a/fs.go
+++ b/fs.go
@@ -958,12 +958,7 @@ func (h *fsHandler) createDirIndex(base *URI, dirPath string, mustCompress bool)
 
 	if mustCompress {
 		var zbuf ByteBuffer
-		zw := acquireGzipWriter(&zbuf, CompressDefaultCompression)
-		_, err = zw.Write(w.B)
-		releaseGzipWriter(zw)
-		if err != nil {
-			return nil, fmt.Errorf("error when compressing automatically generated index for directory %q: %s", dirPath, err)
-		}
+		zbuf.B = AppendGzipBytesLevel(zbuf.B, w.B, CompressDefaultCompression)
 		w = &zbuf
 	}
 
@@ -1048,12 +1043,12 @@ func (h *fsHandler) compressFileNolock(f *os.File, fileInfo os.FileInfo, filePat
 		return nil, errNoCreatePermission
 	}
 
-	zw := acquireGzipWriter(zf, CompressDefaultCompression)
+	zw := acquireStacklessGzipWriter(zf, CompressDefaultCompression)
 	_, err = copyZeroAlloc(zw, f)
 	if err1 := zw.Flush(); err == nil {
 		err = err1
 	}
-	releaseGzipWriter(zw)
+	releaseStacklessGzipWriter(zw, CompressDefaultCompression)
 	zf.Close()
 	f.Close()
 	if err != nil {