download: copy bundle into buffer only if needed (#5767)

The need would be if `d.persist` is true: we're writing it into a file
in a later code path.

However, if we don't have bundle persistency enabled, this should allow
us to get rid of one copy of the bundle in memory. To achieve that, the
mechanism used to figure out the bundle size is replaced:

- Before, we'd write it into a buffer that we might not need, and use
  its Len()
- After, we're writing it into a no-op io.Writer that only keeps track
  of the bytes read.

Signed-off-by: Stephan Renatus <stephan@styra.com>

download/download.go

@@ -315,7 +315,16 @@ func (d *Downloader) download(ctx context.Context, m metrics.Metrics) (*download
 			defer m.Timer(metrics.RegoLoadBundles).Stop()
 			baseURL := path.Join(d.client.Config().URL, d.path)
 
-			loader := bundle.NewTarballLoaderWithBaseURL(io.TeeReader(resp.Body, &buf), baseURL)
+			cnt := &count{}
+			r := io.TeeReader(resp.Body, cnt)
+
+			var loader bundle.DirectoryLoader
+			if d.persist {
+				tee := io.TeeReader(r, &buf)
+				loader = bundle.NewTarballLoaderWithBaseURL(tee, baseURL)
+			} else {
+				loader = bundle.NewTarballLoaderWithBaseURL(r, baseURL)
+			}
 
 			etag := resp.Header.Get("ETag")
 
@@ -356,7 +365,7 @@ func (d *Downloader) download(ctx context.Context, m metrics.Metrics) (*download
 				raw:      &buf,
 				etag:     etag,
 				longPoll: isLongPollSupported(resp.Header),
-				size:     buf.Len(),
+				size:     cnt.Bytes(),
 			}, nil
 		}
 
@@ -383,6 +392,20 @@ func (d *Downloader) download(ctx context.Context, m metrics.Metrics) (*download
 	}
 }
 
+type count struct {
+	total int
+}
+
+func (c *count) Write(p []byte) (n int, err error) {
+	n = len(p)
+	c.total += n
+	return
+}
+
+func (c *count) Bytes() int {
+	return c.total
+}
+
 func isLongPollSupported(header http.Header) bool {
 	return header.Get("Content-Type") == "application/vnd.openpolicyagent.bundles"
 }