cmd: Avoid intermediate buffer when writing bundle (#8909)

While likely not important for small bundles, using `opa build` to build
large bundles would previously allocate much more memory than was
needed, as the bundle would first be written to an intermediate
in-memory buffer before getting written to disk. This fixes that by
deferring the creation of the output file to the first write, then
writing to that directly.

Signed-off-by: Anders Eknert <anders.eknert@apple.com>

cmd/build.go

@@ -5,7 +5,6 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -26,32 +25,50 @@ import (
 
 const defaultPublicKeyID = "default"
 
-type buildParams struct {
+type (
-	capabilities       *capabilitiesFlag
+	buildParams struct {
-	target             *util.EnumFlag
+		capabilities       *capabilitiesFlag
-	planFormat         *util.EnumFlag
+		target             *util.EnumFlag
-	bundleMode         bool
+		planFormat         *util.EnumFlag
-	pruneUnused        bool
+		bundleMode         bool
-	optimizationLevel  int
+		pruneUnused        bool
-	entrypoints        repeatedStringFlag
+		optimizationLevel  int
-	outputFile         string
+		entrypoints        repeatedStringFlag
-	revision           stringptrFlag
+		outputFile         string
-	ignore             []string
+		revision           stringptrFlag
-	debug              bool
+		ignore             []string
-	algorithm          string
+		debug              bool
-	key                string
+		algorithm          string
-	scope              string
+		key                string
-	pubKey             string
+		scope              string
-	pubKeyID           string
+		pubKey             string
-	claimsFile         string
+		pubKeyID           string
-	excludeVerifyFiles []string
+		claimsFile         string
-	plugin             string
+		excludeVerifyFiles []string
-	ns                 string
+		plugin             string
-	v0Compatible       bool
+		ns                 string
-	v1Compatible       bool
+		v0Compatible       bool
-	followSymlinks     bool
+		v1Compatible       bool
-	wasmIncludePrint   bool
+		followSymlinks     bool
-	stderr             io.Writer
+		wasmIncludePrint   bool
+		stderr             io.Writer
+	}
+	// deferredFileWriter is a wrapper around [*os.File] that defers the creation of the file until
+	// the first write, which allows us to pass a file writer without using an intermediate buffer,
+	// and without creating any file in case the build fails.
+	deferredFileWriter struct {
+		*os.File
+		path string
+	}
+)
+
+func (w *deferredFileWriter) Write(p []byte) (n int, err error) {
+	if w.File == nil {
+		if w.File, err = os.Create(w.path); err != nil {
+			return 0, err
+		}
+	}
+	return w.File.Write(p)
 }
 
 func newBuildParams() buildParams {
@@ -290,8 +307,6 @@ against ` + brand + ` v0.22.0:
 }
 
 func dobuild(params buildParams, args []string) error {
-	buf := bytes.NewBuffer(nil)
-
 	// generate the bundle verification and signing config
 	bvc, err := buildVerificationConfig(params.pubKey, params.pubKeyID, params.algorithm, params.scope, params.excludeVerifyFiles)
 	if err != nil {
@@ -303,12 +318,12 @@ func dobuild(params buildParams, args []string) error {
 		return err
 	}
 
-	if (bvc != nil || bsc != nil) && !params.bundleMode {
-		return errors.New("enable bundle mode (ie. --bundle) to verify or sign bundle files or directories")
-	}
-
 	// if manifest files are found in the input directories and the -b flag is not set, this is likely a mistake.
 	if !params.bundleMode {
+		if bvc != nil || bsc != nil {
+			return errors.New("enable bundle mode (ie. --bundle) to verify or sign bundle files or directories")
+		}
+
 		for _, arg := range args {
 			stat, err := os.Stat(arg)
 			if err != nil || !stat.IsDir() {
@@ -317,27 +332,27 @@ func dobuild(params buildParams, args []string) error {
 
 			for _, name := range []string{bundle.ManifestExt, bundle.ManifestProtoExt} {
 				if _, err := os.Stat(filepath.Join(arg, name)); err == nil {
-					fmt.Fprintf(params.stderr, "Warning: %s file found in %q but -b flag not specified. Manifest will be ignored.\n", name, arg)
+					fmt.Fprintf(params.stderr,
+						"Warning: %s file found in %q but -b flag not specified. Manifest will be ignored.\n",
+						name, arg,
+					)
 					break
 				}
 			}
 		}
 	}
 
-	capabilities := params.capabilities.C
+	out := &deferredFileWriter{path: params.outputFile}
-	if capabilities == nil {
+	defer out.Close()
-		// ensure custom builtins are properly captured
-		capabilities = ast.CapabilitiesForThisVersion(ast.CapabilitiesRegoVersion(params.regoVersion()))
-	}
 
 	compiler := compile.New().
-		WithCapabilities(capabilities).
+		WithCapabilities(util.Or(params.capabilities.C, capabilitiesForParamsVersion(params))).
 		WithTarget(params.target.String()).
 		WithPlanFormat(params.planFormat.String()).
 		WithAsBundle(params.bundleMode).
 		WithPruneUnused(params.pruneUnused).
 		WithOptimizationLevel(params.optimizationLevel).
-		WithOutput(buf).
+		WithOutput(out).
 		WithEntrypoints(params.entrypoints.v...).
 		WithRegoAnnotationEntrypoints(true).
 		WithPaths(args...).
@@ -370,22 +385,7 @@ func dobuild(params buildParams, args []string) error {
 		compiler = compiler.WithEnablePrintStatements(params.wasmIncludePrint)
 	}
 
-	err = compiler.Build(context.Background())
+	return compiler.Build(context.Background())
-	if err != nil {
-		return err
-	}
-
-	out, err := os.Create(params.outputFile)
-	if err != nil {
-		return err
-	}
-
-	_, err = io.Copy(out, buf)
-	if err != nil {
-		return err
-	}
-
-	return out.Close()
 }
 
 func buildCommandLoaderFilter(bundleMode bool, ignore []string) func(string, os.FileInfo, int) bool {
@@ -408,15 +408,23 @@ func buildVerificationConfig(pubKey, pubKeyID, alg, scope string, excludeFiles [
 	if err != nil {
 		return nil, err
 	}
-	return bundle.NewVerificationConfig(map[string]*keys.Config{pubKeyID: keyConfig}, pubKeyID, scope, excludeFiles), nil
+	confMap := map[string]*keys.Config{pubKeyID: keyConfig}
+
+	return bundle.NewVerificationConfig(confMap, pubKeyID, scope, excludeFiles), nil
 }
 
 func buildSigningConfig(key, alg, claimsFile, plugin string) (*bundle.SigningConfig, error) {
-	if key == "" && (plugin != "" || claimsFile != "") {
-		return nil, errSigningConfigIncomplete
-	}
 	if key == "" {
+		if plugin != "" || claimsFile != "" {
+			return nil, errSigningConfigIncomplete
+		}
 		return nil, nil
 	}
 	return bundle.NewSigningConfig(key, alg, claimsFile).WithPlugin(plugin), nil
 }
+
+func capabilitiesForParamsVersion(params buildParams) func() *ast.Capabilities {
+	return func() *ast.Capabilities {
+		return ast.CapabilitiesForThisVersion(ast.CapabilitiesRegoVersion(params.regoVersion()))
+	}
+}