ewilliams28/releases
1
0
Fork 0
mirror of https://github.com/open-policy-agent/opa.git synced 2026-08-12 19:32:48 -06:00
Code Issues Packages Projects Releases Wiki Activity

cmd: Avoid intermediate buffer when writing bundle (#8909)

Browse Source 
While likely not important for small bundles, using `opa build` to build
large bundles would previously allocate much more memory than was
needed, as the bundle would first be written to an intermediate
in-memory buffer before getting written to disk. This fixes that by
deferring the creation of the output file to the first write, then
writing to that directly.

Signed-off-by: Anders Eknert <anders.eknert@apple.com>
This commit is contained in:
Anders Eknert
2026-07-20 20:29:46 +02:00
committed by GitHub
parent 4cefcaff21
commit 5a33432bd4
1 changed files with 69 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/build.go
+42 -34
View File
@@ -5,7 +5,6 @@
package cmd
import (
"bytes"
"context"
"errors"
"fmt"
@@ -26,7 +25,8 @@ import (
const defaultPublicKeyID = "default"
type buildParams struct {
type (
buildParams struct {
capabilities *capabilitiesFlag
target *util.EnumFlag
planFormat *util.EnumFlag
@@ -53,6 +53,23 @@ type buildParams struct {
wasmIncludePrint bool
stderr io.Writer
}
// deferredFileWriter is a wrapper around [*os.File] that defers the creation of the file until
// the first write, which allows us to pass a file writer without using an intermediate buffer,
// and without creating any file in case the build fails.
deferredFileWriter struct {
*os.File
path string
}
)
func (w *deferredFileWriter) Write(p []byte) (n int, err error) {
if w.File == nil {
if w.File, err = os.Create(w.path); err != nil {
return 0, err
}
}
return w.File.Write(p)
}
func newBuildParams() buildParams {
return buildParams{
@@ -290,8 +307,6 @@ against ` + brand + ` v0.22.0:
}
func dobuild(params buildParams, args []string) error {
buf := bytes.NewBuffer(nil)
// generate the bundle verification and signing config
bvc, err := buildVerificationConfig(params.pubKey, params.pubKeyID, params.algorithm, params.scope, params.excludeVerifyFiles)
if err != nil {
@@ -303,12 +318,12 @@ func dobuild(params buildParams, args []string) error {
return err
}
if (bvc != nil || bsc != nil) && !params.bundleMode {
// if manifest files are found in the input directories and the -b flag is not set, this is likely a mistake.
if !params.bundleMode {
if bvc != nil || bsc != nil {
return errors.New("enable bundle mode (ie. --bundle) to verify or sign bundle files or directories")
}
// if manifest files are found in the input directories and the -b flag is not set, this is likely a mistake.
if !params.bundleMode {
for _, arg := range args {
stat, err := os.Stat(arg)
if err != nil || !stat.IsDir() {
@@ -317,27 +332,27 @@ func dobuild(params buildParams, args []string) error {
for _, name := range []string{bundle.ManifestExt, bundle.ManifestProtoExt} {
if _, err := os.Stat(filepath.Join(arg, name)); err == nil {
fmt.Fprintf(params.stderr, "Warning: %s file found in %q but -b flag not specified. Manifest will be ignored.\n", name, arg)
fmt.Fprintf(params.stderr,
"Warning: %s file found in %q but -b flag not specified. Manifest will be ignored.\n",
name, arg,
)
break
}
}
}
}
capabilities := params.capabilities.C
if capabilities == nil {
// ensure custom builtins are properly captured
capabilities = ast.CapabilitiesForThisVersion(ast.CapabilitiesRegoVersion(params.regoVersion()))
}
out := &deferredFileWriter{path: params.outputFile}
defer out.Close()
compiler := compile.New().
WithCapabilities(capabilities).
WithCapabilities(util.Or(params.capabilities.C, capabilitiesForParamsVersion(params))).
WithTarget(params.target.String()).
WithPlanFormat(params.planFormat.String()).
WithAsBundle(params.bundleMode).
WithPruneUnused(params.pruneUnused).
WithOptimizationLevel(params.optimizationLevel).
WithOutput(buf).
WithOutput(out).
WithEntrypoints(params.entrypoints.v...).
WithRegoAnnotationEntrypoints(true).
WithPaths(args...).
@@ -370,22 +385,7 @@ func dobuild(params buildParams, args []string) error {
compiler = compiler.WithEnablePrintStatements(params.wasmIncludePrint)
}
err = compiler.Build(context.Background())
if err != nil {
return err
}
out, err := os.Create(params.outputFile)
if err != nil {
return err
}
_, err = io.Copy(out, buf)
if err != nil {
return err
}
return out.Close()
return compiler.Build(context.Background())
}
func buildCommandLoaderFilter(bundleMode bool, ignore []string) func(string, os.FileInfo, int) bool {
@@ -408,15 +408,23 @@ func buildVerificationConfig(pubKey, pubKeyID, alg, scope string, excludeFiles [
if err != nil {
return nil, err
}
return bundle.NewVerificationConfig(map[string]*keys.Config{pubKeyID: keyConfig}, pubKeyID, scope, excludeFiles), nil
confMap := map[string]*keys.Config{pubKeyID: keyConfig}
return bundle.NewVerificationConfig(confMap, pubKeyID, scope, excludeFiles), nil
}
func buildSigningConfig(key, alg, claimsFile, plugin string) (*bundle.SigningConfig, error) {
if key == "" && (plugin != "" || claimsFile != "") {
if key == "" {
if plugin != "" || claimsFile != "" {
return nil, errSigningConfigIncomplete
}
if key == "" {
return nil, nil
}
return bundle.NewSigningConfig(key, alg, claimsFile).WithPlugin(plugin), nil
}
func capabilitiesForParamsVersion(params buildParams) func() *ast.Capabilities {
return func() *ast.Capabilities {
return ast.CapabilitiesForThisVersion(ast.CapabilitiesRegoVersion(params.regoVersion()))
}
}