ast+rego+topdown: external rule source support (#8600)

External rule sources let wrapping projects inject rules at evaluation
time instead of compile time. The compiler marks external packages in
the rule tree but doesn't index them. When topdown hits an external
node, it calls Lookup to get rules, compiles them on the fly with a
scoped compiler, grafts the result into the tree, and caches it for the
duration of the evaluation.

Sources can be isolated (default, no access to surrounding policy) or
non-isolated (can reference static rules and other external sources).
The ExternalRuleIndexCloser interface handles cleanup after evaluation.
Precompiled rules can skip compiler stages via SkippedStages to avoid
redundant work.

This includes:
* hooks: add BundlePreActivate hook This one is handy when registering
external sources.

* topdown: catch `ir == nil` rule index result
This wouldn't ordinarily happen: the compiler is checking refs before.
But in our use case, the SP rules may be configured to be able to reach
into the surrounding Rego (non-isolated mode). If that happens, the IR
lookup may indeed end up as `nil, nil`.

Signed-off-by: Stephan Renatus <stephan.renatus@gmail.com>

internal/runtime/init/init.go

@@ -31,6 +31,7 @@ type InsertAndCompileOptions struct {
 	EnablePrintStatements bool
 	ParserOptions         ast.ParserOptions
 	BundleActivatorPlugin string
+	ExternalSources       *util.HasherMap[ast.Ref, ast.ExternalRuleSource]
 }
 
 // InsertAndCompileResult contains the output of the operation.
@@ -59,18 +60,31 @@ func InsertAndCompile(ctx context.Context, opts InsertAndCompileOptions) (*Inser
 		SetErrorLimit(opts.MaxErrors).
 		WithPathConflictsCheck(storage.NonEmpty(ctx, opts.Store, opts.Txn)).
 		WithEnablePrintStatements(opts.EnablePrintStatements)
+
+	// Apply external sources to the compiler before bundle activation.
+	// Bundle activation applies them again via compileModules, but we need them
+	// here too: there may be no bundles, or a custom activator plugin may not
+	// call compileModules.
+	if opts.ExternalSources != nil {
+		opts.ExternalSources.Iter(func(ref ast.Ref, source ast.ExternalRuleSource) bool {
+			compiler = compiler.WithExternalSource(ref, source)
+			return false
+		})
+	}
+
 	m := metrics.New()
 
 	activation := &bundle.ActivateOpts{
-		Ctx:           ctx,
-		Store:         opts.Store,
-		Txn:           opts.Txn,
-		Compiler:      compiler,
-		Metrics:       m,
-		Bundles:       opts.Bundles,
-		ExtraModules:  policies,
-		ParserOptions: opts.ParserOptions,
-		Plugin:        opts.BundleActivatorPlugin,
+		Ctx:             ctx,
+		Store:           opts.Store,
+		Txn:             opts.Txn,
+		Compiler:        compiler,
+		Metrics:         m,
+		Bundles:         opts.Bundles,
+		ExtraModules:    policies,
+		ExternalSources: opts.ExternalSources,
+		ParserOptions:   opts.ParserOptions,
+		Plugin:          opts.BundleActivatorPlugin,
 	}
 
 	err := bundle.Activate(activation)