docs: Fix CLI documentation generation (#7600)

The new command is based on generating JSON for docusaurus consumption
rather than markdown. This is less error prone as manipulation of
markdown is better contained.

Signed-off-by: Charlie Egan <charlie@styra.com>

build/generate-cli-docs/generate.go

@@ -1,125 +1,98 @@
 package main
 
 import (
-	"bufio"
-	"fmt"
+	"encoding/json"
 	"log"
 	"os"
-	"path/filepath"
-	"regexp"
 	"strings"
 
-	"github.com/spf13/cobra/doc"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 
 	"github.com/open-policy-agent/opa/cmd"
 )
 
-const fileHeader = `---
-title: CLI
-kind: documentation
-weight: 90
-restrictedtoc: true
----
-
-The OPA executable provides the following commands. Note that command line arguments may either be provided as
-traditional flags, or as environment variables. The expected format of environment variables used for this purpose
-follows the pattern OPA_<COMMAND>_<FLAG> where COMMAND is the command name in uppercase (like EVAL) and FLAG is the
-flag name in uppercase (like STRICT), i.e. OPA_EVAL_STRICT would be equivalent to passing the --strict flag to the
-eval command.
-
-`
-
 func main() {
-	if len(os.Args) != 2 {
-		log.Fatal("Required argument: cli docs output directory")
-	}
-	out := os.Args[1]
-
 	command := cmd.RootCommand
 	command.Use = "opa [command]"
 	command.DisableAutoGenTag = true
 
-	dir, err := os.MkdirTemp("", "opa")
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer os.RemoveAll(dir)
+	cmdData := make([]map[string]any, 0)
 
-	err = doc.GenMarkdownTree(command, dir)
-	if err != nil {
-		log.Fatal(err) //nolint: gocritic
-	}
-
-	files, err := os.ReadDir(dir)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	builder := strings.Builder{}
-
-	last := len(files) - 1
-	for i, file := range files {
-		// Skip the first "opa" document as it's rather pointless to include (only shows the --help flag)
-		if i == 0 {
+	for _, c := range command.Commands() {
+		if !showCommand(c) {
 			continue
 		}
-		path := filepath.Join(dir, file.Name())
-		document, err := fixupSection(path)
-		if err != nil {
-			log.Fatal(err)
-		}
-		builder.WriteString(document)
-		if i != last {
-			builder.WriteString("____\n\n")
-		}
+
+		cmdData = append(cmdData, cmdToData(c))
 	}
 
-	heading := regexp.MustCompile(`^[\\-]+$`)
-	lines := strings.Split(builder.String(), "\n")
-	document := make([]string, 0, len(lines))
-	removed := 0
-
-	// The document may contain "----" for headings, which will be converted to h1
-	// elements in markdown. This is undesirable, so let's remove them and prepend
-	// the line before that with ### to instead create a h3
-	for line, str := range lines {
-		if heading.MatchString(str) {
-			document[line-1-removed] = fmt.Sprintf("### %s\n", document[line-1-removed])
-			removed++
-			continue
-		}
-		document = append(document, str+"\n")
-	}
-
-	withHeader := fmt.Sprintf("%s%s", fileHeader, strings.Join(document, ""))
-	err = os.WriteFile(filepath.Join(out, "cli.md"), []byte(withHeader), 0755)
+	err := json.NewEncoder(os.Stdout).Encode(cmdData)
 	if err != nil {
 		log.Fatal(err)
 	}
 }
 
-func fixupSection(path string) (string, error) {
-	f, err := os.Open(path)
-	if err != nil {
-		return "", err
-	}
-	defer f.Close()
-
-	scanner := bufio.NewScanner(f)
-	builder := strings.Builder{}
-
-	for scanner.Scan() {
-		line := scanner.Text()
-		// Remove "See also" section
-		if strings.Contains(line, "### SEE ALSO") {
-			break
-		}
-		if home := os.Getenv("HOME"); home != "" {
-			line = strings.ReplaceAll(line, home, "$HOME")
-		}
-		builder.WriteString(line)
-		builder.WriteString("\n")
+func showCommand(c *cobra.Command) bool {
+	if !c.IsAvailableCommand() ||
+		c.IsAdditionalHelpTopicCommand() ||
+		c.Hidden {
+		return false
 	}
 
-	return builder.String(), scanner.Err()
+	return true
+}
+
+func cmdToID(c *cobra.Command) string {
+	parts := strings.Split(c.Use, " ")
+
+	if len(parts) == 0 {
+		return ""
+	}
+
+	return parts[0]
+}
+
+func extractFlags(flagSet *pflag.FlagSet) []map[string]any {
+	var result []map[string]any
+
+	flagSet.VisitAll(func(f *pflag.Flag) {
+		flagInfo := map[string]any{
+			"name":        "--" + f.Name,
+			"shorthand":   "",
+			"type":        f.Value.Type(),
+			"default":     f.DefValue,
+			"description": f.Usage,
+		}
+
+		if f.Shorthand != "" {
+			flagInfo["shorthand"] = "-" + f.Shorthand
+		}
+
+		result = append(result, flagInfo)
+	})
+
+	return result
+}
+
+func cmdToData(c *cobra.Command) map[string]any {
+	childData := make([]map[string]any, 0)
+	for _, childCmd := range c.Commands() {
+		if !showCommand(childCmd) {
+			continue
+		}
+		childData = append(childData, cmdToData(childCmd))
+	}
+
+	return map[string]any{
+		"id":           cmdToID(c),
+		"use":          c.Use,
+		"useline":      c.UseLine(),
+		"short":        c.Short,
+		"long":         c.Long,
+		"example":      c.Example,
+		"flags":        extractFlags(c.NonInheritedFlags()),
+		"parent_flags": extractFlags(c.InheritedFlags()),
+		"children":     childData,
+	}
 }

cmd/check.go

@@ -169,9 +169,9 @@ func init() {
 		Short: "Check Rego source files",
 		Long: `Check Rego source files for parse and compilation errors.
 	
-	If the 'check' command succeeds in parsing and compiling the source file(s), no output
-	is produced. If the parsing or compiling fails, 'check' will output the errors
-	and exit with a non-zero exit code.`,
+If the 'check' command succeeds in parsing and compiling the source file(s), no output
+is produced. If the parsing or compiling fails, 'check' will output the errors
+and exit with a non-zero exit code.`,
 
 		PreRunE: func(cmd *cobra.Command, args []string) error {
 			if len(args) == 0 {

cmd/deps.go

@@ -57,7 +57,6 @@ func newDepsCommandParams() depsCommandParams {
 }
 
 func init() {
-
 	params := newDepsCommandParams()
 
 	depsCommand := &cobra.Command{
@@ -67,9 +66,9 @@ func init() {
 
 Dependencies are categorized as either base documents, which is any data loaded
 from the outside world, or virtual documents, i.e values that are computed from rules.
+`,
 
-Example
--------
+		Example: `
 Given a policy like this:
 
 	package policy
@@ -117,7 +116,6 @@ data.policy.is_admin.
 }
 
 func deps(args []string, params depsCommandParams, w io.Writer) error {
-
 	query, err := ast.ParseBody(args[0])
 	if err != nil {
 		return err

cmd/eval.go

@@ -35,9 +35,7 @@ import (
 	"github.com/open-policy-agent/opa/v1/util"
 )
 
-var (
-	errIllegalUnknownsArg = errors.New("illegal argument with --unknowns, specify string with one or more --unknowns")
-)
+var errIllegalUnknownsArg = errors.New("illegal argument with --unknowns, specify string with one or more --unknowns")
 
 type evalCommandParams struct {
 	capabilities              *capabilitiesFlag
@@ -114,7 +112,6 @@ func newEvalCommandParams() evalCommandParams {
 }
 
 func validateEvalParams(p *evalCommandParams, cmdArgs []string) error {
-
 	if len(cmdArgs) > 0 && p.stdin {
 		return errors.New("specify query argument or --stdin but not both")
 	} else if len(cmdArgs) == 0 && !p.stdin {
@@ -193,16 +190,13 @@ func (regoError) Error() string {
 }
 
 func init() {
-
 	params := newEvalCommandParams()
 
 	evalCommand := &cobra.Command{
 		Use:   "eval <query>",
 		Short: "Evaluate a Rego query",
-		Long: `Evaluate a Rego query and print the result.
-
-Examples
---------
+		Long:  `Evaluate a Rego query and print the result.`,
+		Example: `
 
 To evaluate a simple query:
 
@@ -310,7 +304,6 @@ access.
 			return env.CmdFlags.CheckEnvironmentVariables(cmd)
 		},
 		Run: func(_ *cobra.Command, args []string) {
-
 			defined, err := eval(args, params, os.Stdout)
 			if err != nil {
 				if _, ok := err.(regoError); !ok {
@@ -373,7 +366,6 @@ access.
 }
 
 func eval(args []string, params evalCommandParams, w io.Writer) (bool, error) {
-
 	ctx := context.Background()
 	if params.timeout != 0 {
 		var cancel func()
@@ -523,7 +515,7 @@ func evalOnce(ctx context.Context, ectx *evalContext) pr.Output {
 	}
 
 	if ectx.params.profile {
-		var sortOrder = pr.DefaultProfileSortOrder
+		sortOrder := pr.DefaultProfileSortOrder
 
 		if len(ectx.params.profileCriteria.v) != 0 {
 			sortOrder = getProfileSortOrder(strings.Split(ectx.params.profileCriteria.String(), ","))
@@ -745,7 +737,6 @@ func (r *resettableProfiler) TraceEvent(ev topdown.Event) { r.p.TraceEvent(ev) }
 func (r *resettableProfiler) Config() topdown.TraceConfig { return r.p.Config() }
 
 func getProfileSortOrder(sortOrder []string) []string {
-
 	// convert the sort order slice to a map for faster lookups
 	sortOrderMap := make(map[string]bool)
 	for _, cr := range sortOrder {

cmd/flags.go

@@ -102,7 +102,7 @@ func addSigningKeyFlag(fs *pflag.FlagSet, key *string) {
 }
 
 func addSigningPluginFlag(fs *pflag.FlagSet, plugin *string) {
-	fs.StringVarP(plugin, "signing-plugin", "", "", "name of the plugin to use for signing/verification (see https://www.openpolicyagent.org/docs/latest/management-bundles/#signature-plugin")
+	fs.StringVarP(plugin, "signing-plugin", "", "", "name of the plugin to use for signing/verification (see https://www.openpolicyagent.org/docs/latest/management-bundles/#signature-plugin)")
 }
 
 func addVerificationKeyFlag(fs *pflag.FlagSet, key *string) {
@@ -228,7 +228,6 @@ func (f *capabilitiesFlag) Set(s string) error {
 		return fmt.Errorf("no such file or capabilities version found: %v", s)
 	}
 	return nil
-
 }
 
 type stringptrFlag struct {

docs/Makefile

@@ -20,4 +20,4 @@ clean:
 
 .PHONY: generate-cli-docs
 generate-cli-docs:
-	$(CURDIR)/../build/gen-cli-docs.sh "$(CURDIR)/content"
+	$(CURDIR)/../build/gen-cli-docs.sh > $(CURDIR)/src/data/cli.json

docs/docusaurus.config.js

@@ -408,6 +408,26 @@ The Linux Foundation has registered trademarks and uses trademarks. For a list o
         };
       },
 
+      async function cliData(context, options) {
+        return {
+          name: "cli-data",
+
+          async loadContent() {
+            const filePath = path.join(context.siteDir, "src/data/cli.json");
+            const cliJson = await fs.readFile(filePath, "utf-8");
+            const parsedData = JSON.parse(cliJson);
+
+            return parsedData;
+          },
+
+          async contentLoaded({ content, actions }) {
+            const { createData } = actions;
+
+            await createData("cli.json", JSON.stringify(content, null, 2));
+          },
+        };
+      },
+
       async function versionsPageGen(context, options) {
         return {
           name: "version-page-gen",

docs/src/components/CommandDoc.js

@@ -0,0 +1,128 @@
+import { useThemeConfig } from "@docusaurus/theme-common";
+import React from "react";
+import { useEffect } from "react";
+import ReactMarkdown from "react-markdown";
+import { useLocation } from "react-router-dom";
+
+const capitalize = (str) => {
+  return str.charAt(0).toUpperCase() + str.slice(1);
+};
+
+const convertUrlsToMarkdownLinks = (text) => {
+  if (typeof text !== "string") {
+    text = String(text);
+  }
+
+  text = text.replace(/<\/?[^>]+(>|$)/g, "");
+
+  const urlRegex = /https?:\/\/(?:www\.)?[^\s<>"'()]+[^\s<>"'.),!?]/g;
+
+  return text.replace(urlRegex, (rawUrl) => {
+    const trailingMatch = rawUrl.match(/[.)]+$/);
+    const trailing = trailingMatch ? trailingMatch[0] : "";
+    const url = trailing ? rawUrl.slice(0, -trailing.length) : rawUrl;
+
+    const displayText = url
+      .replace(/^https?:\/\//, "")
+      .replace(/^www\./, "")
+      .replace(/\/$/, "");
+
+    return `[${displayText}](${url})${trailing}`;
+  });
+};
+
+const htmlSafe = (str) => {
+  return String(str)
+    .replace(/&/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&#39;");
+};
+
+const CommandDoc = ({ command }) => {
+  const { navbar } = useThemeConfig();
+  const location = useLocation();
+
+  const {
+    id,
+    use,
+    useline,
+    long,
+    example,
+    flags,
+  } = command;
+
+  return (
+    <div>
+      <div style={{ marginBottom: "0.5rem" }}>
+        <h2 id={id} style={{ display: "inline" }}>
+          {id}
+          <a href={`#${id}`} style={{ marginLeft: "0.3rem" }} title={`Direct link to ${id}`}>
+            <span style={{ color: "var(--ifm-link-color)" }}>#</span>
+          </a>
+        </h2>
+      </div>
+
+      {useline && (
+        <pre style={{ padding: "0.5em", borderRadius: "var(--ifm-code-border-radius)" }}>
+          {useline}
+        </pre>
+      )}
+
+      {long && <ReactMarkdown>{convertUrlsToMarkdownLinks(long)}</ReactMarkdown>}
+
+      {flags.length > 0 && (
+        <>
+          <h3>Flags</h3>
+          <table style={{ fontSize: "0.9em", borderCollapse: "collapse", width: "100%" }}>
+            <thead>
+              <tr>
+                <th style={{ textAlign: "left", padding: "0.5em" }}>Short</th>
+                <th style={{ textAlign: "left", padding: "0.5em" }}>Flag</th>
+                <th style={{ textAlign: "left", padding: "0.5em" }}>Description</th>
+              </tr>
+            </thead>
+            <tbody>
+              {flags.map((f, idx) => (
+                <tr key={idx}>
+                  <td style={{ whiteSpace: "nowrap", padding: "0.5em" }}>
+                    {f.shorthand && <code>{f.shorthand}</code>}
+                  </td>
+                  <td style={{ whiteSpace: "nowrap", padding: "0.5em" }}>
+                    <code>{f.name}</code>
+                  </td>
+                  <td style={{ padding: "0.5em", width: "100%" }}>
+                    {f.description !== "" && (
+                      <ReactMarkdown>
+                        {convertUrlsToMarkdownLinks(capitalize(htmlSafe(f.description)))}
+                      </ReactMarkdown>
+                    )}
+
+                    {f.type && f.type.startsWith("{") && (
+                      <div style={{ marginTop: "0.25em" }}>
+                        Accepts:{" "}
+                        <code style={{ wordBreak: "break-word" }}>
+                          {f.type.replace(/,/g, ",\u200b")}
+                        </code>
+                      </div>
+                    )}
+                  </td>
+                </tr>
+              ))}
+            </tbody>
+          </table>
+        </>
+      )}
+
+      {example && example.trim() !== "" && (
+        <>
+          <h3>Example</h3>
+          <ReactMarkdown>{example}</ReactMarkdown>
+        </>
+      )}
+    </div>
+  );
+};
+
+export default CommandDoc;

docs/src/components/CommandList.js

@@ -0,0 +1,71 @@
+import React, { useState } from "react";
+import CommandDoc from "./CommandDoc";
+
+function filterCommandById(command, query) {
+  const lowerQuery = query.toLowerCase();
+
+  const matches = (text) => text && text.toLowerCase().includes(lowerQuery);
+
+  const isMatch = matches(command.id);
+
+  // (charlieegan3) we don't have child commands for now and it might need to adjusted when
+  // we do, but I didn't want to ignore the fact they might exist in future.
+  let matchedChildren = [];
+  if (Array.isArray(command.children)) {
+    matchedChildren = command.children
+      .map(child => filterCommandById(child, query))
+      .filter(Boolean);
+  }
+
+  if (isMatch || matchedChildren.length > 0) {
+    return {
+      ...command,
+      children: matchedChildren.length > 0 ? matchedChildren : command.children,
+    };
+  }
+
+  return null;
+}
+
+const CommandList = ({ commands }) => {
+  const [search, setSearch] = useState("");
+
+  const filtered = commands
+    .map(cmd => filterCommandById(cmd, search))
+    .filter(Boolean);
+
+  const totalCommands = commands.length;
+  const filteredCommands = filtered.length;
+
+  return (
+    <div>
+      <input
+        type="text"
+        placeholder="Search by command name..."
+        value={search}
+        onChange={(e) => setSearch(e.target.value)}
+        style={{
+          width: "100%",
+          padding: "0.5rem",
+          marginBottom: "1rem",
+          fontSize: "1rem",
+          border: "1px solid #ccc",
+          borderRadius: "4px",
+        }}
+      />
+
+      {filteredCommands !== totalCommands && filteredCommands > 0 && (
+        <p style={{ marginBottom: "1rem" }}>
+          Showing {filteredCommands}/{totalCommands} commands
+          {filtered.length > 1 && "(" + filtered.map(cmd => cmd.id).join(", ") + ")"}
+        </p>
+      )}
+
+      {filteredCommands === 0 ? <p>No matching commands found.</p> : (
+        filtered.map((cmd, idx) => <CommandDoc key={cmd.id || idx} command={cmd} />)
+      )}
+    </div>
+  );
+};
+
+export default CommandList;