Add CLI section to docs (#4241)

Fixes #3915

Signed-off-by: Anders Eknert <anders@eknert.com>

.github/workflows/post-merge.yaml

@@ -45,6 +45,45 @@ jobs:
             echo "No generated changes to push!"
           fi
 
+  generate-cli-docs:
+    name: Sync Generated CLI Docs
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          token: ${{ secrets.GH_PUSH_TOKEN }}
+
+      - name: Generate
+        run: make docs-generate-cli-docs
+
+      - name: Commit & Push
+        shell: bash
+        run: |
+          # Commit any changes and push as needed.
+
+          # See https://github.com/actions/checkout#push-a-commit-using-the-built-in-token
+          AUTHOR=cli-docs-updater
+          git config user.name ${AUTHOR}
+          git config user.email ${AUTHOR}@github.com
+
+          # Prevent looping if the build was non-deterministic..
+          CAN_PUSH=1
+          if [[ "$(git log -1 --pretty=format:'%an')" == "${AUTHOR}" ]]; then
+              CAN_PUSH=0
+          fi
+
+          if ./build/commit-cli-docs.sh; then
+            if [[ "${CAN_PUSH}" == "1" ]]; then
+              git push
+            else
+              echo "Previous commit was auto-generated -- Aborting!"
+              exit 1
+            fi
+          else
+            echo "No generated changes to push!"
+          fi
+
   code-coverage:
     name: Update Go Test Coverage
     runs-on: ubuntu-18.04

build/commit-cli-docs.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+OPA_DIR=$(dirname "${BASH_SOURCE}")/..
+
+cd "${OPA_DIR}"
+
+git add docs/content/cli.md
+
+if [[ -z "$(git diff --name-only --cached)" ]]; then
+  echo "No CLI doc changes to commit"
+  exit 1
+fi
+
+git commit -m "docs: Update generated CLI docs"
+
+echo ""
+echo "Committed changes for files:"
+git diff-tree --no-commit-id --name-only -r HEAD
+echo ""

build/gen-cli-docs.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
+
+GOOS="" GOARCH="" go run "$SCRIPT_DIR"/generate-cli-docs/generate.go "$@"

build/generate-cli-docs/generate.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/spf13/cobra/doc"
+
+	"github.com/open-policy-agent/opa/cmd"
+)
+
+const fileHeader = `---
+title: CLI
+kind: documentation
+weight: 90
+restrictedtoc: true
+---
+
+The OPA executable provides the following commands.
+
+`
+
+func main() {
+	if len(os.Args) != 2 {
+		log.Fatal("Required argument: cli docs output directory")
+	}
+	out := os.Args[1]
+
+	command := cmd.RootCommand
+	command.Use = "opa [command]"
+	command.DisableAutoGenTag = true
+
+	dir, err := ioutil.TempDir("", "opa")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	err = doc.GenMarkdownTree(command, dir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	builder := strings.Builder{}
+
+	last := len(files) - 1
+	for i, file := range files {
+		// Skip the first "opa" document as it's rather pointless to include (only shows the --help flag)
+		if i == 0 {
+			continue
+		}
+		path := filepath.Join(dir, file.Name())
+		document, err := fixupSection(path)
+		if err != nil {
+			log.Fatal(err)
+		}
+		builder.WriteString(document)
+		if i != last {
+			builder.WriteString("____\n\n")
+		}
+	}
+
+	heading := regexp.MustCompile(`^[\\-]+$`)
+	var document []string
+	removed := 0
+
+	// The document may contain "----" for headings, which will be converted to h1
+	// elements in markdown. This is undesirable, so let's remove them and prepend
+	// the line before that with ### to instead create a h3
+	for line, str := range strings.Split(builder.String(), "\n") {
+		if heading.Match([]byte(str)) {
+			document[line-1-removed] = fmt.Sprintf("### %s\n", document[line-1-removed])
+			removed++
+			continue
+		}
+		document = append(document, fmt.Sprintf("%s\n", str))
+	}
+
+	withHeader := fmt.Sprintf("%s%s", fileHeader, strings.Join(document, ""))
+	err = ioutil.WriteFile(filepath.Join(out, "cli.md"), []byte(withHeader), 0755)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func fixupSection(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	builder := strings.Builder{}
+
+	for scanner.Scan() {
+		// Remove "See also" section
+		if strings.Contains(scanner.Text(), "### SEE ALSO") {
+			break
+		}
+		builder.WriteString(scanner.Text())
+		builder.WriteString("\n")
+	}
+
+	return builder.String(), scanner.Err()
+}

cmd/build.go

@@ -85,9 +85,10 @@ Inside another terminal in the same directory, serve the bundle via HTTP:
 
     $ python3 -m http.server --bind localhost 8080
 
-For more information on bundles see https://www.openpolicyagent.org/docs/latest/management.
+For more information on bundles see https://www.openpolicyagent.org/docs/latest/management-bundles/.
 
-## Common Flags
+Common Flags
+------------
 
 When -b is specified the 'build' command assumes paths refer to existing bundle files
 or directories following the bundle structure. If multiple bundles are provided, their
@@ -117,7 +118,8 @@ The -e flag tells the 'build' command which documents will be queried by the sof
 asking for policy decisions, so that it can focus optimization efforts and ensure
 that document is not eliminated by the optimizer.
 
-## Signing
+Signing
+-------
 
 The 'build' command can be used to verify the signature of a signed bundle and
 also to generate a signature for the output bundle the command creates.
@@ -126,8 +128,8 @@ If the directory path(s) provided to the 'build' command contain a ".signatures.
 it will attempt to verify the signatures included in that file. The bundle files
 or directory path(s) to verify must be specified using --bundle.
 
-For more information on the bundle verification process see
-https://www.openpolicyagent.org/docs/latest/management/#signature-verification.
+For more information on the bundle signing and verification, see
+https://www.openpolicyagent.org/docs/latest/management-bundles/#signing.
 
 Example:
 
@@ -161,9 +163,10 @@ To include additional claims in the payload use the --claims-file flag to provid
 containing optional claims.
 
 For more information on the format of the ".signatures.json" file
-see https://www.openpolicyagent.org/docs/latest/management/#signature-format.
+see https://www.openpolicyagent.org/docs/latest/management-bundles/#signature-format.
 
-## Capabilities
+Capabilities
+------------
 
 The 'build' command can validate policies against a configurable set of OPA capabilities.
 The capabilities define the built-in functions and other language features that policies

cmd/eval.go

@@ -195,7 +195,7 @@ package path contained inside the file. Only data files named data.json or
 data.yaml will be loaded. In the example above the manifest.yaml would be
 ignored.
 
-See https://www.openpolicyagent.org/docs/latest/bundles/ for more details
+See https://www.openpolicyagent.org/docs/latest/management-bundles/ for more details
 on bundle directory structures.
 
 The --data flag can be used to recursively load ALL *.rego, *.json, and

cmd/flags.go

@@ -91,7 +91,7 @@ func addSigningAlgFlag(fs *pflag.FlagSet, alg *string, value string) {
 }
 
 func addClaimsFileFlag(fs *pflag.FlagSet, file *string) {
-	fs.StringVarP(file, "claims-file", "", "", "set path of JSON file containing optional claims (see: https://openpolicyagent.org/docs/latest/management/#signature-format)")
+	fs.StringVarP(file, "claims-file", "", "", "set path of JSON file containing optional claims (see: https://www.openpolicyagent.org/docs/latest/management-bundles/#signature-format)")
 }
 
 func addSigningKeyFlag(fs *pflag.FlagSet, key *string) {
@@ -99,7 +99,7 @@ func addSigningKeyFlag(fs *pflag.FlagSet, key *string) {
 }
 
 func addSigningPluginFlag(fs *pflag.FlagSet, plugin *string) {
-	fs.StringVarP(plugin, "signing-plugin", "", "", "name of the plugin to use for signing/verification (see https://openpolicyagent.org/docs/latest/management/#signature-plugin")
+	fs.StringVarP(plugin, "signing-plugin", "", "", "name of the plugin to use for signing/verification (see https://www.openpolicyagent.org/docs/latest/management-bundles/#signature-plugin")
 }
 
 func addVerificationKeyFlag(fs *pflag.FlagSet, key *string) {

cmd/run.go

@@ -123,7 +123,7 @@ File paths can be specified as URLs to resolve ambiguity in paths containing col
 The 'run' command can also verify the signature of a signed bundle.
 A signed bundle is a normal OPA bundle that includes a file
 named ".signatures.json". For more information on signed bundles
-see https://www.openpolicyagent.org/docs/latest/management/#signing.
+see https://www.openpolicyagent.org/docs/latest/management-bundles/#signing.
 
 The key to verify the signature of signed bundle can be provided
 using the --verification-key flag. For example, for RSA family of algorithms,
@@ -148,7 +148,7 @@ The 'run' command will read the bundle "bundle.tar.gz", check the
 ".signatures.json" file and perform verification using the provided key.
 An error will be generated if "bundle.tar.gz" does not contain a ".signatures.json" file.
 For more information on the bundle verification process see
-https://www.openpolicyagent.org/docs/latest/management/#signature-verification.
+https://www.openpolicyagent.org/docs/latest/management-bundles/#signature-verification.
 
 The 'run' command can ONLY be used with the --bundle flag to verify signatures
 for existing bundle files or directories following the bundle structure.
@@ -191,7 +191,7 @@ To skip bundle verification, use the --skip-verify flag.
 	addConfigOverrides(runCommand.Flags(), &cmdParams.rt.ConfigOverrides)
 	addConfigOverrideFiles(runCommand.Flags(), &cmdParams.rt.ConfigOverrideFiles)
 	addBundleModeFlag(runCommand.Flags(), &cmdParams.rt.BundleMode, false)
-	runCommand.Flags().BoolVar(&cmdParams.skipVersionCheck, "skip-version-check", false, "disables anonymous version reporting (see: https://openpolicyagent.org/docs/latest/privacy)")
+	runCommand.Flags().BoolVar(&cmdParams.skipVersionCheck, "skip-version-check", false, "disables anonymous version reporting (see: https://www.openpolicyagent.org/docs/latest/privacy)")
 	addIgnoreFlag(runCommand.Flags(), &cmdParams.ignore)
 
 	// bundle verification config

cmd/sign.go

@@ -128,7 +128,7 @@ To include additional claims in the payload use the --claims-file flag to provid
 a JSON file containing optional claims.
 
 For more information on the format of the ".signatures.json" file see
-https://www.openpolicyagent.org/docs/latest/management/#signature-format.
+https://www.openpolicyagent.org/docs/latest/management-bundles/#signature-format.
 `,
 		PreRunE: func(Cmd *cobra.Command, args []string) error {
 			return validateSignParams(args, cmdParams)

docs/Makefile

@@ -13,8 +13,12 @@ clean:
 	rm -rf $(CURDIR)/website/public
 	rm -rf $(CURDIR)/website/resources
 
+.PHONY: generate-cli-docs
+generate-cli-docs:
+	$(CURDIR)/../build/gen-cli-docs.sh "$(CURDIR)/content"
+
 .PHONY: generate
-generate:
+generate: generate-cli-docs
 	$(CURDIR)/website/scripts/load-docs.sh
 
 # The website has some npm dependencies saved in ./website/node_modules