diff --git a/.github/workflows/post-merge.yaml b/.github/workflows/post-merge.yaml index 89cb3ea92b..30c710d23a 100644 --- a/.github/workflows/post-merge.yaml +++ b/.github/workflows/post-merge.yaml @@ -45,6 +45,45 @@ jobs: echo "No generated changes to push!" fi + generate-cli-docs: + name: Sync Generated CLI Docs + runs-on: ubuntu-18.04 + steps: + - name: Check out code + uses: actions/checkout@v2 + with: + token: ${{ secrets.GH_PUSH_TOKEN }} + + - name: Generate + run: make docs-generate-cli-docs + + - name: Commit & Push + shell: bash + run: | + # Commit any changes and push as needed. + + # See https://github.com/actions/checkout#push-a-commit-using-the-built-in-token + AUTHOR=cli-docs-updater + git config user.name ${AUTHOR} + git config user.email ${AUTHOR}@github.com + + # Prevent looping if the build was non-deterministic.. + CAN_PUSH=1 + if [[ "$(git log -1 --pretty=format:'%an')" == "${AUTHOR}" ]]; then + CAN_PUSH=0 + fi + + if ./build/commit-cli-docs.sh; then + if [[ "${CAN_PUSH}" == "1" ]]; then + git push + else + echo "Previous commit was auto-generated -- Aborting!" + exit 1 + fi + else + echo "No generated changes to push!" + fi + code-coverage: name: Update Go Test Coverage runs-on: ubuntu-18.04 diff --git a/build/commit-cli-docs.sh b/build/commit-cli-docs.sh new file mode 100755 index 0000000000..a1c00e8f3d --- /dev/null +++ b/build/commit-cli-docs.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +OPA_DIR=$(dirname "${BASH_SOURCE}")/.. + +cd "${OPA_DIR}" + +git add docs/content/cli.md + +if [[ -z "$(git diff --name-only --cached)" ]]; then + echo "No CLI doc changes to commit" + exit 1 +fi + +git commit -m "docs: Update generated CLI docs" + +echo "" +echo "Committed changes for files:" +git diff-tree --no-commit-id --name-only -r HEAD +echo "" diff --git a/build/gen-cli-docs.sh b/build/gen-cli-docs.sh new file mode 100755 index 0000000000..1412b4f77c --- /dev/null +++ b/build/gen-cli-docs.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +SCRIPT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) + +GOOS="" GOARCH="" go run "$SCRIPT_DIR"/generate-cli-docs/generate.go "$@" diff --git a/build/generate-cli-docs/generate.go b/build/generate-cli-docs/generate.go new file mode 100644 index 0000000000..28a72fd75a --- /dev/null +++ b/build/generate-cli-docs/generate.go @@ -0,0 +1,117 @@ +package main + +import ( + "bufio" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/spf13/cobra/doc" + + "github.com/open-policy-agent/opa/cmd" +) + +const fileHeader = `--- +title: CLI +kind: documentation +weight: 90 +restrictedtoc: true +--- + +The OPA executable provides the following commands. + +` + +func main() { + if len(os.Args) != 2 { + log.Fatal("Required argument: cli docs output directory") + } + out := os.Args[1] + + command := cmd.RootCommand + command.Use = "opa [command]" + command.DisableAutoGenTag = true + + dir, err := ioutil.TempDir("", "opa") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(dir) + + err = doc.GenMarkdownTree(command, dir) + if err != nil { + log.Fatal(err) + } + + files, err := ioutil.ReadDir(dir) + if err != nil { + log.Fatal(err) + } + + builder := strings.Builder{} + + last := len(files) - 1 + for i, file := range files { + // Skip the first "opa" document as it's rather pointless to include (only shows the --help flag) + if i == 0 { + continue + } + path := filepath.Join(dir, file.Name()) + document, err := fixupSection(path) + if err != nil { + log.Fatal(err) + } + builder.WriteString(document) + if i != last { + builder.WriteString("____\n\n") + } + } + + heading := regexp.MustCompile(`^[\\-]+$`) + var document []string + removed := 0 + + // The document may contain "----" for headings, which will be converted to h1 + // elements in markdown. This is undesirable, so let's remove them and prepend + // the line before that with ### to instead create a h3 + for line, str := range strings.Split(builder.String(), "\n") { + if heading.Match([]byte(str)) { + document[line-1-removed] = fmt.Sprintf("### %s\n", document[line-1-removed]) + removed++ + continue + } + document = append(document, fmt.Sprintf("%s\n", str)) + } + + withHeader := fmt.Sprintf("%s%s", fileHeader, strings.Join(document, "")) + err = ioutil.WriteFile(filepath.Join(out, "cli.md"), []byte(withHeader), 0755) + if err != nil { + log.Fatal(err) + } +} + +func fixupSection(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + builder := strings.Builder{} + + for scanner.Scan() { + // Remove "See also" section + if strings.Contains(scanner.Text(), "### SEE ALSO") { + break + } + builder.WriteString(scanner.Text()) + builder.WriteString("\n") + } + + return builder.String(), scanner.Err() +} diff --git a/cmd/build.go b/cmd/build.go index b85a32f7f8..ea6a591d85 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -85,9 +85,10 @@ Inside another terminal in the same directory, serve the bundle via HTTP: $ python3 -m http.server --bind localhost 8080 -For more information on bundles see https://www.openpolicyagent.org/docs/latest/management. +For more information on bundles see https://www.openpolicyagent.org/docs/latest/management-bundles/. -## Common Flags +Common Flags +------------ When -b is specified the 'build' command assumes paths refer to existing bundle files or directories following the bundle structure. If multiple bundles are provided, their @@ -117,7 +118,8 @@ The -e flag tells the 'build' command which documents will be queried by the sof asking for policy decisions, so that it can focus optimization efforts and ensure that document is not eliminated by the optimizer. -## Signing +Signing +------- The 'build' command can be used to verify the signature of a signed bundle and also to generate a signature for the output bundle the command creates. @@ -126,8 +128,8 @@ If the directory path(s) provided to the 'build' command contain a ".signatures. it will attempt to verify the signatures included in that file. The bundle files or directory path(s) to verify must be specified using --bundle. -For more information on the bundle verification process see -https://www.openpolicyagent.org/docs/latest/management/#signature-verification. +For more information on the bundle signing and verification, see +https://www.openpolicyagent.org/docs/latest/management-bundles/#signing. Example: @@ -161,9 +163,10 @@ To include additional claims in the payload use the --claims-file flag to provid containing optional claims. For more information on the format of the ".signatures.json" file -see https://www.openpolicyagent.org/docs/latest/management/#signature-format. +see https://www.openpolicyagent.org/docs/latest/management-bundles/#signature-format. -## Capabilities +Capabilities +------------ The 'build' command can validate policies against a configurable set of OPA capabilities. The capabilities define the built-in functions and other language features that policies diff --git a/cmd/eval.go b/cmd/eval.go index f77a94ad2a..65f1d7cc01 100644 --- a/cmd/eval.go +++ b/cmd/eval.go @@ -195,7 +195,7 @@ package path contained inside the file. Only data files named data.json or data.yaml will be loaded. In the example above the manifest.yaml would be ignored. -See https://www.openpolicyagent.org/docs/latest/bundles/ for more details +See https://www.openpolicyagent.org/docs/latest/management-bundles/ for more details on bundle directory structures. The --data flag can be used to recursively load ALL *.rego, *.json, and diff --git a/cmd/flags.go b/cmd/flags.go index db24e1bee6..752c7d84f9 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -91,7 +91,7 @@ func addSigningAlgFlag(fs *pflag.FlagSet, alg *string, value string) { } func addClaimsFileFlag(fs *pflag.FlagSet, file *string) { - fs.StringVarP(file, "claims-file", "", "", "set path of JSON file containing optional claims (see: https://openpolicyagent.org/docs/latest/management/#signature-format)") + fs.StringVarP(file, "claims-file", "", "", "set path of JSON file containing optional claims (see: https://www.openpolicyagent.org/docs/latest/management-bundles/#signature-format)") } func addSigningKeyFlag(fs *pflag.FlagSet, key *string) { @@ -99,7 +99,7 @@ func addSigningKeyFlag(fs *pflag.FlagSet, key *string) { } func addSigningPluginFlag(fs *pflag.FlagSet, plugin *string) { - fs.StringVarP(plugin, "signing-plugin", "", "", "name of the plugin to use for signing/verification (see https://openpolicyagent.org/docs/latest/management/#signature-plugin") + fs.StringVarP(plugin, "signing-plugin", "", "", "name of the plugin to use for signing/verification (see https://www.openpolicyagent.org/docs/latest/management-bundles/#signature-plugin") } func addVerificationKeyFlag(fs *pflag.FlagSet, key *string) { diff --git a/cmd/run.go b/cmd/run.go index 03192efe75..7c2fd7ef30 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -123,7 +123,7 @@ File paths can be specified as URLs to resolve ambiguity in paths containing col The 'run' command can also verify the signature of a signed bundle. A signed bundle is a normal OPA bundle that includes a file named ".signatures.json". For more information on signed bundles -see https://www.openpolicyagent.org/docs/latest/management/#signing. +see https://www.openpolicyagent.org/docs/latest/management-bundles/#signing. The key to verify the signature of signed bundle can be provided using the --verification-key flag. For example, for RSA family of algorithms, @@ -148,7 +148,7 @@ The 'run' command will read the bundle "bundle.tar.gz", check the ".signatures.json" file and perform verification using the provided key. An error will be generated if "bundle.tar.gz" does not contain a ".signatures.json" file. For more information on the bundle verification process see -https://www.openpolicyagent.org/docs/latest/management/#signature-verification. +https://www.openpolicyagent.org/docs/latest/management-bundles/#signature-verification. The 'run' command can ONLY be used with the --bundle flag to verify signatures for existing bundle files or directories following the bundle structure. @@ -191,7 +191,7 @@ To skip bundle verification, use the --skip-verify flag. addConfigOverrides(runCommand.Flags(), &cmdParams.rt.ConfigOverrides) addConfigOverrideFiles(runCommand.Flags(), &cmdParams.rt.ConfigOverrideFiles) addBundleModeFlag(runCommand.Flags(), &cmdParams.rt.BundleMode, false) - runCommand.Flags().BoolVar(&cmdParams.skipVersionCheck, "skip-version-check", false, "disables anonymous version reporting (see: https://openpolicyagent.org/docs/latest/privacy)") + runCommand.Flags().BoolVar(&cmdParams.skipVersionCheck, "skip-version-check", false, "disables anonymous version reporting (see: https://www.openpolicyagent.org/docs/latest/privacy)") addIgnoreFlag(runCommand.Flags(), &cmdParams.ignore) // bundle verification config diff --git a/cmd/sign.go b/cmd/sign.go index f1ecc7309a..8b9f3a1ae3 100644 --- a/cmd/sign.go +++ b/cmd/sign.go @@ -128,7 +128,7 @@ To include additional claims in the payload use the --claims-file flag to provid a JSON file containing optional claims. For more information on the format of the ".signatures.json" file see -https://www.openpolicyagent.org/docs/latest/management/#signature-format. +https://www.openpolicyagent.org/docs/latest/management-bundles/#signature-format. `, PreRunE: func(Cmd *cobra.Command, args []string) error { return validateSignParams(args, cmdParams) diff --git a/docs/Makefile b/docs/Makefile index b530ebf2bf..cc392c01cc 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -13,8 +13,12 @@ clean: rm -rf $(CURDIR)/website/public rm -rf $(CURDIR)/website/resources +.PHONY: generate-cli-docs +generate-cli-docs: + $(CURDIR)/../build/gen-cli-docs.sh "$(CURDIR)/content" + .PHONY: generate -generate: +generate: generate-cli-docs $(CURDIR)/website/scripts/load-docs.sh # The website has some npm dependencies saved in ./website/node_modules