From ed76301eb6d9b56dcc0a12a9db0ba5e0021b716e Mon Sep 17 00:00:00 2001 From: Philip Conrad Date: Fri, 18 Nov 2022 15:11:47 -0500 Subject: [PATCH] builtins: Add AWS Sig v4 signing builtin. (#5376) This commit adds initial support for AWS's SigV4 request signing system, which will allow OPA's existing `http.send` builtin to be used to more conveniently query cloud resources. It automates away most of the pain around signing the request headers and body, and is designed to compose with `http.send` directly. Internally, this also refactors AWS SigV4 request signing, so that the signing logic is shared between the builtin and the REST plugin for AWS. Fixes: #3749 Signed-off-by: Philip Conrad --- ast/builtins.go | 22 ++ build/policy/helpers.rego | 4 +- builtin_metadata.json | 29 +++ capabilities.json | 44 ++++ docs/content/policy-reference.md | 80 ++++++++ internal/providers/aws.go | 171 ++++++++++++++++ plugins/rest/aws.go | 146 ++----------- plugins/rest/aws_test.go | 16 +- plugins/rest/rest_auth.go | 5 +- .../providers-aws/aws-sign_req-errors.yaml | 148 ++++++++++++++ .../testdata/providers-aws/aws-sign_req.yaml | 132 ++++++++++++ topdown/providers.go | 191 ++++++++++++++++++ 12 files changed, 843 insertions(+), 145 deletions(-) create mode 100644 internal/providers/aws.go create mode 100644 test/cases/testdata/providers-aws/aws-sign_req-errors.yaml create mode 100644 test/cases/testdata/providers-aws/aws-sign_req.yaml create mode 100644 topdown/providers.go diff --git a/ast/builtins.go b/ast/builtins.go index d6a017ef98..e112387b17 100644 --- a/ast/builtins.go +++ b/ast/builtins.go @@ -243,6 +243,9 @@ var DefaultBuiltins = [...]*Builtin{ GraphQLIsValid, GraphQLSchemaIsValid, + // Cloud Provider Helpers + ProvidersAWSSignReqObj, + // Rego RegoParseModule, RegoMetadataChain, @@ -2628,6 +2631,25 @@ var GraphQLSchemaIsValid = &Builtin{ ), } +/** + * Cloud Provider Helper Functions + */ +var providersAWSCat = category("providers.aws") + +var ProvidersAWSSignReqObj = &Builtin{ + Name: "providers.aws.sign_req", + Description: "Signs an HTTP request object for Amazon Web Services. Currently implements [AWS Signature Version 4 request signing](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) by the `Authorization` header method.", + Decl: types.NewFunction( + types.Args( + types.Named("request", types.NewObject(nil, types.NewDynamicProperty(types.S, types.A))), + types.Named("aws_config", types.NewObject(nil, types.NewDynamicProperty(types.S, types.A))), + types.Named("time_ns", types.N), + ), + types.Named("signed_request", types.NewObject(nil, types.NewDynamicProperty(types.A, types.A))), + ), + Categories: providersAWSCat, +} + /** * Rego */ diff --git a/build/policy/helpers.rego b/build/policy/helpers.rego index 104b76ffc9..f96460bbc6 100644 --- a/build/policy/helpers.rego +++ b/build/policy/helpers.rego @@ -6,9 +6,7 @@ last_indexof(string, search) = i { all := [i | chars := split(string, ""); chars[i] == search] count(all) > 0 i := all[count(all) - 1] -} else = -1 { - true -} +} else = -1 basename(filename) = substring(filename, last_indexof(filename, "/") + 1, count(filename) - 1) diff --git a/builtin_metadata.json b/builtin_metadata.json index deae467efc..ae3ee99cc0 100644 --- a/builtin_metadata.json +++ b/builtin_metadata.json @@ -128,6 +128,9 @@ "opa": [ "opa.runtime" ], + "providers.aws": [ + "providers.aws.sign_req" + ], "regex": [ "regex.find_all_string_submatch_n", "regex.find_n", @@ -9978,6 +9981,32 @@ }, "wasm": true }, + "providers.aws.sign_req": { + "args": [ + { + "name": "request", + "type": "object[string: any]" + }, + { + "name": "aws_config", + "type": "object[string: any]" + }, + { + "name": "time_ns", + "type": "number" + } + ], + "available": [ + "edge" + ], + "description": "Signs an HTTP request object for Amazon Web Services. Currently implements [AWS Signature Version 4 request signing](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) by the `Authorization` header method.", + "introduced": "edge", + "result": { + "name": "signed_request", + "type": "object[any: any]" + }, + "wasm": false + }, "rand.intn": { "args": [ { diff --git a/capabilities.json b/capabilities.json index 7034879cbe..69d7284a11 100644 --- a/capabilities.json +++ b/capabilities.json @@ -3074,6 +3074,50 @@ "type": "function" } }, + { + "name": "providers.aws.sign_req", + "decl": { + "args": [ + { + "dynamic": { + "key": { + "type": "string" + }, + "value": { + "type": "any" + } + }, + "type": "object" + }, + { + "dynamic": { + "key": { + "type": "string" + }, + "value": { + "type": "any" + } + }, + "type": "object" + }, + { + "type": "number" + } + ], + "result": { + "dynamic": { + "key": { + "type": "any" + }, + "value": { + "type": "any" + } + }, + "type": "object" + }, + "type": "function" + } + }, { "name": "rand.intn", "decl": { diff --git a/docs/content/policy-reference.md b/docs/content/policy-reference.md index 8e40d87e3f..d23454664f 100644 --- a/docs/content/policy-reference.md +++ b/docs/content/policy-reference.md @@ -899,6 +899,86 @@ The table below shows examples of calling `http.send`: | Environment variables containing TLS material | ``http.send({"method": "get", "url": "https://127.0.0.1:65360", "tls_ca_cert_env_variable": "CLIENT_CA_ENV", "tls_client_cert_env_variable": "CLIENT_CERT_ENV", "tls_client_key_env_variable": "CLIENT_KEY_ENV"})`` | | Unix Socket URL Format| ``http.send({"method": "get", "url": "unix://localhost/?socket=%F2path%F2file.socket"})`` | +{{< builtin-table cat=providers.aws title=AWS >}} + +The AWS Request Signing builtin in OPA implements the header-based auth, +single-chunk method described in the [AWS SigV4 docs](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html). +It will always sign the payload when present, and will sign most user-provided +headers for the request, to ensure their integrity. + +{{< info >}} +Note that the `authorization`, `user-agent`, and `x-amzn-trace-id` headers, +are commonly modified by proxy systems, and as such are ignored by OPA +for signing. +{{< /info >}} + +The `request` object parameter may contain any and all of the same fields as for `http.send`. +The following fields will have effects on the output `Authorization` header signature: + +| Field | Required | Type | Description | +| --- | --- | --- | --- | +| `url` | yes | `string` | HTTP URL to specify in the request. Used in the signature. | +| `method` | yes | `string` | HTTP method to specify in request. Used in the signature. | +| `body` | no | `any` | HTTP message body. The JSON serialized version of this value will be used for the payload portion of the signature if present. | +| `raw_body` | no | `string` | HTTP message body. This will be used for the payload portion of the signature if present. | +| `headers` | no | `object` | HTTP headers to include in the request. These will be added to the list of headers to sign. | + +The `aws_config` object parameter may contain the following fields: + +| Field | Required | Type | Description | +| --- | --- | --- | --- | +| `aws_access_key` | yes | `string` | AWS access key. | +| `aws_secret_access_key` | yes | `string` | AWS secret access key. Used in generating the signing key for the request. | +| `aws_service` | yes | `string` | AWS service the request will be valid for. (e.g. `"s3"`) | +| `aws_region` | yes | `string` | AWS region for the request. (e.g. `"us-east-1"`) | +| `aws_session_token` | no | `string` | AWS security token. Used for the `x-amz-security-token` request header. | + +#### AWS Request Signing Examples + +##### Basic Request Signing Example +The example below shows using hard-coded AWS credentials for signing the request +object for `http.send`. + +{{< info >}} +For deployments, a common way to provide AWS credentials is via environment +variables, usually by using the results of `opa.runtime().env`. +{{< /info >}} + +```live:providers/aws/sign_req_basic:module +req := {"method": "get", "url": "https://examplebucket.s3.amazonaws.com/data"} +aws_config := { + "aws_access_key": "MYAWSACCESSKEYGOESHERE", + "aws_secret_access_key": "MYAWSSECRETACCESSKEYGOESHERE", + "aws_service": "s3", + "aws_region": "us-east-1", +} + +example_verify_resource { + resp := http.send(providers.aws.sign_req(req, aws_config, time.now_ns())) + # process response from AWS ... +} +``` + +##### Pre-Signed Request Example +The [AWS S3 request signing API](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html) +supports pre-signing requests, so that they will only be valid at a future date. +To do this in OPA, simply adjust the time parameter: + +```live:providers/aws/sign_req_presign:module +env := opa.runtime().env +req := {"method": "get", "url": "https://examplebucket.s3.amazonaws.com/data"} +aws_config := { + "aws_access_key": env["AWS_ACCESS_KEY"], + "aws_secret_access_key": env["AWS_SECRET_ACCESS_KEY"], + "aws_service": "s3", + "aws_region": env["AWS_REGION"], +} +# Request will become valid 2 days from now. +signing_time := time.add_date(time.now_ns(), 0, 0, 2) + +pre_signed_req := providers.aws.sign_req(req, aws_config, signing_time)) +``` + {{< builtin-table net >}} #### Notes on Name Resolution (`net.lookup_ip_addr`) diff --git a/internal/providers/aws.go b/internal/providers/aws.go new file mode 100644 index 0000000000..0d40a66879 --- /dev/null +++ b/internal/providers/aws.go @@ -0,0 +1,171 @@ +// Copyright 2022 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package providers + +import ( + "crypto/hmac" + "crypto/sha256" + "fmt" + "net/url" + "sort" + "strings" + "time" + + "github.com/open-policy-agent/opa/ast" +) + +func stringFromTerm(t *ast.Term) string { + if v, ok := t.Value.(ast.String); ok { + return string(v) + } + return "" +} + +// Headers that may be mutated before reaching an aws service (eg by a proxy) should be added here to omit them from +// the sigv4 canonical request +// ref. https://github.com/aws/aws-sdk-go/blob/master/aws/signer/v4/v4.go#L92 +var awsSigv4IgnoredHeaders = map[string]struct{}{ + "authorization": {}, + "user-agent": {}, + "x-amzn-trace-id": {}, +} + +type AWSCredentials struct { + AccessKey string + SecretKey string + RegionName string + SessionToken string +} + +func AWSCredentialsFromObject(v ast.Object) AWSCredentials { + var creds AWSCredentials + awsAccessKey := v.Get(ast.StringTerm("aws_access_key")) + awsSecretKey := v.Get(ast.StringTerm("aws_secret_access_key")) + awsRegion := v.Get(ast.StringTerm("aws_region")) + awsSessionToken := v.Get(ast.StringTerm("aws_session_token")) + + creds.AccessKey = stringFromTerm(awsAccessKey) + creds.SecretKey = stringFromTerm(awsSecretKey) + creds.RegionName = stringFromTerm(awsRegion) + if awsSessionToken != nil { + creds.SessionToken = stringFromTerm(awsSessionToken) + } + return creds +} + +func sha256MAC(message string, key []byte) []byte { + mac := hmac.New(sha256.New, key) + mac.Write([]byte(message)) + return mac.Sum(nil) +} + +func sortKeys(strMap map[string][]string) []string { + keys := make([]string, len(strMap)) + + i := 0 + for k := range strMap { + keys[i] = k + i++ + } + sort.Strings(keys) + + return keys +} + +// AWSSignV4 modifies a map[string][]string of headers to include an AWS V4 signature based on the config/credentials provided. +func AWSSignV4(headers map[string][]string, method string, theURL *url.URL, body []byte, service string, awsCreds AWSCredentials, theTime time.Time) map[string][]string { + // General ref. https://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html + // S3 ref. https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html + // APIGateway ref. https://docs.aws.amazon.com/apigateway/api-reference/signing-requests/ + bodyHexHash := fmt.Sprintf("%x", sha256.Sum256(body)) + + now := theTime.UTC() + + // V4 signing has specific ideas of how it wants to see dates/times encoded + dateNow := now.Format("20060102") + iso8601Now := now.Format("20060102T150405Z") + + awsHeaders := map[string]string{ + "host": theURL.Host, + "x-amz-date": iso8601Now, + } + + // s3 and glacier require the extra x-amz-content-sha256 header. other services do not. + if service == "s3" || service == "glacier" { + awsHeaders["x-amz-content-sha256"] = bodyHexHash + } + + // the security token header is necessary for ephemeral credentials, e.g. from + // the EC2 metadata service + if awsCreds.SessionToken != "" { + awsHeaders["x-amz-security-token"] = awsCreds.SessionToken + } + + headersToSign := map[string][]string{} + // sign all of the aws headers that aren't on the ignore list. + for k, v := range headers { + lowercaseHeader := strings.ToLower(k) + if _, ok := awsSigv4IgnoredHeaders[lowercaseHeader]; !ok { + headersToSign[lowercaseHeader] = v + } + } + for k, v := range awsHeaders { + headersToSign[k] = []string{v} + } + + // the "canonical request" is the normalized version of the AWS service access + // that we're attempting to perform; in this case, a GET from an S3 bucket + canonicalReq := method + "\n" // HTTP method + canonicalReq += theURL.EscapedPath() + "\n" // URI-escaped path + canonicalReq += theURL.RawQuery + "\n" // RAW Query String + + // include the values for the signed headers + orderedKeys := sortKeys(headersToSign) + for _, k := range orderedKeys { + canonicalReq += k + ":" + strings.Join(headersToSign[k], ",") + "\n" + } + canonicalReq += "\n" // linefeed to terminate headers + + // include the list of the signed headers + headerList := strings.Join(orderedKeys, ";") + canonicalReq += headerList + "\n" + canonicalReq += bodyHexHash + + // the "string to sign" is a time-bounded, scoped request token which + // is linked to the "canonical request" by inclusion of its SHA-256 hash + strToSign := "AWS4-HMAC-SHA256\n" // V4 signing with SHA-256 HMAC + strToSign += iso8601Now + "\n" // ISO 8601 time + strToSign += dateNow + "/" + awsCreds.RegionName + "/" + service + "/aws4_request\n" // scoping for signature + strToSign += fmt.Sprintf("%x", sha256.Sum256([]byte(canonicalReq))) // SHA-256 of canonical request + + // the "signing key" is generated by repeated HMAC-SHA256 based on the same + // scoping that's included in the "string to sign"; but including the secret key + // to allow AWS to validate it + signingKey := sha256MAC(dateNow, []byte("AWS4"+awsCreds.SecretKey)) + signingKey = sha256MAC(awsCreds.RegionName, signingKey) + signingKey = sha256MAC(service, signingKey) + signingKey = sha256MAC("aws4_request", signingKey) + + // the "signature" is finally the "string to sign" signed by the "signing key" + signature := sha256MAC(strToSign, signingKey) + + // required format of Authorization header; n.b. the access key corresponding to + // the secret key is included here + authHdr := "AWS4-HMAC-SHA256 Credential=" + awsCreds.AccessKey + "/" + dateNow + authHdr += "/" + awsCreds.RegionName + "/" + service + "/aws4_request," + authHdr += "SignedHeaders=" + headerList + "," + authHdr += "Signature=" + fmt.Sprintf("%x", signature) + + // add the computed Authorization + out := make(map[string][]string, len(awsHeaders)+1) + out["Authorization"] = []string{authHdr} + + // populate the other signed headers into the request + for k, v := range awsHeaders { + out[k] = []string{v} + } + + return out +} diff --git a/plugins/rest/aws.go b/plugins/rest/aws.go index 642352a194..059fbd885c 100644 --- a/plugins/rest/aws.go +++ b/plugins/rest/aws.go @@ -6,8 +6,6 @@ package rest import ( "bytes" - "crypto/hmac" - "crypto/sha256" "encoding/json" "encoding/xml" "errors" @@ -17,12 +15,12 @@ import ( "net/url" "os" "path/filepath" - "sort" "strings" "time" "github.com/go-ini/ini" + "github.com/open-policy-agent/opa/internal/providers" "github.com/open-policy-agent/opa/logging" ) @@ -58,26 +56,9 @@ const ( securityTokenGlobalSetting = "aws_session_token" ) -// Headers that may be mutated before reaching an aws service (eg by a proxy) should be added here to omit them from -// the sigv4 canonical request -// ref. https://github.com/aws/aws-sdk-go/blob/master/aws/signer/v4/v4.go#L92 -var awsSigv4IgnoredHeaders = map[string]bool{ - "authorization": true, - "user-agent": true, - "x-amzn-trace-id": true, -} - -// awsCredentials represents the credentials obtained from an AWS credential provider -type awsCredentials struct { - AccessKey string - SecretKey string - RegionName string - SessionToken string -} - // awsCredentialService represents the interface for AWS credential providers type awsCredentialService interface { - credentials() (awsCredentials, error) + credentials() (providers.AWSCredentials, error) } // awsEnvironmentCredentialService represents an static environment-variable credential provider for AWS @@ -85,8 +66,8 @@ type awsEnvironmentCredentialService struct { logger logging.Logger } -func (cs *awsEnvironmentCredentialService) credentials() (awsCredentials, error) { - var creds awsCredentials +func (cs *awsEnvironmentCredentialService) credentials() (providers.AWSCredentials, error) { + var creds providers.AWSCredentials creds.AccessKey = os.Getenv(accessKeyEnvVar) if creds.AccessKey == "" { return creds, errors.New("no " + accessKeyEnvVar + " set in environment") @@ -133,8 +114,8 @@ type awsProfileCredentialService struct { logger logging.Logger } -func (cs *awsProfileCredentialService) credentials() (awsCredentials, error) { - var creds awsCredentials +func (cs *awsProfileCredentialService) credentials() (providers.AWSCredentials, error) { + var creds providers.AWSCredentials filename, err := cs.path() if err != nil { @@ -210,7 +191,7 @@ func (cs *awsProfileCredentialService) profile() string { type awsMetadataCredentialService struct { RoleName string `json:"iam_role,omitempty"` RegionName string `json:"aws_region"` - creds awsCredentials + creds providers.AWSCredentials expiration time.Time credServicePath string tokenPath string @@ -327,7 +308,7 @@ func (cs *awsMetadataCredentialService) refreshFromService() error { return nil } -func (cs *awsMetadataCredentialService) credentials() (awsCredentials, error) { +func (cs *awsMetadataCredentialService) credentials() (providers.AWSCredentials, error) { err := cs.refreshFromService() if err != nil { return cs.creds, err @@ -342,7 +323,7 @@ type awsWebIdentityCredentialService struct { RegionName string `json:"aws_region"` SessionName string `json:"session_name"` stsURL string - creds awsCredentials + creds providers.AWSCredentials expiration time.Time logger logging.Logger } @@ -452,7 +433,7 @@ func (cs *awsWebIdentityCredentialService) refreshFromService() error { return nil } -func (cs *awsWebIdentityCredentialService) credentials() (awsCredentials, error) { +func (cs *awsWebIdentityCredentialService) credentials() (providers.AWSCredentials, error) { err := cs.refreshFromService() if err != nil { return cs.creds, err @@ -500,24 +481,6 @@ func doMetaDataRequestWithClient(req *http.Request, client *http.Client, desc st return body, nil } -func sha256MAC(message []byte, key []byte) []byte { - mac := hmac.New(sha256.New, key) - mac.Write(message) - return mac.Sum(nil) -} - -func sortKeys(strMap map[string][]string) []string { - keys := make([]string, len(strMap)) - - i := 0 - for k := range strMap { - keys[i] = k - i++ - } - sort.Strings(keys) - return keys -} - // signV4 modifies an http.Request to include an AWS V4 signature based on a credential provider func signV4(req *http.Request, service string, credService awsCredentialService, theTime time.Time) error { // General ref. https://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html @@ -542,94 +505,13 @@ func signV4(req *http.Request, service string, credService awsCredentialService, return errors.New("error getting AWS credentials: " + err.Error()) } - bodyHexHash := fmt.Sprintf("%x", sha256.Sum256(body)) - now := theTime.UTC() - // V4 signing has specific ideas of how it wants to see dates/times encoded - dateNow := now.Format("20060102") - iso8601Now := now.Format("20060102T150405Z") + signedHeaders := providers.AWSSignV4(req.Header, req.Method, req.URL, body, service, creds, now) - awsHeaders := map[string]string{ - "host": req.URL.Host, - "x-amz-date": iso8601Now, - } - - // s3 and glacier require the extra x-amz-content-sha256 header. other services do not. - if service == "s3" || service == "glacier" { - awsHeaders["x-amz-content-sha256"] = bodyHexHash - } - - // the security token header is necessary for ephemeral credentials, e.g. from - // the EC2 metadata service - if creds.SessionToken != "" { - awsHeaders["x-amz-security-token"] = creds.SessionToken - } - - headersToSign := map[string][]string{} - - // sign all of the aws headers - for k, v := range awsHeaders { - headersToSign[k] = []string{v} - } - - // sign all of the request's headers, except for those in the ignore list - for k, v := range req.Header { - lowerCaseHeader := strings.ToLower(k) - if !awsSigv4IgnoredHeaders[lowerCaseHeader] { - headersToSign[lowerCaseHeader] = v - } - } - - // the "canonical request" is the normalized version of the AWS service access - // that we're attempting to perform; in this case, a GET from an S3 bucket - canonicalReq := req.Method + "\n" // HTTP method - canonicalReq += req.URL.EscapedPath() + "\n" // URI-escaped path - canonicalReq += req.URL.RawQuery + "\n" // RAW Query String - - // include the values for the signed headers - orderedKeys := sortKeys(headersToSign) - for _, k := range orderedKeys { - canonicalReq += k + ":" + strings.Join(headersToSign[k], ",") + "\n" - } - canonicalReq += "\n" // linefeed to terminate headers - - // include the list of the signed headers - headerList := strings.Join(orderedKeys, ";") - canonicalReq += headerList + "\n" - canonicalReq += bodyHexHash - - // the "string to sign" is a time-bounded, scoped request token which - // is linked to the "canonical request" by inclusion of its SHA-256 hash - strToSign := "AWS4-HMAC-SHA256\n" // V4 signing with SHA-256 HMAC - strToSign += iso8601Now + "\n" // ISO 8601 time - strToSign += dateNow + "/" + creds.RegionName + "/" + service + "/aws4_request\n" // scoping for signature - strToSign += fmt.Sprintf("%x", sha256.Sum256([]byte(canonicalReq))) // SHA-256 of canonical request - - // the "signing key" is generated by repeated HMAC-SHA256 based on the same - // scoping that's included in the "string to sign"; but including the secret key - // to allow AWS to validate it - signingKey := sha256MAC([]byte(dateNow), []byte("AWS4"+creds.SecretKey)) - signingKey = sha256MAC([]byte(creds.RegionName), signingKey) - signingKey = sha256MAC([]byte(service), signingKey) - signingKey = sha256MAC([]byte("aws4_request"), signingKey) - - // the "signature" is finally the "string to sign" signed by the "signing key" - signature := sha256MAC([]byte(strToSign), signingKey) - - // required format of Authorization header; n.b. the access key corresponding to - // the secret key is included here - authHdr := "AWS4-HMAC-SHA256 Credential=" + creds.AccessKey + "/" + dateNow - authHdr += "/" + creds.RegionName + "/" + service + "/aws4_request," - authHdr += "SignedHeaders=" + headerList + "," - authHdr += "Signature=" + fmt.Sprintf("%x", signature) - - // add the computed Authorization - req.Header.Set("Authorization", authHdr) - - // populate the other signed headers into the request - for k := range awsHeaders { - req.Header.Add(k, awsHeaders[k]) + req.Header.Set("Authorization", strings.Join(signedHeaders["Authorization"], "")) + for k, v := range signedHeaders { + req.Header.Add(k, strings.Join(v, ",")) } return nil diff --git a/plugins/rest/aws_test.go b/plugins/rest/aws_test.go index 6f03f51dcc..792d16fa88 100644 --- a/plugins/rest/aws_test.go +++ b/plugins/rest/aws_test.go @@ -16,9 +16,9 @@ import ( "testing" "time" - "github.com/open-policy-agent/opa/util/test" - + "github.com/open-policy-agent/opa/internal/providers" "github.com/open-policy-agent/opa/logging" + "github.com/open-policy-agent/opa/util/test" ) // this is usually private; but we need it here @@ -62,7 +62,7 @@ func TestEnvironmentCredentialService(t *testing.T) { t.Setenv("AWS_REGION", "us-east-1") - expectedCreds := awsCredentials{ + expectedCreds := providers.AWSCredentials{ AccessKey: "MYAWSACCESSKEYGOESHERE", SecretKey: "MYAWSSECRETACCESSKEYGOESHERE", RegionName: "us-east-1", @@ -135,7 +135,7 @@ aws_secret_access_key=%v t.Fatal(err) } - expected := awsCredentials{ + expected := providers.AWSCredentials{ AccessKey: fooKey, SecretKey: fooSecret, RegionName: fooRegion, @@ -158,7 +158,7 @@ aws_secret_access_key=%v t.Fatal(err) } - expected = awsCredentials{ + expected = providers.AWSCredentials{ AccessKey: defaultKey, SecretKey: defaultSecret, RegionName: defaultRegion, @@ -201,7 +201,7 @@ aws_session_token=%s t.Fatal(err) } - expected := awsCredentials{ + expected := providers.AWSCredentials{ AccessKey: defaultKey, SecretKey: defaultSecret, RegionName: defaultRegion, @@ -250,7 +250,7 @@ aws_session_token=%s t.Fatal(err) } - expected := awsCredentials{ + expected := providers.AWSCredentials{ AccessKey: defaultKey, SecretKey: defaultSecret, RegionName: defaultRegion, @@ -414,7 +414,7 @@ func TestMetadataCredentialService(t *testing.T) { tokenPath: ts.server.URL + "/latest/api/token", logger: logging.Get(), } - var creds awsCredentials + var creds providers.AWSCredentials creds, err = cs.credentials() if err != nil { // Cannot proceed with test if unable to fetch credentials. diff --git a/plugins/rest/rest_auth.go b/plugins/rest/rest_auth.go index 986de9be69..d33506c80e 100644 --- a/plugins/rest/rest_auth.go +++ b/plugins/rest/rest_auth.go @@ -26,6 +26,7 @@ import ( "github.com/open-policy-agent/opa/internal/jwx/jwa" "github.com/open-policy-agent/opa/internal/jwx/jws" "github.com/open-policy-agent/opa/internal/jwx/jws/sign" + "github.com/open-policy-agent/opa/internal/providers" "github.com/open-policy-agent/opa/internal/uuid" "github.com/open-policy-agent/opa/keys" "github.com/open-policy-agent/opa/logging" @@ -530,7 +531,7 @@ func (acs *awsCredentialServiceChain) addService(service awsCredentialService) { acs.awsCredentialServices = append(acs.awsCredentialServices, service) } -func (acs *awsCredentialServiceChain) credentials() (awsCredentials, error) { +func (acs *awsCredentialServiceChain) credentials() (providers.AWSCredentials, error) { for _, service := range acs.awsCredentialServices { credential, err := service.credentials() if err == nil { @@ -543,7 +544,7 @@ func (acs *awsCredentialServiceChain) credentials() (awsCredentials, error) { reflect.TypeOf(service).String(), err) } - return awsCredentials{}, errors.New("all AWS credential providers failed") + return providers.AWSCredentials{}, errors.New("all AWS credential providers failed") } func (ap *awsSigningAuthPlugin) awsCredentialService() awsCredentialService { diff --git a/test/cases/testdata/providers-aws/aws-sign_req-errors.yaml b/test/cases/testdata/providers-aws/aws-sign_req-errors.yaml new file mode 100644 index 0000000000..2b12e14448 --- /dev/null +++ b/test/cases/testdata/providers-aws/aws-sign_req-errors.yaml @@ -0,0 +1,148 @@ +cases: +# http request object errors: +- data: + modules: + - | + package test + req := {} + aws_config := { + "aws_access_key": "MYAWSACCESSKEYGOESHERE", + "aws_secret_access_key": "MYAWSSECRETACCESSKEYGOESHERE", + "aws_session_token": "MYAWSSECURITYTOKENGOESHERE", + "aws_service": "s3", + "aws_region": "us-east-1", + } + expected := { + "headers": { + "Authorization": "AWS4-HMAC-SHA256 Credential=MYAWSACCESSKEYGOESHERE/20151228/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=0047a7016c81e5c7f29cd522f97e911e04fc472fced0aee7916cbc287ad6c8e3", + "host": "example.com", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20151228T140825Z" + }, + "method": "get", + "url": "http://example.com" + } + p { + providers.aws.sign_req(req, aws_config, 1451311705000000000) == expected + } + note: providers-aws-sign_req/failure-simple-missing http request keys + query: data.test.p = x + want_error_code: eval_type_error + want_error: "providers.aws.sign_req: operand 1 missing required request parameters(s): {\"method\", \"url\"}" + strict_error: true +- data: + modules: + - | + package test + req := {"method": set(), "url": set()} + aws_config := { + "aws_access_key": "MYAWSACCESSKEYGOESHERE", + "aws_secret_access_key": "MYAWSSECRETACCESSKEYGOESHERE", + "aws_session_token": "MYAWSSECURITYTOKENGOESHERE", + "aws_service": "s3", + "aws_region": "us-east-1", + } + expected := { + "headers": { + "Authorization": "AWS4-HMAC-SHA256 Credential=MYAWSACCESSKEYGOESHERE/20151228/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=0047a7016c81e5c7f29cd522f97e911e04fc472fced0aee7916cbc287ad6c8e3", + "host": "example.com", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20151228T140825Z" + }, + "method": "get", + "url": "http://example.com" + } + p { + providers.aws.sign_req(req, aws_config, 1451311705000000000) == expected + } + note: providers-aws-sign_req/failure-simple-invalid type http request keys + query: data.test.p = x + want_error_code: eval_type_error + want_error: "providers.aws.sign_req: operand 1 invalid values for required request parameters(s): {\"method\", \"url\"}" + strict_error: true +# aws config object errors: +- data: + modules: + - | + package test + req := {"method": "get", "url": "http://example.com"} + aws_config := {"example": "example"} + expected := { + "headers": { + "Authorization": "AWS4-HMAC-SHA256 Credential=MYAWSACCESSKEYGOESHERE/20151228/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=0047a7016c81e5c7f29cd522f97e911e04fc472fced0aee7916cbc287ad6c8e3", + "host": "example.com", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20151228T140825Z" + }, + "method": "get", + "url": "http://example.com" + } + p { + providers.aws.sign_req(req, aws_config, 1451311705000000000) == expected + } + note: providers-aws-sign_req/failure-simple-missing aws keys + query: data.test.p = x + want_error_code: eval_type_error + want_error: "providers.aws.sign_req: operand 2 missing required AWS config parameters(s): {\"aws_access_key\", \"aws_region\", \"aws_secret_access_key\", \"aws_service\"}" + strict_error: true +- data: + modules: + - | + package test + req := {"method": "get", "url": "http://example.com"} + aws_config := { + "aws_access_key": 1, + "aws_secret_access_key": 2, + "aws_session_token": 3, + "aws_service": 4, + "aws_region": 5, + } + expected := { + "headers": { + "Authorization": "AWS4-HMAC-SHA256 Credential=MYAWSACCESSKEYGOESHERE/20151228/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=0047a7016c81e5c7f29cd522f97e911e04fc472fced0aee7916cbc287ad6c8e3", + "host": "example.com", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20151228T140825Z" + }, + "method": "get", + "url": "http://example.com" + } + p { + providers.aws.sign_req(req, aws_config, 1451311705000000000) == expected + } + note: providers-aws-sign_req/failure-simple-invalid type aws keys + query: data.test.p = x + want_error_code: eval_type_error + want_error: "providers.aws.sign_req: operand 2 invalid values for required AWS config parameters(s): {\"aws_access_key\", \"aws_region\", \"aws_secret_access_key\", \"aws_service\"}" + strict_error: true +# timestamp errors: +- data: + modules: + - | + package test + req := {"method": "get", "url": "http://example.com"} + aws_config := { + "aws_access_key": "MYAWSACCESSKEYGOESHERE", + "aws_secret_access_key": "MYAWSSECRETACCESSKEYGOESHERE", + "aws_session_token": "MYAWSSECURITYTOKENGOESHERE", + "aws_service": "s3", + "aws_region": "us-east-1", + } + expected := { + "headers": { + "Authorization": "AWS4-HMAC-SHA256 Credential=MYAWSACCESSKEYGOESHERE/20151228/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=0047a7016c81e5c7f29cd522f97e911e04fc472fced0aee7916cbc287ad6c8e3", + "host": "example.com", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20151228T140825Z" + }, + "method": "get", + "url": "http://example.com" + } + p { + providers.aws.sign_req(req, aws_config, -1e9) == expected + } + note: providers-aws-sign_req/failure-simple-bad timestamp + query: data.test.p = x + want_error_code: eval_type_error + want_error: "providers.aws.sign_req: operand 3 could not convert time_ns value into a unix timestamp" + strict_error: true diff --git a/test/cases/testdata/providers-aws/aws-sign_req.yaml b/test/cases/testdata/providers-aws/aws-sign_req.yaml new file mode 100644 index 0000000000..0da602b23a --- /dev/null +++ b/test/cases/testdata/providers-aws/aws-sign_req.yaml @@ -0,0 +1,132 @@ +cases: +- data: + modules: + - | + package test + req := {"method": "get", "url": "http://example.com"} + aws_config := { + "aws_access_key": "MYAWSACCESSKEYGOESHERE", + "aws_secret_access_key": "MYAWSSECRETACCESSKEYGOESHERE", + "aws_service": "s3", + "aws_region": "us-east-1", + } + expected := { + "headers": { + "Authorization": "AWS4-HMAC-SHA256 Credential=MYAWSACCESSKEYGOESHERE/20151228/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=0047a7016c81e5c7f29cd522f97e911e04fc472fced0aee7916cbc287ad6c8e3", + "host": "example.com", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20151228T140825Z" + }, + "method": "get", + "url": "http://example.com" + } + p { + providers.aws.sign_req(req, aws_config, 1451311705000000000) == expected + } + note: providers-aws-sign_req/success-simple-no body + query: data.test.p = x + want_result: + - x: true +- data: + modules: + - | + package test + req := {"method": "get", "url": "http://example.com"} + aws_config := { + "aws_access_key": "MYAWSACCESSKEYGOESHERE", + "aws_secret_access_key": "MYAWSSECRETACCESSKEYGOESHERE", + "aws_session_token": "MYAWSSECURITYTOKENGOESHERE", + "aws_service": "s3", + "aws_region": "us-east-1", + } + expected := { + "headers": { + "Authorization": "AWS4-HMAC-SHA256 Credential=MYAWSACCESSKEYGOESHERE/20151228/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token,Signature=23c31bda8a74630c0a94f6b82a3511e7e74728df98e6f54ed0840488dbbdc8d1", + "host": "example.com", + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-date": "20151228T140825Z", + "x-amz-security-token": "MYAWSSECURITYTOKENGOESHERE" + }, + "method": "get", + "url": "http://example.com" + } + p { + providers.aws.sign_req(req, aws_config, 1451311705000000000) == expected + } + note: providers-aws-sign_req/success-simple-no body-with session token + query: data.test.p = x + want_result: + - x: true +- data: + modules: + - | + package test + req := {"method": "get", "url": "http://example.com", "body": {"example": {1, 2, 3, 4}}} + aws_config := {"aws_access_key": "MYAWSACCESSKEYGOESHERE", "aws_secret_access_key": "MYAWSSECRETACCESSKEYGOESHERE", "aws_service": "s3", "aws_region": "us-east-1"} + expected := { + "body": {"example": {1, 2, 3, 4}}, + "headers": { + "Authorization": "AWS4-HMAC-SHA256 Credential=MYAWSACCESSKEYGOESHERE/20151228/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=5fb06ab1cfd74c8fcb3af95b8cce696708cf6155d971dac10f254d79799c6e88", + "host": "example.com", + "x-amz-content-sha256": "bacff6243c850423883052ac3c336fd645994442933408dfd3f9e858e69bda07", + "x-amz-date": "20151228T140825Z" + }, + "method": "get", + "url": "http://example.com" + } + p { + providers.aws.sign_req(req, aws_config, 1451311705000000000) == expected + } + note: providers-aws-sign_req/success-simple-body + query: data.test.p = x + want_result: + - x: true +- data: + modules: + - | + package test + req := {"method": "get", "url": "http://example.com", "raw_body": "{\"example\": {1, 2, 3, 4}}"} + aws_config := {"aws_access_key": "MYAWSACCESSKEYGOESHERE", "aws_secret_access_key": "MYAWSSECRETACCESSKEYGOESHERE", "aws_service": "s3", "aws_region": "us-east-1"} + expected := { + "headers": { + "Authorization": "AWS4-HMAC-SHA256 Credential=MYAWSACCESSKEYGOESHERE/20151228/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=5bf26e169cb8b02330dba39d53932532438fc856c2f10537689a09ed807c7195", + "host": "example.com", + "x-amz-content-sha256": "22906461e2a98a3e780d0fd260e341bed5e544661e97c5936fc7f3af11aaad8b", + "x-amz-date": "20151228T140825Z" + }, + "method": "get", + "raw_body": "{\"example\": {1, 2, 3, 4}}", + "url": "http://example.com" + } + p { + providers.aws.sign_req(req, aws_config, 1451311705000000000) == expected + } + note: providers-aws-sign_req/success-simple-raw_body + query: data.test.p = x + want_result: + - x: true +- data: + modules: + - | + package test + req := {"method": "get", "url": "http://example.com", "body": {"example": {1, 2, 3, 4}}, "raw_body": "{\"example\": {1, 2, 3, 4}}"} + aws_config := {"aws_access_key": "MYAWSACCESSKEYGOESHERE", "aws_secret_access_key": "MYAWSSECRETACCESSKEYGOESHERE", "aws_service": "s3", "aws_region": "us-east-1"} + expected := { + "body": {"example": {1, 2, 3, 4}}, + "headers": { + "Authorization": "AWS4-HMAC-SHA256 Credential=MYAWSACCESSKEYGOESHERE/20151228/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=5bf26e169cb8b02330dba39d53932532438fc856c2f10537689a09ed807c7195", + "host": "example.com", + "x-amz-content-sha256": "22906461e2a98a3e780d0fd260e341bed5e544661e97c5936fc7f3af11aaad8b", + "x-amz-date": "20151228T140825Z" + }, + "method": "get", + "raw_body": "{\"example\": {1, 2, 3, 4}}", + "url": "http://example.com" + } + p { + providers.aws.sign_req(req, aws_config, 1451311705000000000) == expected + } + note: providers-aws-sign_req/success-simple-body-and-raw_body + query: data.test.p = x + want_result: + - x: true diff --git a/topdown/providers.go b/topdown/providers.go new file mode 100644 index 0000000000..8d988eab2c --- /dev/null +++ b/topdown/providers.go @@ -0,0 +1,191 @@ +// Copyright 2022 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "encoding/json" + "net/url" + "strings" + "time" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/internal/providers" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +var awsRequiredConfigKeyNames = ast.NewSet( + ast.StringTerm("aws_service"), + ast.StringTerm("aws_access_key"), + ast.StringTerm("aws_secret_access_key"), + ast.StringTerm("aws_region"), +) + +func stringFromTerm(t *ast.Term) string { + if v, ok := t.Value.(ast.String); ok { + return string(v) + } + return "" +} + +func getReqBodyBytes(body, rawBody *ast.Term) ([]byte, error) { + var out []byte + + switch { + case rawBody != nil: + out = []byte(stringFromTerm(rawBody)) + case body != nil: + bodyVal := body.Value + bodyValInterface, err := ast.JSON(bodyVal) + if err != nil { + return nil, err + } + bodyValBytes, err := json.Marshal(bodyValInterface) + if err != nil { + return nil, err + } + out = bodyValBytes + default: + out = []byte("") + } + + return out, nil +} + +func objectToMap(o ast.Object) map[string][]string { + var out map[string][]string + o.Foreach(func(k, v *ast.Term) { + ks := stringFromTerm(k) + vs := stringFromTerm(v) + out[ks] = []string{vs} + }) + return out +} + +// Note(philipc): This is roughly the same approach used for http.send. +func validateAWSAuthParameters(o ast.Object) error { + awsKeys := ast.NewSet(o.Keys()...) + + missingKeys := awsRequiredConfigKeyNames.Diff(awsKeys) + if missingKeys.Len() != 0 { + return builtins.NewOperandErr(2, "missing required AWS config parameters(s): %v", missingKeys) + } + + invalidKeys := ast.NewSet() + awsRequiredConfigKeyNames.Foreach(func(t *ast.Term) { + if v := o.Get(t); v != nil { + if _, ok := v.Value.(ast.String); !ok { + invalidKeys.Add(t) + } + } + }) + if invalidKeys.Len() != 0 { + return builtins.NewOperandErr(2, "invalid values for required AWS config parameters(s): %v", invalidKeys) + } + + return nil +} + +func builtinAWSSigV4SignReq(ctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error { + // Request object. + reqObj, err := builtins.ObjectOperand(operands[0].Value, 1) + if err != nil { + return err + } + + // AWS SigV4 config info object. + awsConfigObj, err := builtins.ObjectOperand(operands[1].Value, 1) + if err != nil { + return err + } + // Make sure our required keys exist! + err = validateAWSAuthParameters(awsConfigObj) + if err != nil { + return err + } + service := stringFromTerm(awsConfigObj.Get(ast.StringTerm("aws_service"))) + awsCreds := providers.AWSCredentialsFromObject(awsConfigObj) + + // Timestamp for signing. + var signingTimestamp time.Time + timestamp, err := builtins.NumberOperand(operands[2].Value, 1) + if err != nil { + return err + } + + ts, ok := timestamp.Int64() + if !ok { + return builtins.NewOperandErr(3, "could not convert time_ns value into a unix timestamp") + } + + signingTimestamp = time.Unix(0, ts) + if err != nil { + return err + } + + // Make sure our required keys exist! + // This check is stricter than required, but better to break here than downstream. + _, err = validateHTTPRequestOperand(operands[0], 1) + if err != nil { + return err + } + + // Prepare required fields from the HTTP request object. + var theURL *url.URL + var method string + reqURL := reqObj.Get(ast.StringTerm("url")) + reqMethod := reqObj.Get(ast.StringTerm("method")) + + headers := ast.NewObject() + headersTerm := reqObj.Get(ast.StringTerm("headers")) + if headersTerm != nil { + var ok bool + headers, ok = headersTerm.Value.(ast.Object) + if !ok { + return builtins.NewOperandTypeErr(0, headersTerm.Value, "object") + } + } + + // Check types on the request parameters. + invalidParameters := ast.NewSet() + if _, ok := reqURL.Value.(ast.String); !ok { + invalidParameters.Add(ast.StringTerm("url")) + } + if _, ok := reqMethod.Value.(ast.String); !ok { + invalidParameters.Add(ast.StringTerm("method")) + } + if invalidParameters.Len() > 0 { + return builtins.NewOperandErr(1, "invalid values for required request parameters(s): %v", invalidParameters) + } + + theURL, err = url.Parse(stringFromTerm(reqURL)) + if err != nil { + return err + } + method = stringFromTerm(reqMethod) + + bodyTerm := reqObj.Get(ast.StringTerm("body")) + rawBodyTerm := reqObj.Get(ast.StringTerm("raw_body")) + body, err := getReqBodyBytes(bodyTerm, rawBodyTerm) + if err != nil { + return err + } + + // Sign the request object's headers, and reconstruct the headers map. + signedHeadersMap := providers.AWSSignV4(objectToMap(headers), method, theURL, body, service, awsCreds, signingTimestamp) + signedHeadersObj := ast.NewObject() + for k, v := range signedHeadersMap { + signedHeadersObj.Insert(ast.StringTerm(k), ast.StringTerm(strings.Join(v, ","))) + } + + // Create new request object with updated headers. + out := reqObj.Copy() + out.Insert(ast.StringTerm("headers"), ast.NewTerm(signedHeadersObj)) + + return iter(ast.NewTerm(out)) +} + +func init() { + RegisterBuiltinFunc(ast.ProvidersAWSSignReqObj.Name, builtinAWSSigV4SignReq) +}