From 2aa5511eff3140d2a5576389dc3951fcf98a78dc Mon Sep 17 00:00:00 2001 From: bob-bot Date: Wed, 28 May 2025 19:26:16 -0400 Subject: [PATCH 1/4] add insecure flag for http_request table --- docs/tables/net_http_request.md | 29 +++++++++++++++++++++++++++++ net/table_net_http_request.go | 28 ++++++++++++++++++++++++++-- net/utils.go | 2 ++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/docs/tables/net_http_request.md b/docs/tables/net_http_request.md index c4acb63..e341235 100644 --- a/docs/tables/net_http_request.md +++ b/docs/tables/net_http_request.md @@ -186,4 +186,33 @@ from net_http_request where url = 'http://microsoft.com'; +``` + +### Send a GET request with TLS certificate verification disabled +Explore how to make a request to a site with an invalid or self-signed certificate by disabling TLS certificate verification. This is similar to using curl with the -k flag. + +```sql+postgres +select + url, + method, + response_status_code, + jsonb_pretty(response_body::jsonb) as response_body +from + net_http_request +where + url = 'https://self-signed.example.com' + and insecure = true; +``` + +```sql+sqlite +select + url, + method, + response_status_code, + response_body +from + net_http_request +where + url = 'https://self-signed.example.com' + and insecure = true; ``` \ No newline at end of file diff --git a/net/table_net_http_request.go b/net/table_net_http_request.go index ecf197c..0c3c822 100644 --- a/net/table_net_http_request.go +++ b/net/table_net_http_request.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" "strings" + "crypto/tls" "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" "github.com/turbot/steampipe-plugin-sdk/v5/plugin" @@ -28,12 +29,14 @@ func tableNetHTTPRequest() *plugin.Table { {Name: "follow_redirects", Require: plugin.Optional, Operators: []string{"=", "<>"}, CacheMatch: "exact"}, {Name: "request_headers", Require: plugin.Optional, CacheMatch: "exact"}, {Name: "request_body", Require: plugin.Optional, CacheMatch: "exact"}, + {Name: "insecure", Require: plugin.Optional, Operators: []string{"=", "<>"}, CacheMatch: "exact"}, }, }, Columns: []*plugin.Column{ {Name: "url", Transform: transform.FromField("Url"), Type: proto.ColumnType_STRING, Description: "URL of the site."}, {Name: "method", Type: proto.ColumnType_STRING, Description: "Specifies the HTTP method (GET, POST)."}, {Name: "follow_redirects", Type: proto.ColumnType_BOOL, Description: "If true, the requests will follow the redirects."}, + {Name: "insecure", Type: proto.ColumnType_BOOL, Description: "If true, TLS certificate verification will be skipped (similar to curl -k)."}, {Name: "request_body", Type: proto.ColumnType_STRING, Description: "The request's body."}, {Name: "request_headers", Type: proto.ColumnType_JSON, Transform: transform.FromQual("request_headers"), Description: "A map of headers passed in the request."}, {Name: "response_status_code", Type: proto.ColumnType_INT, Description: "HTTP status code is a server response to a browser's request."}, @@ -50,6 +53,7 @@ func listBaseRequestAttributes(ctx context.Context, d *plugin.QueryData, h *plug var methods []string var requestBody string headers := make(map[string]interface{}) + insecure := false queryCols := d.EqualsQuals @@ -62,6 +66,18 @@ func listBaseRequestAttributes(ctx context.Context, d *plugin.QueryData, h *plug methods = []string{"GET"} } + // Check for insecure option + if d.Quals["insecure"] != nil { + for _, q := range d.Quals["insecure"].Quals { + switch q.Operator { + case "=": + insecure = q.Value.GetBoolValue() + case "<>": + insecure = !q.Value.GetBoolValue() + } + } + } + requestHeadersString := queryCols["request_headers"].GetJsonbValue() logger.Debug("listBaseRequestAttributes", "request headers", requestHeadersString) @@ -86,7 +102,7 @@ func listBaseRequestAttributes(ctx context.Context, d *plugin.QueryData, h *plug logger.Debug("listBaseRequestAttributes", "headers", headers) for _, url := range urls { - d.StreamListItem(ctx, baseRequestAttributes{url, methods, requestBody, headers}) + d.StreamListItem(ctx, baseRequestAttributes{url, methods, requestBody, headers, insecure}) } return nil, nil @@ -102,7 +118,14 @@ func listRequestResponses(ctx context.Context, d *plugin.QueryData, h *plugin.Hy methods := baseRequestAttribute.Methods headers := baseRequestAttribute.Headers requestBody := baseRequestAttribute.RequestBody - client := &http.Client{} + insecure := baseRequestAttribute.Insecure + + // Create custom transport with TLS config if insecure is true + transport := &http.Transport{} + if insecure { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + client := &http.Client{Transport: transport} // Set true to follow the redirects // Default set to true @@ -159,6 +182,7 @@ func listRequestResponses(ctx context.Context, d *plugin.QueryData, h *plugin.Hy Method: method, RequestBody: requestBody, FollowRedirects: followRedirects, + Insecure: insecure, } // Make request diff --git a/net/utils.go b/net/utils.go index 03fee5e..12b791e 100644 --- a/net/utils.go +++ b/net/utils.go @@ -62,6 +62,7 @@ type tableNetWebRequestRow struct { RequestBody string RequestHeaders string FollowRedirects bool + Insecure bool Status int ResponseStatusCode int ResponseHeaders map[string][]string @@ -82,6 +83,7 @@ type baseRequestAttributes struct { Methods []string RequestBody string Headers map[string]interface{} + Insecure bool } func removeInvalidUTF8Char(str string) string { From b0db5f9e843794c1bc07652cd56435bdd6d7abda Mon Sep 17 00:00:00 2001 From: bob-bot Date: Wed, 28 May 2025 20:00:50 -0400 Subject: [PATCH 2/4] updated doc example --- docs/tables/net_http_request.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tables/net_http_request.md b/docs/tables/net_http_request.md index e341235..83f4fee 100644 --- a/docs/tables/net_http_request.md +++ b/docs/tables/net_http_request.md @@ -200,7 +200,7 @@ select from net_http_request where - url = 'https://self-signed.example.com' + url = 'https://self-signed.badssl.com' and insecure = true; ``` @@ -213,6 +213,6 @@ select from net_http_request where - url = 'https://self-signed.example.com' + url = 'https://self-signed.badssl.com' and insecure = true; ``` \ No newline at end of file From eb752ec096f070ca8c5bb99e90c8bcc4d7ddc969 Mon Sep 17 00:00:00 2001 From: bob-bot Date: Wed, 28 May 2025 20:07:21 -0400 Subject: [PATCH 3/4] updated doc example --- docs/tables/net_http_request.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/tables/net_http_request.md b/docs/tables/net_http_request.md index 83f4fee..e6d5d9c 100644 --- a/docs/tables/net_http_request.md +++ b/docs/tables/net_http_request.md @@ -196,7 +196,8 @@ select url, method, response_status_code, - jsonb_pretty(response_body::jsonb) as response_body + response_error, + response_body from net_http_request where @@ -209,6 +210,7 @@ select url, method, response_status_code, + response_error, response_body from net_http_request From 64d177970ea83940a81388c73fa678072fbe1121 Mon Sep 17 00:00:00 2001 From: bob-bot Date: Wed, 28 May 2025 22:01:14 -0400 Subject: [PATCH 4/4] adding user auth column --- docs/tables/net_http_request.md | 65 ++++++++++++++++++++++++++++++++- net/table_net_http_request.go | 28 ++++++++++++-- net/utils.go | 12 +++--- 3 files changed, 96 insertions(+), 9 deletions(-) diff --git a/docs/tables/net_http_request.md b/docs/tables/net_http_request.md index e6d5d9c..c5d5667 100644 --- a/docs/tables/net_http_request.md +++ b/docs/tables/net_http_request.md @@ -217,4 +217,67 @@ from where url = 'https://self-signed.badssl.com' and insecure = true; -``` \ No newline at end of file +``` + +### Send a GET request with Basic Authentication +Explore how to authenticate with APIs that require Basic Authentication by providing credentials in a simple username:password format. This is equivalent to using curl's --user flag and automatically handles the base64 encoding of credentials. + +```sql+postgres +select + url, + method, + response_status_code, + response_headers, + response_body +from + net_http_request +where + url = 'https://httpbin.org/basic-auth/myuser/mypass' + and user_credentials = 'myuser:mypass'; +``` + +```sql+sqlite +select + url, + method, + response_status_code, + response_headers, + response_body +from + net_http_request +where + url = 'https://httpbin.org/basic-auth/myuser/mypass' + and user_credentials = 'myuser:mypass'; +``` + +### Send a request with Basic Authentication and skip TLS verification +This example demonstrates how to connect to an API with a self-signed certificate while using Basic Authentication. This is useful for internal or development APIs that require authentication but don't have valid certificates. + +```sql+postgres +select + url, + method, + response_status_code, + response_headers, + response_body +from + net_http_request +where + url = 'https://internal-api.example.com:123/api/help' + and user_credentials = 'apiuser:secretpass' + and insecure = true; +``` + +```sql+sqlite +select + url, + method, + response_status_code, + response_headers, + response_body +from + net_http_request +where + url = 'https://internal-api.example.com:123/api/help' + and user_credentials = 'apiuser:secretpass' + and insecure = true; \ No newline at end of file diff --git a/net/table_net_http_request.go b/net/table_net_http_request.go index 0c3c822..8a99f9f 100644 --- a/net/table_net_http_request.go +++ b/net/table_net_http_request.go @@ -3,11 +3,12 @@ package net import ( "bytes" "context" + "crypto/tls" + "encoding/base64" "encoding/json" "fmt" "net/http" "strings" - "crypto/tls" "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" "github.com/turbot/steampipe-plugin-sdk/v5/plugin" @@ -30,6 +31,7 @@ func tableNetHTTPRequest() *plugin.Table { {Name: "request_headers", Require: plugin.Optional, CacheMatch: "exact"}, {Name: "request_body", Require: plugin.Optional, CacheMatch: "exact"}, {Name: "insecure", Require: plugin.Optional, Operators: []string{"=", "<>"}, CacheMatch: "exact"}, + {Name: "user_credentials", Require: plugin.Optional, CacheMatch: "exact"}, }, }, Columns: []*plugin.Column{ @@ -39,6 +41,7 @@ func tableNetHTTPRequest() *plugin.Table { {Name: "insecure", Type: proto.ColumnType_BOOL, Description: "If true, TLS certificate verification will be skipped (similar to curl -k)."}, {Name: "request_body", Type: proto.ColumnType_STRING, Description: "The request's body."}, {Name: "request_headers", Type: proto.ColumnType_JSON, Transform: transform.FromQual("request_headers"), Description: "A map of headers passed in the request."}, + {Name: "user_credentials", Type: proto.ColumnType_STRING, Transform: transform.FromQual("user_credentials"), Description: "Basic auth credentials in the format username:password (similar to curl --user)."}, {Name: "response_status_code", Type: proto.ColumnType_INT, Description: "HTTP status code is a server response to a browser's request."}, {Name: "response_body", Type: proto.ColumnType_STRING, Description: "Represents the response body."}, {Name: "response_error", Type: proto.ColumnType_STRING, Description: "Represents an error or failure, either from a non-successful HTTP status, an error while executing the request, or some other failure which occurred during the parsing of the response.", Transform: transform.FromField("Error")}, @@ -52,6 +55,7 @@ func listBaseRequestAttributes(ctx context.Context, d *plugin.QueryData, h *plug var methods []string var requestBody string + var userCredentials string headers := make(map[string]interface{}) insecure := false @@ -97,12 +101,19 @@ func listBaseRequestAttributes(ctx context.Context, d *plugin.QueryData, h *plug requestBody = requestBodyData } + // Get user credentials if provided + if creds, present := getAuthHeaderQuals(queryCols["user_credentials"]); present { + userCredentials = creds + } + logger.Debug("listBaseRequestAttributes", "urls", urls) logger.Debug("listBaseRequestAttributes", "methods", methods) logger.Debug("listBaseRequestAttributes", "headers", headers) + logger.Debug("listBaseRequestAttributes", "insecure", insecure) + logger.Debug("listBaseRequestAttributes", "user_credentials", userCredentials != "") for _, url := range urls { - d.StreamListItem(ctx, baseRequestAttributes{url, methods, requestBody, headers, insecure}) + d.StreamListItem(ctx, baseRequestAttributes{url, methods, requestBody, headers, insecure, userCredentials}) } return nil, nil @@ -119,6 +130,7 @@ func listRequestResponses(ctx context.Context, d *plugin.QueryData, h *plugin.Hy headers := baseRequestAttribute.Headers requestBody := baseRequestAttribute.RequestBody insecure := baseRequestAttribute.Insecure + userCredentials := baseRequestAttribute.UserCredentials // Create custom transport with TLS config if insecure is true transport := &http.Transport{} @@ -174,7 +186,17 @@ func listRequestResponses(ctx context.Context, d *plugin.QueryData, h *plugin.Hy continue } + // Add request headers req = addRequestHeaders(req, headers) + + // Add Basic Authentication if user credentials are provided + if userCredentials != "" { + // Encode the credentials in base64 + auth := base64.StdEncoding.EncodeToString([]byte(userCredentials)) + req.Header.Set("Authorization", "Basic "+auth) + logger.Debug("listRequestResponses", "added basic auth header", true) + } + logger.Debug("listRequestResponses", "request", req) item := tableNetWebRequestRow{ @@ -215,4 +237,4 @@ func listRequestResponses(ctx context.Context, d *plugin.QueryData, h *plugin.Hy } return nil, nil -} +} \ No newline at end of file diff --git a/net/utils.go b/net/utils.go index 12b791e..de4da0c 100644 --- a/net/utils.go +++ b/net/utils.go @@ -63,6 +63,7 @@ type tableNetWebRequestRow struct { RequestHeaders string FollowRedirects bool Insecure bool + UserCredentials string Status int ResponseStatusCode int ResponseHeaders map[string][]string @@ -79,11 +80,12 @@ type tableNetWebRequestRow struct { } type baseRequestAttributes struct { - Url string - Methods []string - RequestBody string - Headers map[string]interface{} - Insecure bool + Url string + Methods []string + RequestBody string + Headers map[string]interface{} + Insecure bool + UserCredentials string } func removeInvalidUTF8Char(str string) string {