diff --git a/docs/tables/net_http_request.md b/docs/tables/net_http_request.md index c4acb63..c5d5667 100644 --- a/docs/tables/net_http_request.md +++ b/docs/tables/net_http_request.md @@ -186,4 +186,98 @@ from net_http_request where url = 'http://microsoft.com'; -``` \ No newline at end of file +``` + +### Send a GET request with TLS certificate verification disabled +Explore how to make a request to a site with an invalid or self-signed certificate by disabling TLS certificate verification. This is similar to using curl with the -k flag. + +```sql+postgres +select + url, + method, + response_status_code, + response_error, + response_body +from + net_http_request +where + url = 'https://self-signed.badssl.com' + and insecure = true; +``` + +```sql+sqlite +select + url, + method, + response_status_code, + response_error, + response_body +from + net_http_request +where + url = 'https://self-signed.badssl.com' + and insecure = true; +``` + +### Send a GET request with Basic Authentication +Explore how to authenticate with APIs that require Basic Authentication by providing credentials in a simple username:password format. This is equivalent to using curl's --user flag and automatically handles the base64 encoding of credentials. + +```sql+postgres +select + url, + method, + response_status_code, + response_headers, + response_body +from + net_http_request +where + url = 'https://httpbin.org/basic-auth/myuser/mypass' + and user_credentials = 'myuser:mypass'; +``` + +```sql+sqlite +select + url, + method, + response_status_code, + response_headers, + response_body +from + net_http_request +where + url = 'https://httpbin.org/basic-auth/myuser/mypass' + and user_credentials = 'myuser:mypass'; +``` + +### Send a request with Basic Authentication and skip TLS verification +This example demonstrates how to connect to an API with a self-signed certificate while using Basic Authentication. This is useful for internal or development APIs that require authentication but don't have valid certificates. + +```sql+postgres +select + url, + method, + response_status_code, + response_headers, + response_body +from + net_http_request +where + url = 'https://internal-api.example.com:123/api/help' + and user_credentials = 'apiuser:secretpass' + and insecure = true; +``` + +```sql+sqlite +select + url, + method, + response_status_code, + response_headers, + response_body +from + net_http_request +where + url = 'https://internal-api.example.com:123/api/help' + and user_credentials = 'apiuser:secretpass' + and insecure = true; \ No newline at end of file diff --git a/net/table_net_http_request.go b/net/table_net_http_request.go index ecf197c..8a99f9f 100644 --- a/net/table_net_http_request.go +++ b/net/table_net_http_request.go @@ -3,6 +3,8 @@ package net import ( "bytes" "context" + "crypto/tls" + "encoding/base64" "encoding/json" "fmt" "net/http" @@ -28,14 +30,18 @@ func tableNetHTTPRequest() *plugin.Table { {Name: "follow_redirects", Require: plugin.Optional, Operators: []string{"=", "<>"}, CacheMatch: "exact"}, {Name: "request_headers", Require: plugin.Optional, CacheMatch: "exact"}, {Name: "request_body", Require: plugin.Optional, CacheMatch: "exact"}, + {Name: "insecure", Require: plugin.Optional, Operators: []string{"=", "<>"}, CacheMatch: "exact"}, + {Name: "user_credentials", Require: plugin.Optional, CacheMatch: "exact"}, }, }, Columns: []*plugin.Column{ {Name: "url", Transform: transform.FromField("Url"), Type: proto.ColumnType_STRING, Description: "URL of the site."}, {Name: "method", Type: proto.ColumnType_STRING, Description: "Specifies the HTTP method (GET, POST)."}, {Name: "follow_redirects", Type: proto.ColumnType_BOOL, Description: "If true, the requests will follow the redirects."}, + {Name: "insecure", Type: proto.ColumnType_BOOL, Description: "If true, TLS certificate verification will be skipped (similar to curl -k)."}, {Name: "request_body", Type: proto.ColumnType_STRING, Description: "The request's body."}, {Name: "request_headers", Type: proto.ColumnType_JSON, Transform: transform.FromQual("request_headers"), Description: "A map of headers passed in the request."}, + {Name: "user_credentials", Type: proto.ColumnType_STRING, Transform: transform.FromQual("user_credentials"), Description: "Basic auth credentials in the format username:password (similar to curl --user)."}, {Name: "response_status_code", Type: proto.ColumnType_INT, Description: "HTTP status code is a server response to a browser's request."}, {Name: "response_body", Type: proto.ColumnType_STRING, Description: "Represents the response body."}, {Name: "response_error", Type: proto.ColumnType_STRING, Description: "Represents an error or failure, either from a non-successful HTTP status, an error while executing the request, or some other failure which occurred during the parsing of the response.", Transform: transform.FromField("Error")}, @@ -49,7 +55,9 @@ func listBaseRequestAttributes(ctx context.Context, d *plugin.QueryData, h *plug var methods []string var requestBody string + var userCredentials string headers := make(map[string]interface{}) + insecure := false queryCols := d.EqualsQuals @@ -62,6 +70,18 @@ func listBaseRequestAttributes(ctx context.Context, d *plugin.QueryData, h *plug methods = []string{"GET"} } + // Check for insecure option + if d.Quals["insecure"] != nil { + for _, q := range d.Quals["insecure"].Quals { + switch q.Operator { + case "=": + insecure = q.Value.GetBoolValue() + case "<>": + insecure = !q.Value.GetBoolValue() + } + } + } + requestHeadersString := queryCols["request_headers"].GetJsonbValue() logger.Debug("listBaseRequestAttributes", "request headers", requestHeadersString) @@ -81,12 +101,19 @@ func listBaseRequestAttributes(ctx context.Context, d *plugin.QueryData, h *plug requestBody = requestBodyData } + // Get user credentials if provided + if creds, present := getAuthHeaderQuals(queryCols["user_credentials"]); present { + userCredentials = creds + } + logger.Debug("listBaseRequestAttributes", "urls", urls) logger.Debug("listBaseRequestAttributes", "methods", methods) logger.Debug("listBaseRequestAttributes", "headers", headers) + logger.Debug("listBaseRequestAttributes", "insecure", insecure) + logger.Debug("listBaseRequestAttributes", "user_credentials", userCredentials != "") for _, url := range urls { - d.StreamListItem(ctx, baseRequestAttributes{url, methods, requestBody, headers}) + d.StreamListItem(ctx, baseRequestAttributes{url, methods, requestBody, headers, insecure, userCredentials}) } return nil, nil @@ -102,7 +129,15 @@ func listRequestResponses(ctx context.Context, d *plugin.QueryData, h *plugin.Hy methods := baseRequestAttribute.Methods headers := baseRequestAttribute.Headers requestBody := baseRequestAttribute.RequestBody - client := &http.Client{} + insecure := baseRequestAttribute.Insecure + userCredentials := baseRequestAttribute.UserCredentials + + // Create custom transport with TLS config if insecure is true + transport := &http.Transport{} + if insecure { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + client := &http.Client{Transport: transport} // Set true to follow the redirects // Default set to true @@ -151,7 +186,17 @@ func listRequestResponses(ctx context.Context, d *plugin.QueryData, h *plugin.Hy continue } + // Add request headers req = addRequestHeaders(req, headers) + + // Add Basic Authentication if user credentials are provided + if userCredentials != "" { + // Encode the credentials in base64 + auth := base64.StdEncoding.EncodeToString([]byte(userCredentials)) + req.Header.Set("Authorization", "Basic "+auth) + logger.Debug("listRequestResponses", "added basic auth header", true) + } + logger.Debug("listRequestResponses", "request", req) item := tableNetWebRequestRow{ @@ -159,6 +204,7 @@ func listRequestResponses(ctx context.Context, d *plugin.QueryData, h *plugin.Hy Method: method, RequestBody: requestBody, FollowRedirects: followRedirects, + Insecure: insecure, } // Make request @@ -191,4 +237,4 @@ func listRequestResponses(ctx context.Context, d *plugin.QueryData, h *plugin.Hy } return nil, nil -} +} \ No newline at end of file diff --git a/net/utils.go b/net/utils.go index 03fee5e..de4da0c 100644 --- a/net/utils.go +++ b/net/utils.go @@ -62,6 +62,8 @@ type tableNetWebRequestRow struct { RequestBody string RequestHeaders string FollowRedirects bool + Insecure bool + UserCredentials string Status int ResponseStatusCode int ResponseHeaders map[string][]string @@ -78,10 +80,12 @@ type tableNetWebRequestRow struct { } type baseRequestAttributes struct { - Url string - Methods []string - RequestBody string - Headers map[string]interface{} + Url string + Methods []string + RequestBody string + Headers map[string]interface{} + Insecure bool + UserCredentials string } func removeInvalidUTF8Char(str string) string {