http,https: add built-in proxy support in http/https.request and Agent

doc/api/errors.md

@@ -2491,6 +2491,18 @@ Accessing `Object.prototype.__proto__` has been forbidden using
 [`Object.setPrototypeOf`][] should be used to get and set the prototype of an
 object.
 
+<a id="ERR_PROXY_INVALID_CONFIG"></a>
+
+### `ERR_PROXY_INVALID_CONFIG`
+
+Failed to proxy a request because the proxy configuration is invalid.
+
+<a id="ERR_PROXY_TUNNEL"></a>
+
+### `ERR_PROXY_TUNNEL`
+
+Failed to establish proxy tunnel when `NODE_USE_ENV_PROXY` is enabled.
+
 <a id="ERR_QUIC_APPLICATION_ERROR"></a>
 
 ### `ERR_QUIC_APPLICATION_ERROR`

doc/api/http.md

@@ -116,6 +116,14 @@ http.get({
 <!-- YAML
 added: v0.3.4
 changes:
+  - version:
+    - REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/58980
+    description: Add support for `proxyEnv`.
+  - version:
+    - REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/58980
+    description: Add support for `defaultPort` and `protocol`.
   - version:
       - v15.6.0
       - v14.17.0
@@ -178,6 +186,20 @@ changes:
     **Default:** `'lifo'`.
   * `timeout` {number} Socket timeout in milliseconds.
     This will set the timeout when the socket is created.
+  * `proxyEnv` {Object|undefined} Environment variables for proxy configuration.
+    See [Built-in Proxy Support][] for details. **Default:** `undefined`
+    * `HTTP_PROXY` {string|undefined} URL for the proxy server that HTTP requests should use.
+      If undefined, no proxy is used for HTTP requests.
+    * `HTTPS_PROXY` {string|undefined} URL for the proxy server that HTTPS requests should use.
+      If undefined, no proxy is used for HTTPS requests.
+    * `NO_PROXY` {string|undefined} Patterns specifying the endpoints
+      that should not be routed through a proxy.
+    * `http_proxy` {string|undefined} Same as `HTTP_PROXY`. If both are set, `http_proxy` takes precedence.
+    * `https_proxy` {string|undefined} Same as `HTTPS_PROXY`. If both are set, `https_proxy` takes precedence.
+    * `no_proxy` {string|undefined} Same as `NO_PROXY`. If both are set, `no_proxy` takes precedence.
+  * `defaultPort` {number} Default port to use when the port is not specified
+    in requests. **Default:** `80`.
+  * `protocol` {string} The protocol to use for the agent. **Default:** `'http:'`.
 
 `options` in [`socket.connect()`][] are also supported.
 
@@ -4243,6 +4265,98 @@ added:
 
 A browser-compatible implementation of {WebSocket}.
 
+## Built-in Proxy Support
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1.1 - Active development
+
+When Node.js creates the global agent, it checks the `NODE_USE_ENV_PROXY`
+environment variable. If it is set to `1`, the global agent will be constructed
+with `proxyEnv: process.env`, enabling proxy support based on the environment variables.
+
+Custom agents can also be created with proxy support by passing a
+`proxyEnv` option when constructing the agent. The value can be `process.env`
+if they just want to inherit the configuration from the environment variables,
+or an object with specific setting overriding the environment.
+
+The following properties of the `proxyEnv` are checked to configure proxy
+support.
+
+* `HTTP_PROXY` or `http_proxy`: Proxy server URL for HTTP requests. If both are set,
+  `http_proxy` takes precedence.
+* `HTTPS_PROXY` or `https_proxy`: Proxy server URL for HTTPS requests. If both are set,
+  `https_proxy` takes precedence.
+* `NO_PROXY` or `no_proxy`: Comma-separated list of hosts to bypass the proxy. If both are set,
+  `no_proxy` takes precedence.
+
+If the request is made to a Unix domain socket, the proxy settings will be ignored.
+
+### Proxy URL Format
+
+Proxy URLs can use either HTTP or HTTPS protocols:
+
+* HTTP proxy: `http://proxy.example.com:8080`
+* HTTPS proxy: `https://proxy.example.com:8080`
+* Proxy with authentication: `http://username:[email protected]:8080`
+
+### `NO_PROXY` Format
+
+The `NO_PROXY` environment variable supports several formats:
+
+* `*` - Bypass proxy for all hosts
+* `example.com` - Exact host name match
+* `.example.com` - Domain suffix match (matches `sub.example.com`)
+* `*.example.com` - Wildcard domain match
+* `192.168.1.100` - Exact IP address match
+* `192.168.1.1-192.168.1.100` - IP address range
+* `example.com:8080` - Hostname with specific port
+
+Multiple entries should be separated by commas.
+
+### Example
+
+Starting a Node.js process with proxy support enabled for all requests sent
+through the default global agent:
+
+```console
+NODE_USE_ENV_PROXY=1 HTTP_PROXY=http://proxy.example.com:8080 NO_PROXY=localhost,127.0.0.1 node client.js
+```
+
+To create a custom agent with built-in proxy support:
+
+```cjs
+const http = require('node:http');
+
+// Creating a custom agent with custom proxy support.
+const agent = new http.Agent({ proxyEnv: { HTTP_PROXY: 'http://proxy.example.com:8080' } });
+
+http.request({
+  hostname: 'www.example.com',
+  port: 80,
+  path: '/',
+  agent,
+}, (res) => {
+  // This request will be proxied through proxy.example.com:8080 using the HTTP protocol.
+  console.log(`STATUS: ${res.statusCode}`);
+});
+```
+
+Alternatively, the following also works:
+
+```cjs
+const http = require('node:http');
+// Use lower-cased option name.
+const agent1 = new http.Agent({ proxyEnv: { http_proxy: 'http://proxy.example.com:8080' } });
+// Use values inherited from the environment variables, if the process is started with
+// HTTP_PROXY=http://proxy.example.com:8080 this will use the proxy server specified
+// in process.env.HTTP_PROXY.
+const agent2 = new http.Agent({ proxyEnv: process.env });
+```
+
+[Built-in Proxy Support]: #built-in-proxy-support
 [RFC 8187]: https://www.rfc-editor.org/rfc/rfc8187.txt
 [`'ERR_HTTP_CONTENT_LENGTH_MISMATCH'`]: errors.md#err_http_content_length_mismatch
 [`'checkContinue'`]: #event-checkcontinue

doc/api/https.md

@@ -65,6 +65,14 @@ An [`Agent`][] object for HTTPS similar to [`http.Agent`][]. See
 
 <!-- YAML
 changes:
+  - version:
+    - REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/58980
+    description: Add support for `proxyEnv`.
+  - version:
+    - REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/58980
+    description: Add support for `defaultPort` and `protocol`.
   - version: v12.5.0
     pr-url: https://github.com/nodejs/node/pull/28209
     description: do not automatically set servername if the target host was

lib/_http_agent.js

@@ -34,9 +34,17 @@ const EventEmitter = require('events');
 let debug = require('internal/util/debuglog').debuglog('http', (fn) => {
   debug = fn;
 });
+const {
+  parseProxyConfigFromEnv,
+  kProxyConfig,
+  checkShouldUseProxy,
+  kWaitForProxyTunnel,
+  filterEnvForProxies,
+} = require('internal/http');
 const { AsyncResource } = require('async_hooks');
 const { async_id_symbol } = require('internal/async_hooks').symbols;
 const {
+  getLazy,
   kEmptyObject,
   once,
 } = require('internal/util');
@@ -45,6 +53,7 @@ const {
   validateOneOf,
   validateString,
 } = require('internal/validators');
+const assert = require('internal/assert');
 
 const kOnKeylog = Symbol('onkeylog');
 const kRequestOptions = Symbol('requestOptions');
@@ -84,11 +93,11 @@ function Agent(options) {
 
   EventEmitter.call(this);
 
-  this.defaultPort = 80;
-  this.protocol = 'http:';
-
   this.options = { __proto__: null, ...options };
 
+  this.defaultPort = this.options.defaultPort || 80;
+  this.protocol = this.options.protocol || 'http:';
+
   if (this.options.noDelay === undefined)
     this.options.noDelay = true;
 
@@ -104,6 +113,11 @@ function Agent(options) {
   this.scheduling = this.options.scheduling || 'lifo';
   this.maxTotalSockets = this.options.maxTotalSockets;
   this.totalSocketCount = 0;
+  const proxyEnv = this.options.proxyEnv;
+  if (typeof proxyEnv === 'object' && proxyEnv !== null) {
+    this[kProxyConfig] = parseProxyConfigFromEnv(proxyEnv, this.protocol, this.keepAlive);
+    debug(`new ${this.protocol} agent with proxy config`, this[kProxyConfig]);
+  }
 
   validateOneOf(this.scheduling, 'scheduling', ['fifo', 'lifo']);
 
@@ -200,9 +214,40 @@ function maybeEnableKeylog(eventName) {
   }
 }
 
+const lazyTLS = getLazy(() => require('tls'));
+
 Agent.defaultMaxSockets = Infinity;
 
-Agent.prototype.createConnection = net.createConnection;
+// See ProxyConfig in internal/http.js for how the connection should be handled
+// when the agent is configured to use a proxy server.
+Agent.prototype.createConnection = function createConnection(...args) {
+  const normalized = net._normalizeArgs(args);
+  const options = normalized[0];
+  const cb = normalized[1];
+
+  // Check if this specific request should bypass the proxy
+  const shouldUseProxy = checkShouldUseProxy(this[kProxyConfig], options);
+  debug(`http createConnection should use proxy for ${options.host}:${options.port}:`, shouldUseProxy);
+  if (!shouldUseProxy) {  // Forward to net.createConnection if no proxying is needed.
+    return net.createConnection(...args);
+  }
+
+  // Create a copy of the shared proxy connection options and connect
+  // to the proxy server instead of the endpoint. For Agent.prototype.createConnection
+  // which is used by the http agent, this is enough
+  const connectOptions = {
+    ...this[kProxyConfig].proxyConnectionOptions,
+  };
+  const proxyProtocol = this[kProxyConfig].protocol;
+  if (proxyProtocol === 'http:') {
+    return net.connect(connectOptions, cb);
+  } else if (proxyProtocol === 'https:') {
+    return lazyTLS().connect(connectOptions, cb);
+  }
+  // This should be unreachable because proxy config should be null for other protocols.
+  assert.fail(`Unexpected proxy protocol ${proxyProtocol}`);
+
+};
 
 // Get the key for a given set of request options
 Agent.prototype.getName = function getName(options = kEmptyObject) {
@@ -227,6 +272,16 @@ Agent.prototype.getName = function getName(options = kEmptyObject) {
   return name;
 };
 
+function handleSocketAfterProxy(err, req) {
+  if (err.code === 'ERR_PROXY_TUNNEL') {
+    if (err.proxyTunnelTimeout) {
+      req.emit('timeout');  // Propagate the timeout from the tunnel to the request.
+    } else {
+      req.emit('error', err);
+    }
+  }
+}
+
 Agent.prototype.addRequest = function addRequest(req, options, port/* legacy */,
                                                  localAddress/* legacy */) {
   // Legacy API: addRequest(req, host, port, localAddress)
@@ -239,6 +294,7 @@ Agent.prototype.addRequest = function addRequest(req, options, port/* legacy */,
     };
   }
 
+  // Here the agent options will override per-request options.
   options = { __proto__: null, ...options, ...this.options };
   if (options.socketPath)
     options.path = options.socketPath;
@@ -264,20 +320,24 @@ Agent.prototype.addRequest = function addRequest(req, options, port/* legacy */,
   const freeLen = freeSockets ? freeSockets.length : 0;
   const sockLen = freeLen + this.sockets[name].length;
 
+  // Reusing a socket from the pool.
   if (socket) {
     asyncResetHandle(socket);
     this.reuseSocket(socket, req);
     setRequestSocket(this, req, socket);
     this.sockets[name].push(socket);
   } else if (sockLen < this.maxSockets &&
              this.totalSocketCount < this.maxTotalSockets) {
-    debug('call onSocket', sockLen, freeLen);
     // If we are under maxSockets create a new one.
     this.createSocket(req, options, (err, socket) => {
-      if (err)
+      if (err) {
+        handleSocketAfterProxy(err, req);
+        debug('call onSocket', sockLen, freeLen);
         req.onSocket(socket, err);
-      else
-        setRequestSocket(this, req, socket);
+        return;
+      }
+
+      setRequestSocket(this, req, socket);
     });
   } else {
     debug('wait for socket');
@@ -294,16 +354,23 @@ Agent.prototype.addRequest = function addRequest(req, options, port/* legacy */,
 };
 
 Agent.prototype.createSocket = function createSocket(req, options, cb) {
+  // Here the agent options will override per-request options.
   options = { __proto__: null, ...options, ...this.options };
   if (options.socketPath)
     options.path = options.socketPath;
 
   normalizeServerName(options, req);
 
+  // Make sure per-request timeout is respected.
+  const timeout = req.timeout || this.options.timeout || undefined;
+  if (timeout) {
+    options.timeout = timeout;
+  }
+
   const name = this.getName(options);
   options._agentKey = name;
 
-  debug('createConnection', name, options);
+  debug('createConnection', name);
   options.encoding = null;
 
   const oncreate = once((err, s) => {
@@ -321,8 +388,15 @@ Agent.prototype.createSocket = function createSocket(req, options, cb) {
     options.keepAlive = this.keepAlive;
     options.keepAliveInitialDelay = this.keepAliveMsecs;
   }
+
   const newSocket = this.createConnection(options, oncreate);
-  if (newSocket)
+  // In the case where we are proxying through a tunnel for HTTPS, only add
+  // the socket to the pool and install/invoke the listeners after
+  // the tunnel is successfully established, so that actual operations
+  // on the socket all go through the tunnel. Errors emitted during
+  // tunnel establishment will be handled in the createConnection method
+  // in lib/https.js.
+  if (newSocket && !newSocket[kWaitForProxyTunnel])
     oncreate(null, newSocket);
 };
 
@@ -456,10 +530,13 @@ Agent.prototype.removeSocket = function removeSocket(s, options) {
     req[kRequestOptions] = undefined;
     // If we have pending requests and a socket gets closed make a new one
     this.createSocket(req, options, (err, socket) => {
-      if (err)
-        req.onSocket(socket, err);
-      else
-        socket.emit('free');
+      if (err) {
+        handleSocketAfterProxy(err, req);
+        req.onSocket(null, err);
+        return;
+      }
+
+      socket.emit('free');
     });
   }
 
@@ -543,5 +620,8 @@ function asyncResetHandle(socket) {
 
 module.exports = {
   Agent,
-  globalAgent: new Agent({ keepAlive: true, scheduling: 'lifo', timeout: 5000 }),
+  globalAgent: new Agent({
+    keepAlive: true, scheduling: 'lifo', timeout: 5000,
+    proxyEnv: process.env.NODE_USE_ENV_PROXY ? filterEnvForProxies(process.env) : undefined,
+  }),
 };