Core: Use hpagent for proxies (#6779)

## What's the problem this PR addresses?

Proxying with the `tunnel` package is... finnicky. I did not extensively
investigate these but as a few examples of issues I encountered when I
used it in the past:

- Does not use the specified CA certificates, even for HTTPS over HTTP
proxy
- Does not use proxy after redirects

Fixes #6568 
Fixes #5244 

Hard to tell definitively without reproductions but these should be also
fixed:

Fixes #3215 
Fixes #2250 

## How did you fix it?

Use `hpagent` instead. This by itself fixes most HTTPS over HTTP
proxying issues.

I have also reused HTTPS network settings (e.g. CA certs) for connecting
to HTTPS proxies.

Side note: `pem` needs openssl installed to work which makes testing
this (and running other HTTPS-related test) difficult in some cases
(e.g. on Windows). Should we use another library for HTTPS testing?

## Checklist

<!--- Don't worry if you miss something, chores are automatically
tested. -->
<!--- This checklist exists to help you remember doing the chores when
you submit a PR. -->
<!--- Put an `x` in all the boxes that apply. -->
- [x] I have read the [Contributing
Guide](https://yarnpkg.com/advanced/contributing).

<!-- See
https://yarnpkg.com/advanced/contributing#preparing-your-pr-to-be-released
for more details. -->
<!-- Check with `yarn version check` and fix with `yarn version check
-i` -->
- [x] I have set the packages that need to be released for my changes to
be effective.

<!-- The "Testing chores" workflow validates that your PR follows our
guidelines. -->
<!-- If it doesn't pass, click on it to see details as to what your PR
might be missing. -->
- [x] I will check that all automated PR checks pass before the PR gets
reviewed.

---------

Co-authored-by: Maël Nison <[email protected]>

Author: clemyan

.pnp.cjs

@@ -0,0 +1,35 @@
+releases:
+  "@yarnpkg/cli": patch
+  "@yarnpkg/core": patch
+  "@yarnpkg/plugin-essentials": patch
+  "@yarnpkg/plugin-git": patch
+  "@yarnpkg/plugin-github": patch
+  "@yarnpkg/plugin-http": patch
+  "@yarnpkg/plugin-npm": patch
+  "@yarnpkg/plugin-npm-cli": patch
+  "@yarnpkg/plugin-typescript": patch
+
+declined:
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-exec"
+  - "@yarnpkg/plugin-file"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-jsr"
+  - "@yarnpkg/plugin-link"
+  - "@yarnpkg/plugin-nm"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-patch"
+  - "@yarnpkg/plugin-pnp"
+  - "@yarnpkg/plugin-pnpm"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-version"
+  - "@yarnpkg/plugin-workspace-tools"
+  - "@yarnpkg/builder"
+  - "@yarnpkg/doctor"
+  - "@yarnpkg/extensions"
+  - "@yarnpkg/nm"
+  - "@yarnpkg/pnpify"
+  - "@yarnpkg/sdks"

.yarn/cache/@types-tunnel-npm-0.0.0-60b0691686-bd3a79bd7b.zip

@@ -9,6 +9,7 @@ import {IncomingMessage, ServerResponse}           from 'http';
 import http                                        from 'http';
 import invariant                                   from 'invariant';
 import {AddressInfo}                               from 'net';
+import net                                         from 'net';
 import os                                          from 'os';
 import pem                                         from 'pem';
 import semver                                      from 'semver';
@@ -808,6 +809,110 @@ export const startPackageServer = ({type}: {type: keyof typeof packageServerUrls
   });
 };
 
+const proxyServerUrls: {
+  http: Promise<string> | null;
+  https: Promise<string> | null;
+} = {http: null, https: null};
+
+export const startProxyServer = ({type = `http`}: {type?: keyof typeof proxyServerUrls} = {}): Promise<string> => {
+  const serverUrl = proxyServerUrls[type];
+  if (serverUrl !== null)
+    return serverUrl;
+
+  const sendError = (res: ServerResponse, statusCode: number, errorMessage: string): void => {
+    res.writeHead(statusCode);
+    res.end(errorMessage);
+  };
+
+  return proxyServerUrls[type] = new Promise((resolve, reject) => {
+    const listener: http.RequestListener = (req, res) => {
+      void (async () => {
+        try {
+          if (!req.url) {
+            sendError(res, 400, `Missing URL in request`);
+            return;
+          }
+
+          const url = new URL(req.url);
+          const options = {
+            hostname: url.hostname,
+            port: url.port || (url.protocol === `https:` ? 443 : 80),
+            path: `${url.pathname}${url.search}`,
+            method: req.method,
+            headers: {...req.headers},
+          };
+
+          delete options.headers[`proxy-authorization`];
+          delete options.headers[`proxy-connection`];
+          options.headers.connection = `close`;
+
+          const proxyReq = (url.protocol === `https:` ? https : http).request(options, proxyRes => {
+            res.writeHead(proxyRes.statusCode || 500, proxyRes.headers);
+            proxyRes.pipe(res);
+          });
+
+          req.pipe(proxyReq);
+
+          proxyReq.on(`error`, err => {
+            sendError(res, 502, `Proxy error: ${err.message}`);
+          });
+        } catch (error) {
+          sendError(res, 500, `Proxy server error: ${error.message}`);
+        }
+      })();
+    };
+
+    (async () => {
+      let server: https.Server | http.Server;
+
+      if (type === `https`) {
+        const certs = await getHttpsCertificates();
+
+        server = https.createServer({
+          cert: certs.server.certificate,
+          key: certs.server.clientKey,
+          ca: certs.ca.certificate,
+        }, listener);
+      } else {
+        server = http.createServer(listener);
+      }
+
+      // Handle the CONNECT method using the 'connect' event
+      server.on(`connect`, (req, clientSocket, head) => {
+        // Parse the target and establish the connection
+        const [targetHost, targetPort] = (req.url || ``).split(`:`);
+        const port = parseInt(targetPort) || 443;
+
+        const serverSocket = net.connect(port, targetHost, () => {
+          clientSocket.write(
+            `HTTP/1.1 200 Connection Established\r\n` +
+            `\r\n`,
+          );
+
+          serverSocket.write(head);
+
+          serverSocket.pipe(clientSocket);
+          clientSocket.pipe(serverSocket);
+        });
+
+        serverSocket.on(`error`, () => {
+          clientSocket.end();
+        });
+        clientSocket.on(`error`, () => {
+          serverSocket.end();
+        });
+      });
+
+      // We don't want the server to prevent the process from exiting
+      server.unref();
+      server.listen(() => {
+        const {port} = server.address() as AddressInfo;
+        resolve(`${type}://localhost:${port}`);
+      });
+    })();
+  });
+};
+
 export interface PackageDriver {
   (packageJson: Record<string, any>, subDefinition: Record<string, any> | RunFunction, fn?: RunFunction): any;
   getPackageManagerName: () => string;