From 27c6a7529a29c088de59cf78c7b2c5763e0347c3 Mon Sep 17 00:00:00 2001
From: Sascha Wildner <saw@online.de>
Date: Fri, 19 Apr 2019 15:25:38 +0200
Subject: [PATCH] Sync libfetch and fetch with FreeBSD.

---
 lib/libfetch/common.c  | 305 ++++++++++++++++++++++++++++++-----------
 lib/libfetch/common.h  |  19 ++-
 lib/libfetch/fetch.3   |  63 +++++----
 lib/libfetch/fetch.c   | 111 ++++++++-------
 lib/libfetch/fetch.h   |   7 +-
 lib/libfetch/file.c    |  13 +-
 lib/libfetch/ftp.c     |  36 ++---
 lib/libfetch/http.c    | 266 ++++++++++++++++++++++-------------
 usr.bin/fetch/Makefile |   7 +-
 usr.bin/fetch/fetch.1  |  42 +++---
 usr.bin/fetch/fetch.c  | 146 ++++++++++++--------
 11 files changed, 650 insertions(+), 365 deletions(-)

diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c
index 86f480f580..16912f651e 100644
--- a/lib/libfetch/common.c
+++ b/lib/libfetch/common.c
@@ -1,5 +1,7 @@
 /*-
- * Copyright (c) 1998-2014 Dag-Erling Smørgrav
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (c) 1998-2016 Dag-Erling Smørgrav
  * Copyright (c) 2013 Michael Gmelin <freebsd@grem.de>
  * All rights reserved.
  *
@@ -25,7 +27,8 @@
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- * $FreeBSD: head/lib/libfetch/common.c 266291 2014-05-17 03:39:56Z des $
+ *
+ * $FreeBSD: head/lib/libfetch/common.c 341013 2018-11-27 10:45:14Z des $
  */
 
 #include <sys/cdefs.h>
@@ -186,9 +189,9 @@ fetch_default_port(const char *scheme)
 
 	if ((se = getservbyname(scheme, "tcp")) != NULL)
 		return (ntohs(se->s_port));
-	if (strcasecmp(scheme, SCHEME_FTP) == 0)
+	if (strcmp(scheme, SCHEME_FTP) == 0)
 		return (FTP_DEFAULT_PORT);
-	if (strcasecmp(scheme, SCHEME_HTTP) == 0)
+	if (strcmp(scheme, SCHEME_HTTP) == 0)
 		return (HTTP_DEFAULT_PORT);
 	return (0);
 }
@@ -199,9 +202,9 @@ fetch_default_port(const char *scheme)
 int
 fetch_default_proxy_port(const char *scheme)
 {
-	if (strcasecmp(scheme, SCHEME_FTP) == 0)
+	if (strcmp(scheme, SCHEME_FTP) == 0)
 		return (FTP_DEFAULT_PROXY_PORT);
-	if (strcasecmp(scheme, SCHEME_HTTP) == 0)
+	if (strcmp(scheme, SCHEME_HTTP) == 0)
 		return (HTTP_DEFAULT_PROXY_PORT);
 	return (0);
 }
@@ -239,25 +242,97 @@ fetch_ref(conn_t *conn)
 }
 
 
+/*
+ * Resolve an address
+ */
+struct addrinfo *
+fetch_resolve(const char *addr, int port, int af)
+{
+	char hbuf[256], sbuf[8];
+	struct addrinfo hints, *res;
+	const char *hb, *he, *sep;
+	const char *host, *service;
+	int err, len;
+
+	/* first, check for a bracketed IPv6 address */
+	if (*addr == '[') {
+		hb = addr + 1;
+		if ((sep = strchr(hb, ']')) == NULL) {
+			errno = EINVAL;
+			goto syserr;
+		}
+		he = sep++;
+	} else {
+		hb = addr;
+		sep = strchrnul(hb, ':');
+		he = sep;
+	}
+
+	/* see if we need to copy the host name */
+	if (*he != '\0') {
+		len = snprintf(hbuf, sizeof(hbuf),
+		    "%.*s", (int)(he - hb), hb);
+		if (len < 0)
+			goto syserr;
+		if (len >= (int)sizeof(hbuf)) {
+			errno = ENAMETOOLONG;
+			goto syserr;
+		}
+		host = hbuf;
+	} else {
+		host = hb;
+	}
+
+	/* was it followed by a service name? */
+	if (*sep == '\0' && port != 0) {
+		if (port < 1 || port > 65535) {
+			errno = EINVAL;
+			goto syserr;
+		}
+		if (snprintf(sbuf, sizeof(sbuf), "%d", port) < 0)
+			goto syserr;
+		service = sbuf;
+	} else if (*sep != '\0') {
+		service = sep + 1;
+	} else {
+		service = NULL;
+	}
+
+	/* resolve */
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = af;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_ADDRCONFIG;
+	if ((err = getaddrinfo(host, service, &hints, &res)) != 0) {
+		netdb_seterr(err);
+		return (NULL);
+	}
+	return (res);
+syserr:
+	fetch_syserr();
+	return (NULL);
+}
+
+
+
 /*
  * Bind a socket to a specific local address
  */
 int
 fetch_bind(int sd, int af, const char *addr)
 {
-	struct addrinfo hints, *res, *res0;
+	struct addrinfo *cliai, *ai;
 	int err;
 
-	memset(&hints, 0, sizeof(hints));
-	hints.ai_family = af;
-	hints.ai_socktype = SOCK_STREAM;
-	hints.ai_protocol = 0;
-	if ((err = getaddrinfo(addr, NULL, &hints, &res0)) != 0)
+	if ((cliai = fetch_resolve(addr, 0, af)) == NULL)
 		return (-1);
-	for (res = res0; res; res = res->ai_next)
-		if (bind(sd, res->ai_addr, res->ai_addrlen) == 0)
-			return (0);
-	return (-1);
+	for (ai = cliai; ai != NULL; ai = ai->ai_next)
+		if ((err = bind(sd, ai->ai_addr, ai->ai_addrlen)) == 0)
+			break;
+	if (err != 0)
+		fetch_syserr();
+	freeaddrinfo(cliai);
+	return (err == 0 ? 0 : -1);
 }
 
 
@@ -267,59 +342,76 @@ fetch_bind(int sd, int af, const char *addr)
 conn_t *
 fetch_connect(const char *host, int port, int af, int verbose)
 {
-	conn_t *conn;
-	char pbuf[10];
+	struct addrinfo *cais = NULL, *sais = NULL, *cai, *sai;
 	const char *bindaddr;
-	struct addrinfo hints, *res, *res0;
-	int sd, err;
+	conn_t *conn = NULL;
+	int err = 0, sd = -1;
 
-	DEBUG(fprintf(stderr, "---> %s:%d\n", host, port));
+	DEBUGF("---> %s:%d\n", host, port);
 
+	/* resolve server address */
 	if (verbose)
-		fetch_info("looking up %s", host);
+		fetch_info("resolving server address: %s:%d", host, port);
+	if ((sais = fetch_resolve(host, port, af)) == NULL)
+		goto fail;
 
-	/* look up host name and set up socket address structure */
-	snprintf(pbuf, sizeof(pbuf), "%d", port);
-	memset(&hints, 0, sizeof(hints));
-	hints.ai_family = af;
-	hints.ai_socktype = SOCK_STREAM;
-	hints.ai_protocol = 0;
-	if ((err = getaddrinfo(host, pbuf, &hints, &res0)) != 0) {
-		netdb_seterr(err);
-		return (NULL);
-	}
+	/* resolve client address */
 	bindaddr = getenv("FETCH_BIND_ADDRESS");
+	if (bindaddr != NULL && *bindaddr != '\0') {
+		if (verbose)
+			fetch_info("resolving client address: %s", bindaddr);
+		if ((cais = fetch_resolve(bindaddr, 0, af)) == NULL)
+			goto fail;
+	}
 
-	if (verbose)
-		fetch_info("connecting to %s:%d", host, port);
-
-	/* try to connect */
-	for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) {
-		if ((sd = socket(res->ai_family, res->ai_socktype,
-			 res->ai_protocol)) == -1)
-			continue;
-		if (bindaddr != NULL && *bindaddr != '\0' &&
-		    fetch_bind(sd, res->ai_family, bindaddr) != 0) {
-			fetch_info("failed to bind to '%s'", bindaddr);
-			close(sd);
-			continue;
+	/* try each server address in turn */
+	for (err = 0, sai = sais; sai != NULL; sai = sai->ai_next) {
+		/* open socket */
+		if ((sd = socket(sai->ai_family, SOCK_STREAM, 0)) < 0)
+			goto syserr;
+		/* attempt to bind to client address */
+		for (err = 0, cai = cais; cai != NULL; cai = cai->ai_next) {
+			if (cai->ai_family != sai->ai_family)
+				continue;
+			if ((err = bind(sd, cai->ai_addr, cai->ai_addrlen)) == 0)
+				break;
 		}
-		if (connect(sd, res->ai_addr, res->ai_addrlen) == 0 &&
-		    fcntl(sd, F_SETFL, O_NONBLOCK) == 0)
+		if (err != 0) {
+			if (verbose)
+				fetch_info("failed to bind to %s", bindaddr);
+			goto syserr;
+		}
+		/* attempt to connect to server address */
+		if ((err = connect(sd, sai->ai_addr, sai->ai_addrlen)) == 0)
 			break;
+		/* clean up before next attempt */
 		close(sd);
+		sd = -1;
 	}
-	freeaddrinfo(res0);
-	if (sd == -1) {
-		fetch_syserr();
-		return (NULL);
+	if (err != 0) {
+		if (verbose)
+			fetch_info("failed to connect to %s:%d", host, port);
+		goto syserr;
 	}
 
-	if ((conn = fetch_reopen(sd)) == NULL) {
-		fetch_syserr();
-		close(sd);
-	}
+	if ((conn = fetch_reopen(sd)) == NULL)
+		goto syserr;
+	if (cais != NULL)
+		freeaddrinfo(cais);
+	if (sais != NULL)
+		freeaddrinfo(sais);
 	return (conn);
+syserr:
+	fetch_syserr();
+	goto fail;
+fail:
+	if (sd >= 0)
+		close(sd);
+	if (cais != NULL)
+		freeaddrinfo(cais);
+	if (sais != NULL)
+		freeaddrinfo(sais);
+	return (NULL);
 }
 
 #ifdef WITH_SSL
@@ -494,7 +586,8 @@ fetch_ssl_get_numeric_addrinfo(const char *hostname, size_t len)
 	hints.ai_protocol = 0;
 	hints.ai_flags = AI_NUMERICHOST;
 	/* port is not relevant for this purpose */
-	getaddrinfo(host, "443", &hints, &res);
+	if (getaddrinfo(host, "443", &hints, &res) != 0)
+		res = NULL;
 	free(host);
 	return res;
 }
@@ -581,7 +674,11 @@ fetch_ssl_verify_altname(STACK_OF(GENERAL_NAME) *altnames,
 #else
 		name = sk_GENERAL_NAME_value(altnames, i);
 #endif
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
 		ns = (const char *)ASN1_STRING_data(name->d.ia5);
+#else
+		ns = (const char *)ASN1_STRING_get0_data(name->d.ia5);
+#endif
 		nslen = (size_t)ASN1_STRING_length(name->d.ia5);
 
 		if (name->type == GEN_DNS && ip == NULL &&
@@ -671,13 +768,15 @@ fetch_ssl_setup_transport_layer(SSL_CTX *ctx, int verbose)
 {
 	long ssl_ctx_options;
 
-	ssl_ctx_options = SSL_OP_ALL | SSL_OP_NO_TICKET;
-	if (getenv("SSL_ALLOW_SSL2") == NULL)
-		ssl_ctx_options |= SSL_OP_NO_SSLv2;
-	if (getenv("SSL_NO_SSL3") != NULL)
+	ssl_ctx_options = SSL_OP_ALL | SSL_OP_NO_SSLv2 | SSL_OP_NO_TICKET;
+	if (getenv("SSL_ALLOW_SSL3") == NULL)
 		ssl_ctx_options |= SSL_OP_NO_SSLv3;
 	if (getenv("SSL_NO_TLS1") != NULL)
 		ssl_ctx_options |= SSL_OP_NO_TLSv1;
+	if (getenv("SSL_NO_TLS1_1") != NULL)
+		ssl_ctx_options |= SSL_OP_NO_TLSv1_1;
+	if (getenv("SSL_NO_TLS1_2") != NULL)
+		ssl_ctx_options |= SSL_OP_NO_TLSv1_2;
 	if (verbose)
 		fetch_info("SSL options: %lx", ssl_ctx_options);
 	SSL_CTX_set_options(ctx, ssl_ctx_options);
@@ -701,7 +800,8 @@ fetch_ssl_setup_peer_verification(SSL_CTX *ctx, int verbose)
 		if (ca_cert_file == NULL &&
 		    access(LOCAL_CERT_FILE, R_OK) == 0)
 			ca_cert_file = LOCAL_CERT_FILE;
-		if (ca_cert_file == NULL)
+		if (ca_cert_file == NULL &&
+		    access(BASE_CERT_FILE, R_OK) == 0)
 			ca_cert_file = BASE_CERT_FILE;
 		ca_cert_path = getenv("SSL_CA_CERT_PATH");
 		if (verbose) {
@@ -712,11 +812,17 @@ fetch_ssl_setup_peer_verification(SSL_CTX *ctx, int verbose)
 			if (ca_cert_path != NULL)
 				fetch_info("Using CA cert path: %s",
 				    ca_cert_path);
+			if (ca_cert_file == NULL && ca_cert_path == NULL)
+				fetch_info("Using OpenSSL default "
+				    "CA cert file and path");
 		}
 		SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER,
 		    fetch_ssl_cb_verify_crt);
-		SSL_CTX_load_verify_locations(ctx, ca_cert_file,
-		    ca_cert_path);
+		if (ca_cert_file != NULL || ca_cert_path != NULL)
+			SSL_CTX_load_verify_locations(ctx, ca_cert_file,
+			    ca_cert_path);
+		else
+			SSL_CTX_set_default_verify_paths(ctx);
 		if ((crl_file = getenv("SSL_CRL_FILE")) != NULL) {
 			if (verbose)
 				fetch_info("Using CRL file: %s", crl_file);
@@ -872,8 +978,8 @@ fetch_ssl(conn_t *conn, const struct url *URL, int verbose)
 	}
 
 	if (verbose) {
-		fetch_info("SSL connection established using %s",
-		    SSL_get_cipher(conn->ssl));
+		fetch_info("%s connection established using %s",
+		    SSL_get_version(conn->ssl), SSL_get_cipher(conn->ssl));
 		name = X509_get_subject_name(conn->ssl_cert);
 		str = X509_NAME_oneline(name, 0, 0);
 		fetch_info("Certificate subject: %s", str);
@@ -1056,7 +1162,7 @@ fetch_getln(conn_t *conn)
 	} while (c != '\n');
 
 	conn->buf[conn->buflen] = '\0';
-	DEBUG(fprintf(stderr, "<<< %s", conn->buf));
+	DEBUGF("<<< %s", conn->buf);
 	return (0);
 }
 
@@ -1109,6 +1215,9 @@ fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt)
 			errno = 0;
 			pfd.revents = 0;
 			if (poll(&pfd, 1, deltams) < 0) {
+				/* POSIX compliance */
+				if (errno == EAGAIN)
+					continue;
 				if (errno == EINTR && fetchRestartCalls)
 					continue;
 				return (-1);
@@ -1158,7 +1267,7 @@ fetch_putln(conn_t *conn, const char *str, size_t len)
 	struct iovec iov[2];
 	int ret;
 
-	DEBUG(fprintf(stderr, ">>> %s\n", str));
+	DEBUGF(">>> %s\n", str);
 	iov[0].iov_base = __DECONST(char *, str);
 	iov[0].iov_len = len;
 	iov[1].iov_base = __DECONST(char *, ENDL);
@@ -1220,7 +1329,7 @@ fetch_add_entry(struct url_ent **p, int *size, int *len,
 	}
 
 	if (*len >= *size - 1) {
-		tmp = realloc(*p, (*size * 2 + 1) * sizeof(**p));
+		tmp = reallocarray(*p, *size * 2 + 1, sizeof(**p));
 		if (tmp == NULL) {
 			errno = ENOMEM;
 			fetch_syserr();
@@ -1253,27 +1362,23 @@ fetch_read_word(FILE *f)
 	return (word);
 }
 
-/*
- * Get authentication data for a URL from .netrc
- */
-int
-fetch_netrc_auth(struct url *url)
+static int
+fetch_netrc_open(void)
 {
+	struct passwd *pwd;
 	char fn[PATH_MAX];
-	const char *word;
-	char *p;
-	FILE *f;
+	const char *p;
+	int fd, serrno;
 
 	if ((p = getenv("NETRC")) != NULL) {
+		DEBUGF("NETRC=%s\n", p);
 		if (snprintf(fn, sizeof(fn), "%s", p) >= (int)sizeof(fn)) {
 			fetch_info("$NETRC specifies a file name "
 			    "longer than PATH_MAX");
 			return (-1);
 		}
 	} else {
-		if ((p = getenv("HOME")) != NULL) {
-			struct passwd *pwd;
-
+		if ((p = getenv("HOME")) == NULL) {
 			if ((pwd = getpwuid(getuid())) == NULL ||
 			    (p = pwd->pw_dir) == NULL)
 				return (-1);
@@ -1282,17 +1387,47 @@ fetch_netrc_auth(struct url *url)
 			return (-1);
 	}
 
-	if ((f = fopen(fn, "r")) == NULL)
+	if ((fd = open(fn, O_RDONLY)) < 0) {
+		serrno = errno;
+		DEBUGF("%s: %s\n", fn, strerror(serrno));
+		errno = serrno;
+	}
+	return (fd);
+}
+
+/*
+ * Get authentication data for a URL from .netrc
+ */
+int
+fetch_netrc_auth(struct url *url)
+{
+	const char *word;
+	int serrno;
+	FILE *f;
+
+	if (url->netrcfd < 0)
+		url->netrcfd = fetch_netrc_open();
+	if (url->netrcfd < 0)
 		return (-1);
+	if ((f = fdopen(url->netrcfd, "r")) == NULL) {
+		serrno = errno;
+		DEBUGF("fdopen(netrcfd): %s", strerror(errno));
+		close(url->netrcfd);
+		url->netrcfd = -1;
+		errno = serrno;
+		return (-1);
+	}
+	rewind(f);
+	DEBUGF("searching netrc for %s\n", url->host);
 	while ((word = fetch_read_word(f)) != NULL) {
 		if (strcmp(word, "default") == 0) {
-			DEBUG(fetch_info("Using default .netrc settings"));
+			DEBUGF("using default netrc settings\n");
 			break;
 		}
 		if (strcmp(word, "machine") == 0 &&
 		    (word = fetch_read_word(f)) != NULL &&
 		    strcasecmp(word, url->host) == 0) {
-			DEBUG(fetch_info("Using .netrc settings for %s", word));
+			DEBUGF("using netrc settings for %s\n", word);
 			break;
 		}
 	}
@@ -1324,9 +1459,13 @@ fetch_netrc_auth(struct url *url)
 		}
 	}
 	fclose(f);
+	url->netrcfd = -1;
 	return (0);
- ferr:
+ferr:
+	serrno = errno;
 	fclose(f);
+	url->netrcfd = -1;
+	errno = serrno;
 	return (-1);
 }
 
diff --git a/lib/libfetch/common.h b/lib/libfetch/common.h
index 4dc9fff9d5..f2001fab91 100644
--- a/lib/libfetch/common.h
+++ b/lib/libfetch/common.h
@@ -1,4 +1,6 @@
 /*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
  * Copyright (c) 1998-2014 Dag-Erling Smørgrav
  * All rights reserved.
  *
@@ -25,7 +27,7 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
- * $FreeBSD: head/lib/libfetch/common.h 261284 2014-01-30 08:37:23Z des $
+ * $FreeBSD: head/lib/libfetch/common.h 334317 2018-05-29 10:28:20Z des $
  */
 
 #ifndef _COMMON_H_INCLUDED
@@ -76,6 +78,7 @@ void		 fetch_syserr(void);
 void		 fetch_info(const char *, ...) __printflike(1, 2);
 int		 fetch_default_port(const char *);
 int		 fetch_default_proxy_port(const char *);
+struct addrinfo *fetch_resolve(const char *, int, int);
 int		 fetch_bind(int, int, const char *);
 conn_t		*fetch_connect(const char *, int, int, int);
 conn_t		*fetch_reopen(int);
@@ -101,9 +104,16 @@ int		 fetch_no_proxy_match(const char *);
 #define url_seterr(n)	 fetch_seterr(url_errlist, n)
 
 #ifndef NDEBUG
-#define DEBUG(x) do { if (fetchDebug) { x; } } while (0)
+#define DEBUGF(...)							\
+	do {								\
+		if (fetchDebug)						\
+			fprintf(stderr, __VA_ARGS__);			\
+	} while (0)
 #else
-#define DEBUG(x) do { } while (0)
+#define DEBUGF(...)							\
+	do {								\
+		/* nothing */						\
+	} while (0)
 #endif
 
 /*
@@ -117,6 +127,9 @@ int		 fetch_no_proxy_match(const char *);
  */
 FILE		*http_request(struct url *, const char *,
 		     struct url_stat *, struct url *, const char *);
+FILE		*http_request_body(struct url *, const char *,
+		     struct url_stat *, struct url *, const char *,
+		     const char *, const char *);
 FILE		*ftp_request(struct url *, const char *,
 		     struct url_stat *, struct url *, const char *);
 
diff --git a/lib/libfetch/fetch.3 b/lib/libfetch/fetch.3
index 135dd74666..0c6eb12847 100644
--- a/lib/libfetch/fetch.3
+++ b/lib/libfetch/fetch.3
@@ -1,6 +1,6 @@
 .\"-
 .\" Copyright (c) 1998-2013 Dag-Erling Smørgrav
-.\" Copyright (c) 2013 Michael Gmelin <freebsd@grem.de>
+.\" Copyright (c) 2013-2016 Michael Gmelin <freebsd@grem.de>
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -24,9 +24,9 @@
 .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
-.\" $FreeBSD: head/lib/libfetch/fetch.3 253805 2013-07-30 13:07:55Z des $
+.\" $FreeBSD: head/lib/libfetch/fetch.3 297355 2016-03-28 16:48:28Z trasz $
 .\"
-.Dd July 30, 2013
+.Dd April 19, 2019
 .Dt FETCH 3
 .Os
 .Sh NAME
@@ -396,8 +396,15 @@ is currently unimplemented.
 .Sh HTTPS SCHEME
 Based on HTTP SCHEME.
 By default the peer is verified using the CA bundle located in
-.Pa /etc/ssl/cert.pem .
-The file may contain multiple CA certificates.
+.Pa /usr/local/etc/ssl/cert.pem .
+If this file does not exist,
+.Pa /etc/ssl/cert.pem
+is used instead.
+If neither file exists, and
+.Ev SSL_CA_CERT_PATH
+has not been set,
+OpenSSL's default CA cert and path settings apply.
+The certificate bundle can contain multiple CA certificates.
 A common source of a current CA bundle is
 .Pa \%security/ca_root_nss .
 .Pp
@@ -428,25 +435,26 @@ Client certificate based authentication is supported.
 The environment variable
 .Ev SSL_CLIENT_CERT_FILE
 should be set to point to a file containing key and client certificate
-to be used in PEM format. In case the key is stored in a separate
-file, the environment variable
+to be used in PEM format.
+When a PEM-format key is in a separate file from the client certificate,
+the environment variable
 .Ev SSL_CLIENT_KEY_FILE
-can be set to point to the key in PEM format.
+can be set to point to the key file.
 In case the key uses a password, the user will be prompted on standard
 input (see
 .Xr PEM 3 ) .
 .Pp
 By default
 .Nm libfetch
-allows SSLv3 and TLSv1 when negotiating the connecting with the remote
+allows TLSv1 and newer when negotiating the connecting with the remote
 peer.
-You can change this behavior by setting the environment variable
-.Ev SSL_ALLOW_SSL2
-to allow SSLv2 (not recommended) and
-.Ev SSL_NO_SSL3
-or
-.Ev SSL_NO_TLS1
-to disable the respective methods.
+You can change this behavior by setting the
+.Ev SSL_ALLOW_SSL3
+environment variable to allow SSLv3 and
+.Ev SSL_NO_TLS1 ,
+.Ev SSL_NO_TLS1_1 and
+.Ev SSL_NO_TLS1_2
+to disable TLS 1.0, 1.1 and 1.2 respectively.
 .Sh AUTHENTICATION
 Apart from setting the appropriate environment variables and
 specifying the user name and password in the URL or the
@@ -531,7 +539,7 @@ Invalid URL
 .El
 .Pp
 The accompanying error message includes a protocol-specific error code
-and message, e.g.\& "File is not available (404 Not Found)"
+and message, like "File is not available (404 Not Found)"
 .Sh ENVIRONMENT
 .Bl -tag -width ".Ev FETCH_BIND_ADDRESS"
 .It Ev FETCH_BIND_ADDRESS
@@ -627,14 +635,15 @@ the document URL will be used as referrer URL.
 Specifies the User-Agent string to use for HTTP requests.
 This can be useful when working with HTTP origin or proxy servers that
 differentiate between user agents.
+If defined but empty, no User-Agent header is sent.
 .It Ev NETRC
 Specifies a file to use instead of
 .Pa ~/.netrc
-to look up login names and passwords for FTP sites.
+to look up login names and passwords for FTP and HTTP sites as well as
+HTTP proxies.
 See
 .Xr ftp 1
 for a description of the file format.
-This feature is experimental.
 .It Ev NO_PROXY
 Either a single asterisk, which disables the use of proxies
 altogether, or a comma- or whitespace-separated list of hosts for
@@ -643,12 +652,11 @@ which proxies should not be used.
 Same as
 .Ev NO_PROXY ,
 for compatibility.
-.It Ev SSL_ALLOW_SSL2
-Allow SSL version 2 when negotiating the connection (not recommended).
+.It Ev SSL_ALLOW_SSL3
+Allow SSL version 3 when negotiating the connection (not recommended).
 .It Ev SSL_CA_CERT_FILE
 CA certificate bundle containing trusted CA certificates.
-Default value:
-.Pa /etc/ssl/cert.pem .
+Default value: See HTTPS SCHEME above.
 .It Ev SSL_CA_CERT_PATH
 Path containing trusted CA hashes.
 .It Ev SSL_CLIENT_CERT_FILE
@@ -659,10 +667,12 @@ PEM encoded client key in case key and client certificate
 are stored separately.
 .It Ev SSL_CRL_FILE
 File containing certificate revocation list.
-.It Ev SSL_NO_SSL3
-Don't allow SSL version 3 when negotiating the connection.
 .It Ev SSL_NO_TLS1
-Don't allow TLV version 1 when negotiating the connection.
+Do not allow TLS version 1.0 when negotiating the connection.
+.It Ev SSL_NO_TLS1_1
+Do not allow TLS version 1.1 when negotiating the connection.
+.It Ev SSL_NO_TLS1_2
+Do not allow TLS version 1.2 when negotiating the connection.
 .It Ev SSL_NO_VERIFY_HOSTNAME
 If set, do not verify that the hostname matches the subject of the
 certificate presented by the server.
@@ -714,7 +724,6 @@ SSL_CA_CERT_FILE=/path/to/myca.pem
 .Ed
 .Sh SEE ALSO
 .Xr fetch 1 ,
-.Xr ftpio 3 ,
 .Xr ip 4
 .Rs
 .%A J. Postel
diff --git a/lib/libfetch/fetch.c b/lib/libfetch/fetch.c
index f1493a4c9d..f1605ed9bc 100644
--- a/lib/libfetch/fetch.c
+++ b/lib/libfetch/fetch.c
@@ -1,4 +1,6 @@
 /*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
  * Copyright (c) 1998-2004 Dag-Erling Smørgrav
  * All rights reserved.
  *
@@ -25,12 +27,14 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
- * $FreeBSD: head/lib/libfetch/fetch.c 252375 2013-06-29 15:51:27Z kientzle $
+ * $FreeBSD: head/lib/libfetch/fetch.c 341013 2018-11-27 10:45:14Z des $
  */
 
 #include <sys/param.h>
-#include <sys/errno.h>
 
+#include <netinet/in.h>
+
+#include <errno.h>
 #include <ctype.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -78,13 +82,13 @@ fetchXGet(struct url *URL, struct url_stat *us, const char *flags)
 		us->size = -1;
 		us->atime = us->mtime = 0;
 	}
-	if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
+	if (strcmp(URL->scheme, SCHEME_FILE) == 0)
 		return (fetchXGetFile(URL, us, flags));
-	else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
+	else if (strcmp(URL->scheme, SCHEME_FTP) == 0)
 		return (fetchXGetFTP(URL, us, flags));
-	else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
+	else if (strcmp(URL->scheme, SCHEME_HTTP) == 0)
 		return (fetchXGetHTTP(URL, us, flags));
-	else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
+	else if (strcmp(URL->scheme, SCHEME_HTTPS) == 0)
 		return (fetchXGetHTTP(URL, us, flags));
 	url_seterr(URL_BAD_SCHEME);
 	return (NULL);
@@ -108,13 +112,13 @@ FILE *
 fetchPut(struct url *URL, const char *flags)
 {
 
-	if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
+	if (strcmp(URL->scheme, SCHEME_FILE) == 0)
 		return (fetchPutFile(URL, flags));
-	else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
+	else if (strcmp(URL->scheme, SCHEME_FTP) == 0)
 		return (fetchPutFTP(URL, flags));
-	else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
+	else if (strcmp(URL->scheme, SCHEME_HTTP) == 0)
 		return (fetchPutHTTP(URL, flags));
-	else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
+	else if (strcmp(URL->scheme, SCHEME_HTTPS) == 0)
 		return (fetchPutHTTP(URL, flags));
 	url_seterr(URL_BAD_SCHEME);
 	return (NULL);
@@ -132,13 +136,13 @@ fetchStat(struct url *URL, struct url_stat *us, const char *flags)
 		us->size = -1;
 		us->atime = us->mtime = 0;
 	}
-	if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
+	if (strcmp(URL->scheme, SCHEME_FILE) == 0)
 		return (fetchStatFile(URL, us, flags));
-	else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
+	else if (strcmp(URL->scheme, SCHEME_FTP) == 0)
 		return (fetchStatFTP(URL, us, flags));
-	else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
+	else if (strcmp(URL->scheme, SCHEME_HTTP) == 0)
 		return (fetchStatHTTP(URL, us, flags));
-	else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
+	else if (strcmp(URL->scheme, SCHEME_HTTPS) == 0)
 		return (fetchStatHTTP(URL, us, flags));
 	url_seterr(URL_BAD_SCHEME);
 	return (-1);
@@ -152,13 +156,13 @@ struct url_ent *
 fetchList(struct url *URL, const char *flags)
 {
 
-	if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
+	if (strcmp(URL->scheme, SCHEME_FILE) == 0)
 		return (fetchListFile(URL, flags));
-	else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
+	else if (strcmp(URL->scheme, SCHEME_FTP) == 0)
 		return (fetchListFTP(URL, flags));
-	else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
+	else if (strcmp(URL->scheme, SCHEME_HTTP) == 0)
 		return (fetchListHTTP(URL, flags));
-	else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
+	else if (strcmp(URL->scheme, SCHEME_HTTPS) == 0)
 		return (fetchListHTTP(URL, flags));
 	url_seterr(URL_BAD_SCHEME);
 	return (NULL);
@@ -269,6 +273,7 @@ fetchMakeURL(const char *scheme, const char *host, int port, const char *doc,
 		fetch_syserr();
 		return (NULL);
 	}
+	u->netrcfd = -1;
 
 	if ((u->doc = strdup(doc ? doc : "/")) == NULL) {
 		fetch_syserr();
@@ -341,18 +346,21 @@ fetchParseURL(const char *URL)
 	char *doc;
 	const char *p, *q;
 	struct url *u;
-	int i;
+	int i, n;
 
 	/* allocate struct url */
 	if ((u = calloc(1, sizeof(*u))) == NULL) {
 		fetch_syserr();
 		return (NULL);
 	}
+	u->netrcfd = -1;
 
 	/* scheme name */
 	if ((p = strstr(URL, ":/"))) {
-		snprintf(u->scheme, URL_SCHEMELEN+1,
-		    "%.*s", (int)(p - URL), URL);
+                if (p - URL > URL_SCHEMELEN)
+                        goto ouch;
+                for (i = 0; URL + i < p; i++)
+                        u->scheme[i] = tolower((unsigned char)URL[i]);
 		URL = ++p;
 		/*
 		 * Only one slash: no host, leave slash as part of document
@@ -383,29 +391,37 @@ fetchParseURL(const char *URL)
 	}
 
 	/* hostname */
-#ifdef INET6
-	if (*p == '[' && (q = strchr(p + 1, ']')) != NULL &&
-	    (*++q == '\0' || *q == '/' || *q == ':')) {
-		if ((i = q - p - 2) > MAXHOSTNAMELEN)
-			i = MAXHOSTNAMELEN;
-		strncpy(u->host, ++p, i);
-		p = q;
-	} else
-#endif
-		for (i = 0; *p && (*p != '/') && (*p != ':'); p++)
-			if (i < MAXHOSTNAMELEN)
-				u->host[i++] = *p;
+	if (*p == '[') {
+		q = p + 1 + strspn(p + 1, ":0123456789ABCDEFabcdef");
+		if (*q++ != ']')
+			goto ouch;
+	} else {
+		/* valid characters in a DNS name */
+		q = p + strspn(p, "-." "0123456789"
+		    "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "_"
+		    "abcdefghijklmnopqrstuvwxyz");
+	}
+	if ((*q != '\0' && *q != '/' && *q != ':') || q - p > MAXHOSTNAMELEN)
+		goto ouch;
+	for (i = 0; p + i < q; i++)
+		u->host[i] = tolower((unsigned char)p[i]);
+	u->host[i] = '\0';
+	p = q;
 
 	/* port */
 	if (*p == ':') {
-		for (q = ++p; *q && (*q != '/'); q++)
-			if (isdigit((unsigned char)*q))
-				u->port = u->port * 10 + (*q - '0');
-			else {
+		for (n = 0, q = ++p; *q && (*q != '/'); q++) {
+			if (*q >= '0' && *q <= '9' && n < INT_MAX / 10) {
+				n = n * 10 + (*q - '0');
+			} else {
 				/* invalid port */
 				url_seterr(URL_BAD_PORT);
 				goto ouch;
 			}
+		}
+		if (n < 1 || n > IPPORT_MAX)
+			goto ouch;
+		u->port = n;
 		p = q;
 	}
 
@@ -414,8 +430,8 @@ fetchParseURL(const char *URL)
 	if (!*p)
 		p = "/";
 
-	if (strcasecmp(u->scheme, SCHEME_HTTP) == 0 ||
-	    strcasecmp(u->scheme, SCHEME_HTTPS) == 0) {
+	if (strcmp(u->scheme, SCHEME_HTTP) == 0 ||
+	    strcmp(u->scheme, SCHEME_HTTPS) == 0) {
 		const char hexnums[] = "0123456789abcdef";
 
 		/* percent-escape whitespace. */
@@ -440,15 +456,14 @@ fetchParseURL(const char *URL)
 		goto ouch;
 	}
 
-	DEBUG(fprintf(stderr,
-		  "scheme:   [%s]\n"
-		  "user:     [%s]\n"
-		  "password: [%s]\n"
-		  "host:     [%s]\n"
-		  "port:     [%d]\n"
-		  "document: [%s]\n",
-		  u->scheme, u->user, u->pwd,
-		  u->host, u->port, u->doc));
+	DEBUGF("scheme:   \"%s\"\n"
+	    "user:     \"%s\"\n"
+	    "password: \"%s\"\n"
+	    "host:     \"%s\"\n"
+	    "port:     \"%d\"\n"
+	    "document: \"%s\"\n",
+	    u->scheme, u->user, u->pwd,
+	    u->host, u->port, u->doc);
 
 	return (u);
 
diff --git a/lib/libfetch/fetch.h b/lib/libfetch/fetch.h
index 64669967a0..da35ba062e 100644
--- a/lib/libfetch/fetch.h
+++ b/lib/libfetch/fetch.h
@@ -1,4 +1,6 @@
 /*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
  * Copyright (c) 1998-2004 Dag-Erling Smørgrav
  * All rights reserved.
  *
@@ -25,7 +27,7 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
- * $FreeBSD: head/lib/libfetch/fetch.h 226537 2011-10-19 11:43:51Z des $
+ * $FreeBSD: head/lib/libfetch/fetch.h 326219 2017-11-26 02:00:33Z pfg $
  */
 
 #ifndef _FETCH_H_INCLUDED
@@ -47,6 +49,7 @@ struct url {
 	off_t		 offset;
 	size_t		 length;
 	time_t		 ims_time;
+	int		 netrcfd;
 };
 
 struct url_stat {
@@ -102,6 +105,8 @@ FILE		*fetchGetHTTP(struct url *, const char *);
 FILE		*fetchPutHTTP(struct url *, const char *);
 int		 fetchStatHTTP(struct url *, struct url_stat *, const char *);
 struct url_ent	*fetchListHTTP(struct url *, const char *);
+FILE		*fetchReqHTTP(struct url *, const char *, const char *,
+		    const char *, const char *);
 
 /* FTP-specific functions */
 FILE		*fetchXGetFTP(struct url *, struct url_stat *, const char *);
diff --git a/lib/libfetch/file.c b/lib/libfetch/file.c
index 7105b2c37e..43a848ebb1 100644
--- a/lib/libfetch/file.c
+++ b/lib/libfetch/file.c
@@ -1,4 +1,6 @@
 /*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
  * Copyright (c) 1998-2011 Dag-Erling Smørgrav
  * All rights reserved.
  *
@@ -25,7 +27,7 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
- * $FreeBSD: head/lib/libfetch/file.c 240495 2012-09-14 12:15:13Z eadler $
+ * $FreeBSD: head/lib/libfetch/file.c 326219 2017-11-26 02:00:33Z pfg $
  */
 
 #include <sys/param.h>
@@ -47,7 +49,7 @@ fetchXGetFile(struct url *u, struct url_stat *us, const char *flags)
 	if (us && fetchStatFile(u, us, flags) == -1)
 		return (NULL);
 
-	f = fopen(u->doc, "r");
+	f = fopen(u->doc, "re");
 
 	if (f == NULL) {
 		fetch_syserr();
@@ -60,7 +62,6 @@ fetchXGetFile(struct url *u, struct url_stat *us, const char *flags)
 		return (NULL);
 	}
 
-	fcntl(fileno(f), F_SETFD, FD_CLOEXEC);
 	return (f);
 }
 
@@ -76,9 +77,9 @@ fetchPutFile(struct url *u, const char *flags)
 	FILE *f;
 
 	if (CHECK_FLAG('a'))
-		f = fopen(u->doc, "a");
+		f = fopen(u->doc, "ae");
 	else
-		f = fopen(u->doc, "w+");
+		f = fopen(u->doc, "w+e");
 
 	if (f == NULL) {
 		fetch_syserr();
@@ -91,7 +92,6 @@ fetchPutFile(struct url *u, const char *flags)
 		return (NULL);
 	}
 
-	fcntl(fileno(f), F_SETFD, FD_CLOEXEC);
 	return (f);
 }
 
@@ -150,5 +150,6 @@ fetchListFile(struct url *u, const char *flags __unused)
 		fetch_add_entry(&ue, &size, &len, de->d_name, &us);
 	}
 
+	closedir(dir);
 	return (ue);
 }
diff --git a/lib/libfetch/ftp.c b/lib/libfetch/ftp.c
index 425ca289b4..c91c1c6b9a 100644
--- a/lib/libfetch/ftp.c
+++ b/lib/libfetch/ftp.c
@@ -1,4 +1,6 @@
 /*-
+ * SPDX-License-Identifier: (BSD-3-Clause AND Beerware)
+ *
  * Copyright (c) 1998-2011 Dag-Erling Smørgrav
  * All rights reserved.
  *
@@ -25,7 +27,7 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
- * $FreeBSD: head/lib/libfetch/ftp.c 226537 2011-10-19 11:43:51Z des $
+ * $FreeBSD: head/lib/libfetch/ftp.c 341013 2018-11-27 10:45:14Z des $
  */
 
 /*
@@ -257,7 +259,7 @@ ftp_pwd(conn_t *conn, char *pwd, size_t pwdlen)
 		return (FTP_PROTOCOL_ERROR);
 	*dst = '\0';
 #if 0
-	DEBUG(fprintf(stderr, "pwd: [%s]\n", pwd));
+	DEBUGF("pwd: [%s]\n", pwd);
 #endif
 	return (FTP_OK);
 }
@@ -289,8 +291,8 @@ ftp_cwd(conn_t *conn, const char *file)
 			if (pwd[i] != file[i])
 				break;
 #if 0
-		DEBUG(fprintf(stderr, "have: [%.*s|%s]\n", i, pwd, pwd + i));
-		DEBUG(fprintf(stderr, "want: [%.*s|%s]\n", i, file, file + i));
+		DEBUGF("have: [%.*s|%s]\n", i, pwd, pwd + i);
+		DEBUGF("want: [%.*s|%s]\n", i, file, file + i);
 #endif
 		/* Keep going up a dir until we have a matching prefix. */
 		if (pwd[i] == '\0' && (file[i - 1] == '/' || file[i] == '/'))
@@ -432,7 +434,7 @@ ftp_stat(conn_t *conn, const char *file, struct url_stat *us)
 	}
 	if (us->size == 0)
 		us->size = -1;
-	DEBUG(fprintf(stderr, "size: [%lld]\n", (long long)us->size));
+	DEBUGF("size: [%lld]\n", (long long)us->size);
 
 	e = ftp_cmd(conn, "MDTM %.*s", filenamelen, filename);
 	if (e != FTP_FILE_STATUS) {
@@ -467,10 +469,9 @@ ftp_stat(conn_t *conn, const char *file, struct url_stat *us)
 		t = time(NULL);
 	us->mtime = t;
 	us->atime = t;
-	DEBUG(fprintf(stderr,
-	    "last modified: [%04d-%02d-%02d %02d:%02d:%02d]\n",
+	DEBUGF("last modified: [%04d-%02d-%02d %02d:%02d:%02d]\n",
 	    tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
-	    tm.tm_hour, tm.tm_min, tm.tm_sec));
+	    tm.tm_hour, tm.tm_min, tm.tm_sec);
 	return (0);
 }
 
@@ -584,7 +585,7 @@ ftp_closefn(void *v)
 	fetch_close(io->dconn);
 	io->dir = -1;
 	io->dconn = NULL;
-	DEBUG(fprintf(stderr, "Waiting for final status\n"));
+	DEBUGF("Waiting for final status\n");
 	r = ftp_chkerr(io->cconn);
 	if (io->cconn == cached_connection && io->cconn->ref == 1)
 		cached_connection = NULL;
@@ -769,8 +770,8 @@ ftp_transfer(conn_t *conn, const char *oper, const char *file,
 			fetch_info("opening data connection");
 		bindaddr = getenv("FETCH_BIND_ADDRESS");
 		if (bindaddr != NULL && *bindaddr != '\0' &&
-		    fetch_bind(sd, sa.ss_family, bindaddr) != 0)
-			goto sysouch;
+		    (e = fetch_bind(sd, sa.ss_family, bindaddr)) != 0)
+			goto ouch;
 		if (connect(sd, (struct sockaddr *)&sa, sa.ss_len) == -1)
 			goto sysouch;
 
@@ -914,7 +915,8 @@ ftp_authenticate(conn_t *conn, struct url *url, struct url *purl)
 		fetch_netrc_auth(url);
 	user = url->user;
 	if (*user == '\0')
-		user = getenv("FTP_LOGIN");
+		if ((user = getenv("FTP_LOGIN")) != NULL)
+			DEBUGF("FTP_LOGIN=%s\n", user);
 	if (user == NULL || *user == '\0')
 		user = FTP_ANONYMOUS_USER;
 	if (purl && url->port == fetch_default_port(url->scheme))
@@ -928,7 +930,8 @@ ftp_authenticate(conn_t *conn, struct url *url, struct url *purl)
 	if (e == FTP_NEED_PASSWORD) {
 		pwd = url->pwd;
 		if (*pwd == '\0')
-			pwd = getenv("FTP_PASSWORD");
+			if ((pwd = getenv("FTP_PASSWORD")) != NULL)
+				DEBUGF("FTP_PASSWORD=%s\n", pwd);
 		if (pwd == NULL || *pwd == '\0') {
 			if ((logname = getlogin()) == NULL)
 				logname = FTP_ANONYMOUS_USER;
@@ -1083,8 +1086,8 @@ ftp_get_proxy(struct url * url, const char *flags)
 		}
 		if (!purl->port)
 			purl->port = fetch_default_proxy_port(purl->scheme);
-		if (strcasecmp(purl->scheme, SCHEME_FTP) == 0 ||
-		    strcasecmp(purl->scheme, SCHEME_HTTP) == 0)
+		if (strcmp(purl->scheme, SCHEME_FTP) == 0 ||
+		    strcmp(purl->scheme, SCHEME_HTTP) == 0)
 			return (purl);
 		fetchFreeURL(purl);
 	}
@@ -1102,7 +1105,8 @@ ftp_request(struct url *url, const char *op, struct url_stat *us,
 	int oflag;
 
 	/* check if we should use HTTP instead */
-	if (purl && strcasecmp(purl->scheme, SCHEME_HTTP) == 0) {
+	if (purl && (strcmp(purl->scheme, SCHEME_HTTP) == 0 ||
+	    strcmp(purl->scheme, SCHEME_HTTPS) == 0)) {
 		if (strcmp(op, "STAT") == 0)
 			return (http_request(url, "HEAD", us, purl, flags));
 		else if (strcmp(op, "RETR") == 0)
diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c
index b48d895060..f05db55f9d 100644
--- a/lib/libfetch/http.c
+++ b/lib/libfetch/http.c
@@ -1,4 +1,6 @@
 /*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
  * Copyright (c) 2000-2014 Dag-Erling Smørgrav
  * All rights reserved.
  *
@@ -25,7 +27,7 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
- * $FreeBSD: head/lib/libfetch/http.c 261284 2014-01-30 08:37:23Z des $
+ * $FreeBSD: head/lib/libfetch/http.c 341072 2018-11-27 16:23:17Z des $
  */
 
 /*
@@ -113,10 +115,11 @@
 #define HTTP_REDIRECT(xyz) ((xyz) == HTTP_MOVED_PERM \
 			    || (xyz) == HTTP_MOVED_TEMP \
 			    || (xyz) == HTTP_TEMP_REDIRECT \
+			    || (xyz) == HTTP_PERM_REDIRECT \
 			    || (xyz) == HTTP_USE_PROXY \
 			    || (xyz) == HTTP_SEE_OTHER)
 
-#define HTTP_ERROR(xyz) ((xyz) > 400 && (xyz) < 599)
+#define HTTP_ERROR(xyz) ((xyz) >= 400 && (xyz) <= 599)
 
 
 /*****************************************************************************
@@ -129,8 +132,8 @@ struct httpio
 	int		 chunked;	/* chunked mode */
 	char		*buf;		/* chunk buffer */
 	size_t		 bufsize;	/* size of chunk buffer */
-	ssize_t		 buflen;	/* amount of data currently in buffer */
-	int		 bufpos;	/* current read offset in buffer */
+	size_t		 buflen;	/* amount of data currently in buffer */
+	size_t		 bufpos;	/* current read offset in buffer */
 	int		 eof;		/* end-of-file flag */
 	int		 error;		/* error flag */
 	size_t		 chunksize;	/* remaining size of current chunk */
@@ -214,6 +217,7 @@ http_fillbuf(struct httpio *io, size_t len)
 	if (io->eof)
 		return (0);
 
+	/* not chunked: just fetch the requested amount */
 	if (io->chunked == 0) {
 		if (http_growbuf(io, len) == -1)
 			return (-1);
@@ -226,6 +230,7 @@ http_fillbuf(struct httpio *io, size_t len)
 		return (io->buflen);
 	}
 
+	/* chunked, but we ran out: get the next chunk header */
 	if (io->chunksize == 0) {
 		switch (http_new_chunk(io)) {
 		case -1:
@@ -237,6 +242,7 @@ http_fillbuf(struct httpio *io, size_t len)
 		}
 	}
 
+	/* fetch the requested amount, but no more than the current chunk */
 	if (len > io->chunksize)
 		len = io->chunksize;
 	if (http_growbuf(io, len) == -1)
@@ -245,8 +251,9 @@ http_fillbuf(struct httpio *io, size_t len)
 		io->error = errno;
 		return (-1);
 	}
+	io->bufpos = 0;
 	io->buflen = nbytes;
-	io->chunksize -= io->buflen;
+	io->chunksize -= nbytes;
 
 	if (io->chunksize == 0) {
 		if (fetch_read(io->conn, &ch, 1) != 1 || ch != '\r' ||
@@ -254,8 +261,6 @@ http_fillbuf(struct httpio *io, size_t len)
 			return (-1);
 	}
 
-	io->bufpos = 0;
-
 	return (io->buflen);
 }
 
@@ -774,9 +779,9 @@ http_parse_authenticate(const char *cp, http_auth_challenges_t *cs)
 			goto out;
 		}
 		init_http_auth_challenge(cs->challenges[cs->count]);
-		if (!strcasecmp(key, "basic")) {
+		if (strcasecmp(key, "basic") == 0) {
 			cs->challenges[cs->count]->scheme = HTTPAS_BASIC;
-		} else if (!strcasecmp(key, "digest")) {
+		} else if (strcasecmp(key, "digest") == 0) {
 			cs->challenges[cs->count]->scheme = HTTPAS_DIGEST;
 		} else {
 			cs->challenges[cs->count]->scheme = HTTPAS_UNKNOWN;
@@ -805,25 +810,27 @@ http_parse_authenticate(const char *cp, http_auth_challenges_t *cs)
 			if (lex != HTTPHL_WORD && lex != HTTPHL_STRING)
 				goto out;
 
-			if (!strcasecmp(key, "realm"))
+			if (strcasecmp(key, "realm") == 0) {
 				cs->challenges[cs->count]->realm =
-					strdup(value);
-			else if (!strcasecmp(key, "qop"))
+				    strdup(value);
+			} else if (strcasecmp(key, "qop") == 0) {
 				cs->challenges[cs->count]->qop =
-					strdup(value);
-			else if (!strcasecmp(key, "nonce"))
+				    strdup(value);
+			} else if (strcasecmp(key, "nonce") == 0) {
 				cs->challenges[cs->count]->nonce =
-					strdup(value);
-			else if (!strcasecmp(key, "opaque"))
+				    strdup(value);
+			} else if (strcasecmp(key, "opaque") == 0) {
 				cs->challenges[cs->count]->opaque =
-					strdup(value);
-			else if (!strcasecmp(key, "algorithm"))
+				    strdup(value);
+			} else if (strcasecmp(key, "algorithm") == 0) {
 				cs->challenges[cs->count]->algo =
-					strdup(value);
-			else if (!strcasecmp(key, "stale"))
+				    strdup(value);
+			} else if (strcasecmp(key, "stale") == 0) {
 				cs->challenges[cs->count]->stale =
-					strcasecmp(value, "no");
-			/* Else ignore unknown attributes */
+				    strcasecmp(value, "no");
+			} else {
+				/* ignore unknown attributes */
+			}
 
 			/* Comma or Next challenge or End */
 			lex = http_header_lex(&cp, key);
@@ -872,17 +879,22 @@ http_parse_mtime(const char *p, time_t *mtime)
 	char locale[64], *r;
 	struct tm tm;
 
-	strncpy(locale, setlocale(LC_TIME, NULL), sizeof(locale));
+	strlcpy(locale, setlocale(LC_TIME, NULL), sizeof(locale));
 	setlocale(LC_TIME, "C");
 	r = strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm);
+	/*
+	 * Some proxies use UTC in response, but it should still be
+	 * parsed. RFC2616 states GMT and UTC are exactly equal for HTTP.
+	 */
+	if (r == NULL)
+		r = strptime(p, "%a, %d %b %Y %H:%M:%S UTC", &tm);
 	/* XXX should add support for date-2 and date-3 */
 	setlocale(LC_TIME, locale);
 	if (r == NULL)
 		return (-1);
-	DEBUG(fprintf(stderr, "last modified: [%04d-%02d-%02d "
-		  "%02d:%02d:%02d]\n",
-		  tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
-		  tm.tm_hour, tm.tm_min, tm.tm_sec));
+	DEBUGF("last modified: [%04d-%02d-%02d %02d:%02d:%02d]\n",
+	    tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
+	    tm.tm_hour, tm.tm_min, tm.tm_sec);
 	*mtime = timegm(&tm);
 	return (0);
 }
@@ -899,8 +911,7 @@ http_parse_length(const char *p, off_t *length)
 		len = len * 10 + (*p - '0');
 	if (*p)
 		return (-1);
-	DEBUG(fprintf(stderr, "content length: [%lld]\n",
-	    (long long)len));
+	DEBUGF("content length: [%lld]\n", (long long)len);
 	*length = len;
 	return (0);
 }
@@ -934,12 +945,11 @@ http_parse_range(const char *p, off_t *offset, off_t *length, off_t *size)
 	if (*p || len < last - first + 1)
 		return (-1);
 	if (first == -1) {
-		DEBUG(fprintf(stderr, "content range: [*/%lld]\n",
-		    (long long)len));
+		DEBUGF("content range: [*/%lld]\n", (long long)len);
 		*length = 0;
 	} else {
-		DEBUG(fprintf(stderr, "content range: [%lld-%lld/%lld]\n",
-		    (long long)first, (long long)last, (long long)len));
+		DEBUGF("content range: [%lld-%lld/%lld]\n",
+		    (long long)first, (long long)last, (long long)len);
 		*length = last - first + 1;
 	}
 	*offset = first;
@@ -1175,9 +1185,10 @@ DigestCalcResponse(
 	OUT HASHHEX Response      /* request-digest or response-digest */
 	)
 {
-/*	DEBUG(fprintf(stderr,
-		      "Calc: HA1[%s] Nonce[%s] qop[%s] method[%s] URI[%s]\n",
-		      HA1, pszNonce, pszQop, pszMethod, pszDigestUri));*/
+#if 0
+	DEBUGF("Calc: HA1[%s] Nonce[%s] qop[%s] method[%s] URI[%s]\n",
+	    HA1, pszNonce, pszQop, pszMethod, pszDigestUri);
+#endif
 	MD5_CTX Md5Ctx;
 	HASH HA2;
 	HASH RespHash;
@@ -1245,15 +1256,15 @@ http_digest_auth(conn_t *conn, const char *hdr, http_auth_challenge_t *c,
 	char *options = NULL;
 
 	if (!c->realm || !c->nonce) {
-		DEBUG(fprintf(stderr, "realm/nonce not set in challenge\n"));
+		DEBUGF("realm/nonce not set in challenge\n");
 		return(-1);
 	}
 	if (!c->algo)
 		c->algo = strdup("");
 
 	if (asprintf(&options, "%s%s%s%s",
-		     *c->algo? ",algorithm=" : "", c->algo,
-		     c->opaque? ",opaque=" : "", c->opaque?c->opaque:"")== -1)
+	    *c->algo? ",algorithm=" : "", c->algo,
+	    c->opaque? ",opaque=" : "", c->opaque?c->opaque:"") < 0)
 		return (-1);
 
 	if (!c->qop) {
@@ -1270,7 +1281,7 @@ http_digest_auth(conn_t *conn, const char *hdr, http_auth_challenge_t *c,
 	HASHHEX HA1;
 	DigestCalcHA1(c->algo, parms->user, c->realm,
 		      parms->password, c->nonce, cnonce, HA1);
-	DEBUG(fprintf(stderr, "HA1: [%s]\n", HA1));
+	DEBUGF("HA1: [%s]\n", HA1);
 	HASHHEX digest;
 	DigestCalcResponse(HA1, c->nonce, noncecount, cnonce, c->qop,
 			   "GET", url->doc, "", digest);
@@ -1302,8 +1313,8 @@ http_basic_auth(conn_t *conn, const char *hdr, const char *usr, const char *pwd)
 	char *upw, *auth;
 	int r;
 
-	DEBUG(fprintf(stderr, "basic: usr: [%s]\n", usr));
-	DEBUG(fprintf(stderr, "basic: pwd: [%s]\n", pwd));
+	DEBUGF("basic: usr: [%s]\n", usr);
+	DEBUGF("basic: pwd: [%s]\n", pwd);
 	if (asprintf(&upw, "%s:%s", usr, pwd) == -1)
 		return (-1);
 	auth = http_base64(upw);
@@ -1328,7 +1339,7 @@ http_authorize(conn_t *conn, const char *hdr, http_auth_challenges_t *cs,
 
 	/* If user or pass are null we're not happy */
 	if (!parms->user || !parms->password) {
-		DEBUG(fprintf(stderr, "NULL usr or pass\n"));
+		DEBUGF("NULL usr or pass\n");
 		return (-1);
 	}
 
@@ -1339,10 +1350,9 @@ http_authorize(conn_t *conn, const char *hdr, http_auth_challenges_t *cs,
 	}
 
 	/* Error if "Digest" was specified and there is no Digest challenge */
-	if (!digest && (parms->scheme &&
-			!strcasecmp(parms->scheme, "digest"))) {
-		DEBUG(fprintf(stderr,
-			      "Digest auth in env, not supported by peer\n"));
+	if (!digest &&
+	    (parms->scheme && strcasecmp(parms->scheme, "digest") == 0)) {
+		DEBUGF("Digest auth in env, not supported by peer\n");
 		return (-1);
 	}
 	/*
@@ -1350,7 +1360,8 @@ http_authorize(conn_t *conn, const char *hdr, http_auth_challenges_t *cs,
 	 * challenge, do the basic thing. Don't need a challenge for this,
 	 * so no need to check basic!=NULL
 	 */
-	if (!digest || (parms->scheme && !strcasecmp(parms->scheme,"basic")))
+	if (!digest ||
+	    (parms->scheme && strcasecmp(parms->scheme, "basic") == 0))
 		return (http_basic_auth(conn,hdr,parms->user,parms->password));
 
 	/* Else, prefer digest. We just checked that it's not NULL */
@@ -1369,8 +1380,12 @@ http_connect(struct url *URL, struct url *purl, const char *flags)
 {
 	struct url *curl;
 	conn_t *conn;
+	hdr_t h;
+	http_headerbuf_t headerbuf;
+	const char *p;
 	int verbose;
 	int af, val;
+	int serrno;
 
 #ifdef INET6
 	af = AF_UNSPEC;
@@ -1391,31 +1406,54 @@ http_connect(struct url *URL, struct url *purl, const char *flags)
 	if ((conn = fetch_connect(curl->host, curl->port, af, verbose)) == NULL)
 		/* fetch_connect() has already set an error code */
 		return (NULL);
-	if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && purl) {
+	init_http_headerbuf(&headerbuf);
+	if (strcmp(URL->scheme, SCHEME_HTTPS) == 0 && purl) {
 		http_cmd(conn, "CONNECT %s:%d HTTP/1.1",
 		    URL->host, URL->port);
 		http_cmd(conn, "Host: %s:%d",
 		    URL->host, URL->port);
 		http_cmd(conn, "");
 		if (http_get_reply(conn) != HTTP_OK) {
-			fetch_close(conn);
-			return (NULL);
+			http_seterr(conn->err);
+			goto ouch;
+		}
+		/* Read and discard the rest of the proxy response */
+		if (fetch_getln(conn) < 0) {
+			fetch_syserr();
+			goto ouch;
 		}
-		http_get_reply(conn);
+		do {
+			switch ((h = http_next_header(conn, &headerbuf, &p))) {
+			case hdr_syserror:
+				fetch_syserr();
+				goto ouch;
+			case hdr_error:
+				http_seterr(HTTP_PROTOCOL_ERROR);
+				goto ouch;
+			default:
+				/* ignore */ ;
+			}
+		} while (h > hdr_end);
 	}
-	if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 &&
+	if (strcmp(URL->scheme, SCHEME_HTTPS) == 0 &&
 	    fetch_ssl(conn, URL, verbose) == -1) {
-		fetch_close(conn);
 		/* grrr */
 		errno = EAUTH;
 		fetch_syserr();
-		return (NULL);
+		goto ouch;
 	}
 
 	val = 1;
 	setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val, sizeof(val));
 
+	clean_http_headerbuf(&headerbuf);
 	return (conn);
+ouch:
+	serrno = errno;
+	clean_http_headerbuf(&headerbuf);
+	fetch_close(conn);
+	errno = serrno;
+	return (NULL);
 }
 
 static struct url *
@@ -1434,7 +1472,7 @@ http_get_proxy(struct url * url, const char *flags)
 			strcpy(purl->scheme, SCHEME_HTTP);
 		if (!purl->port)
 			purl->port = fetch_default_proxy_port(purl->scheme);
-		if (strcasecmp(purl->scheme, SCHEME_HTTP) == 0)
+		if (strcmp(purl->scheme, SCHEME_HTTP) == 0)
 			return (purl);
 		fetchFreeURL(purl);
 	}
@@ -1484,6 +1522,14 @@ http_print_html(FILE *out, FILE *in)
  * Core
  */
 
+FILE *
+http_request(struct url *URL, const char *op, struct url_stat *us,
+	struct url *purl, const char *flags)
+{
+
+	return (http_request_body(URL, op, us, purl, flags, NULL, NULL));
+}
+
 /*
  * Send a request and process the reply
  *
@@ -1491,8 +1537,9 @@ http_print_html(FILE *out, FILE *in)
  * XXX off into a separate function.
  */
 FILE *
-http_request(struct url *URL, const char *op, struct url_stat *us,
-	struct url *purl, const char *flags)
+http_request_body(struct url *URL, const char *op, struct url_stat *us,
+	struct url *purl, const char *flags, const char *content_type,
+	const char *body)
 {
 	char timebuf[80];
 	char hbuf[MAXHOSTNAMELEN + 7], *host;
@@ -1509,6 +1556,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 	http_headerbuf_t headerbuf;
 	http_auth_challenges_t server_challenges;
 	http_auth_challenges_t proxy_challenges;
+	size_t body_len;
 
 	/* The following calls don't allocate anything */
 	init_http_headerbuf(&headerbuf);
@@ -1557,27 +1605,18 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 		if ((conn = http_connect(url, purl, flags)) == NULL)
 			goto ouch;
 
+		/* append port number only if necessary */
 		host = url->host;
-#ifdef INET6
-		if (strchr(url->host, ':')) {
-			snprintf(hbuf, sizeof(hbuf), "[%s]", url->host);
-			host = hbuf;
-		}
-#endif
 		if (url->port != fetch_default_port(url->scheme)) {
-			if (host != hbuf) {
-				strcpy(hbuf, host);
-				host = hbuf;
-			}
-			snprintf(hbuf + strlen(hbuf),
-			    sizeof(hbuf) - strlen(hbuf), ":%d", url->port);
+			snprintf(hbuf, sizeof(hbuf), "%s:%d", host, url->port);
+			host = hbuf;
 		}
 
 		/* send request */
 		if (verbose)
 			fetch_info("requesting %s://%s%s",
 			    url->scheme, host, url->doc);
-		if (purl && strcasecmp(URL->scheme, SCHEME_HTTPS) != 0) {
+		if (purl && strcmp(url->scheme, SCHEME_HTTPS) != 0) {
 			http_cmd(conn, "%s %s://%s%s HTTP/1.1",
 			    op, url->scheme, host, url->doc);
 		} else {
@@ -1605,16 +1644,17 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 			http_auth_params_t aparams;
 			init_http_auth_params(&aparams);
 			if (*purl->user || *purl->pwd) {
-				aparams.user = purl->user ?
-					strdup(purl->user) : strdup("");
-				aparams.password = purl->pwd?
-					strdup(purl->pwd) : strdup("");
+				aparams.user = strdup(purl->user);
+				aparams.password = strdup(purl->pwd);
 			} else if ((p = getenv("HTTP_PROXY_AUTH")) != NULL &&
 				   *p != '\0') {
 				if (http_authfromenv(p, &aparams) < 0) {
 					http_seterr(HTTP_NEED_PROXY_AUTH);
 					goto ouch;
 				}
+			} else if (fetch_netrc_auth(purl) == 0) {
+				aparams.user = strdup(purl->user);
+				aparams.password = strdup(purl->pwd);
 			}
 			http_authorize(conn, "Proxy-Authorization",
 				       &proxy_challenges, &aparams, url);
@@ -1634,22 +1674,21 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 			http_auth_params_t aparams;
 			init_http_auth_params(&aparams);
 			if (*url->user || *url->pwd) {
-				aparams.user = url->user ?
-					strdup(url->user) : strdup("");
-				aparams.password = url->pwd ?
-					strdup(url->pwd) : strdup("");
+				aparams.user = strdup(url->user);
+				aparams.password = strdup(url->pwd);
 			} else if ((p = getenv("HTTP_AUTH")) != NULL &&
 				   *p != '\0') {
 				if (http_authfromenv(p, &aparams) < 0) {
 					http_seterr(HTTP_NEED_AUTH);
 					goto ouch;
 				}
+			} else if (fetch_netrc_auth(url) == 0) {
+				aparams.user = strdup(url->user);
+				aparams.password = strdup(url->pwd);
 			} else if (fetchAuthMethod &&
 				   fetchAuthMethod(url) == 0) {
-				aparams.user = url->user ?
-					strdup(url->user) : strdup("");
-				aparams.password = url->pwd ?
-					strdup(url->pwd) : strdup("");
+				aparams.user = strdup(url->user);
+				aparams.password = strdup(url->pwd);
 			} else {
 				http_seterr(HTTP_NEED_AUTH);
 				goto ouch;
@@ -1673,15 +1712,31 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 			else
 				http_cmd(conn, "Referer: %s", p);
 		}
-		if ((p = getenv("HTTP_USER_AGENT")) != NULL && *p != '\0')
-			http_cmd(conn, "User-Agent: %s", p);
-		else
-			http_cmd(conn, "User-Agent: %s " _LIBFETCH_VER, getprogname());
+		if ((p = getenv("HTTP_USER_AGENT")) != NULL) {
+			/* no User-Agent if defined but empty */
+			if  (*p != '\0')
+				http_cmd(conn, "User-Agent: %s", p);
+		} else {
+			/* default User-Agent */
+			http_cmd(conn, "User-Agent: %s " _LIBFETCH_VER,
+			    getprogname());
+		}
 		if (url->offset > 0)
 			http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset);
 		http_cmd(conn, "Connection: close");
+
+		if (body) {
+			body_len = strlen(body);
+			http_cmd(conn, "Content-Length: %zu", body_len);
+			if (content_type != NULL)
+				http_cmd(conn, "Content-Type: %s", content_type);
+		}
+
 		http_cmd(conn, "");
 
+		if (body)
+			fetch_write(conn, body, body_len);
+
 		/*
 		 * Force the queued request to be dispatched.  Normally, one
 		 * would do this with shutdown(2) but squid proxies can be
@@ -1705,6 +1760,8 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 			break;
 		case HTTP_MOVED_PERM:
 		case HTTP_MOVED_TEMP:
+		case HTTP_TEMP_REDIRECT:
+		case HTTP_PERM_REDIRECT:
 		case HTTP_SEE_OTHER:
 		case HTTP_USE_PROXY:
 			/*
@@ -1796,26 +1853,29 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 				if (new)
 					free(new);
 				if (verbose)
-					fetch_info("%d redirect to %s", conn->err, p);
+					fetch_info("%d redirect to %s",
+					    conn->err, p);
 				if (*p == '/')
 					/* absolute path */
-					new = fetchMakeURL(url->scheme, url->host, url->port, p,
-					    url->user, url->pwd);
+					new = fetchMakeURL(url->scheme, url->host,
+					    url->port, p, url->user, url->pwd);
 				else
 					new = fetchParseURL(p);
 				if (new == NULL) {
 					/* XXX should set an error code */
-					DEBUG(fprintf(stderr, "failed to parse new URL\n"));
+					DEBUGF("failed to parse new URL\n");
 					goto ouch;
 				}
 
 				/* Only copy credentials if the host matches */
-				if (!strcmp(new->host, url->host) && !*new->user && !*new->pwd) {
+				if (strcmp(new->host, url->host) == 0 &&
+				    !*new->user && !*new->pwd) {
 					strcpy(new->user, url->user);
 					strcpy(new->pwd, url->pwd);
 				}
 				new->offset = url->offset;
 				new->length = url->length;
+				new->ims_time = url->ims_time;
 				break;
 			case hdr_transfer_encoding:
 				/* XXX weak test*/
@@ -1850,7 +1910,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 			    (conn->err == HTTP_NEED_PROXY_AUTH &&
 			     !proxy_challenges.valid)) {
 				/* 401/7 but no www/proxy-authenticate ?? */
-				DEBUG(fprintf(stderr, "401/7 and no auth header\n"));
+				DEBUGF("%03d without auth header\n", conn->err);
 				goto ouch;
 			}
 			fetch_close(conn);
@@ -1860,7 +1920,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 
 		/* requested range not satisfiable */
 		if (conn->err == HTTP_BAD_RANGE) {
-			if (url->offset == size && url->length == 0) {
+			if (url->offset > 0 && url->length == 0) {
 				/* asked for 0 bytes; fake it */
 				offset = url->offset;
 				clength = -1;
@@ -1885,7 +1945,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 		fetch_close(conn);
 		conn = NULL;
 		if (!new) {
-			DEBUG(fprintf(stderr, "redirect with no new location\n"));
+			DEBUGF("redirect with no new location\n");
 			break;
 		}
 		if (url != URL)
@@ -1899,10 +1959,9 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 		goto ouch;
 	}
 
-	DEBUG(fprintf(stderr, "offset %lld, length %lld,"
-		  " size %lld, clength %lld\n",
-		  (long long)offset, (long long)length,
-		  (long long)size, (long long)clength));
+	DEBUGF("offset %lld, length %lld, size %lld, clength %lld\n",
+	    (long long)offset, (long long)length,
+	    (long long)size, (long long)clength);
 
 	if (conn->err == HTTP_NOT_MODIFIED) {
 		http_seterr(HTTP_NOT_MODIFIED);
@@ -2032,3 +2091,12 @@ fetchListHTTP(struct url *url __unused, const char *flags __unused)
 	warnx("fetchListHTTP(): not implemented");
 	return (NULL);
 }
+
+FILE *
+fetchReqHTTP(struct url *URL, const char *method, const char *flags,
+	const char *content_type, const char *body)
+{
+
+	return (http_request_body(URL, method, NULL, http_get_proxy(URL, flags),
+	    flags, content_type, body));
+}
diff --git a/usr.bin/fetch/Makefile b/usr.bin/fetch/Makefile
index f0c342d3e6..cd80e75e65 100644
--- a/usr.bin/fetch/Makefile
+++ b/usr.bin/fetch/Makefile
@@ -1,12 +1,7 @@
-# $FreeBSD: src/usr.bin/fetch/Makefile,v 1.3.6.4 2003/01/09 11:54:58 des Exp $
+# $FreeBSD: head/usr.bin/fetch/Makefile 275042 2014-11-25 14:29:10Z bapt $
 
 PROG=		fetch
 DPADD=		${LIBFETCH}
 LDADD=		-lfetch
 
-DPADD+=		${LIBSSL} ${LIBCRYPTO}
-LDADD+=		-lprivate_ssl -lprivate_crypto
-LDFLAGS+=	${PRIVATELIB_LDFLAGS}
-CFLAGS+=	-I${.CURDIR}/../../crypto/libressl/include
-
 .include <bsd.prog.mk>
diff --git a/usr.bin/fetch/fetch.1 b/usr.bin/fetch/fetch.1
index dc9f88fd0b..f594b3064b 100644
--- a/usr.bin/fetch/fetch.1
+++ b/usr.bin/fetch/fetch.1
@@ -1,6 +1,6 @@
 .\"-
 .\" Copyright (c) 2000-2014 Dag-Erling Smørgrav
-.\" Copyright (c) 2013 Michael Gmelin <freebsd@grem.de>
+.\" Copyright (c) 2013-2016 Michael Gmelin <freebsd@grem.de>
 .\" All rights reserved.
 .\" Portions Copyright (c) 1999 Massachusetts Institute of Technology; used
 .\" by permission.
@@ -28,9 +28,9 @@
 .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 .\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .\"
-.\" $FreeBSD: head/usr.bin/fetch/fetch.1 261234 2014-01-28 14:32:04Z des $
+.\" $FreeBSD: head/usr.bin/fetch/fetch.1 333306 2018-05-06 15:59:03Z bjk $
 .\"
-.Dd January 28, 2014
+.Dd April 19, 2019
 .Dt FETCH 1
 .Os
 .Sh NAME
@@ -39,7 +39,6 @@
 .Sh SYNOPSIS
 .Nm
 .Op Fl 146AadFlMmnPpqRrsUv
-.Op Fl -allow-sslv2
 .Op Fl B Ar bytes
 .Op Fl -bind-address= Ns Ar host
 .Op Fl -ca-cert= Ns Ar file
@@ -113,9 +112,6 @@ Some broken Web sites will return a redirect instead of a not-found
 error when the requested object does not exist.
 .It Fl a , -retry
 Automatically retry the transfer upon soft failures.
-.It Fl -allow-sslv2
-[SSL]
-Allow SSL version 2 when negotiating the connection.
 .It Fl B Ar bytes , Fl -buffer-size= Ns Ar bytes
 Specify the read buffer size in bytes.
 The default is 16,384 bytes.
@@ -138,11 +134,17 @@ only.
 [SSL]
 Path to certificate bundle containing trusted CA certificates.
 If not specified,
-.Pa /etc/ssl/cert.pem
+.Pa /usr/local/etc/ssl/cert.pem
 is used.
-The file may contain multiple CA certificates. The port
+If this file does not exist,
+.Pa /etc/ssl/cert.pem
+is used instead.
+If neither file exists and no CA path has been configured,
+OpenSSL's default CA cert and path settings apply.
+The certificate bundle can contain multiple CA certificates.
+The
 .Pa security/ca_root_nss
-is a common source of a current CA bundle.
+port is a common source of a current CA bundle.
 .It Fl -ca-path= Ns Ar dir
 [SSL]
 The directory
@@ -222,10 +224,16 @@ altogether, or a comma- or whitespace-separated list of hosts for
 which proxies should not be used.
 .It Fl -no-sslv3
 [SSL]
-Don't allow SSL version 3 when negotiating the connection.
+Do not allow SSL version 3 when negotiating the connection.
+This option is deprecated and is provided for backward compatibility
+only.
+SSLv3 is disabled by default.
+Set
+.Ev SSL_ALLOW_SSL3
+to change this behavior.
 .It Fl -no-tlsv1
 [SSL]
-Don't allow TLS version 1 when negotiating the connection.
+Do not allow TLS version 1 when negotiating the connection.
 .It Fl -no-verify-hostname
 [SSL]
 Do not verify that the hostname matches the subject of the
@@ -233,7 +241,7 @@ certificate presented by the server.
 .It Fl -no-verify-peer
 [SSL]
 Do not verify the peer certificate against trusted CAs.
-.It Fl o Ar file , Fl output= Ns Ar file
+.It Fl o Ar file , Fl -output= Ns Ar file
 Set the output file name to
 .Ar file .
 By default, a ``pathname'' is extracted from the specified URI, and
@@ -350,14 +358,15 @@ for a description of additional environment variables, including
 .Ev NETRC ,
 .Ev NO_PROXY ,
 .Ev no_proxy ,
-.Ev SSL_ALLOW_SSL2 ,
 .Ev SSL_CA_CERT_FILE ,
 .Ev SSL_CA_CERT_PATH ,
 .Ev SSL_CLIENT_CERT_FILE ,
 .Ev SSL_CLIENT_KEY_FILE ,
 .Ev SSL_CRL_FILE ,
-.Ev SSL_NO_SSL3 ,
+.Ev SSL_ALLOW_SSL3 ,
 .Ev SSL_NO_TLS1 ,
+.Ev SSL_NO_TLS1_1 ,
+.Ev SSL_NO_TLS1_2 ,
 .Ev SSL_NO_VERIFY_HOSTNAME
 and
 .Ev SSL_NO_VERIFY_PEER .
@@ -376,7 +385,8 @@ argument is used and the remote file is not newer than the
 specified file then the command will still return success,
 although no file is transferred.
 .Sh SEE ALSO
-.Xr fetch 3
+.Xr fetch 3 ,
+.Xr phttpget 8
 .Sh HISTORY
 The
 .Nm
diff --git a/usr.bin/fetch/fetch.c b/usr.bin/fetch/fetch.c
index 98554fc619..bce54e30d5 100644
--- a/usr.bin/fetch/fetch.c
+++ b/usr.bin/fetch/fetch.c
@@ -1,4 +1,6 @@
 /*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
  * Copyright (c) 2000-2014 Dag-Erling Smørgrav
  * Copyright (c) 2013 Michael Gmelin <freebsd@grem.de>
  * All rights reserved.
@@ -26,7 +28,7 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
- * $FreeBSD: head/usr.bin/fetch/fetch.c 261234 2014-01-28 14:32:04Z des $
+ * $FreeBSD: head/usr.bin/fetch/fetch.c 338572 2018-09-10 19:39:20Z des $
  */
 
 #include <sys/param.h>
@@ -82,6 +84,7 @@ static int	 t_flag;	/*!   -t: workaround TCP bug */
 static int	 U_flag;	/*    -U: do not use high ports */
 static int	 v_level = 1;	/*    -v: verbosity level */
 static int	 v_tty;		/*        stdout is a tty */
+static int	 v_progress;	/*        whether to display progress */
 static pid_t	 pgrp;		/*        our process group */
 static long	 w_secs;	/*    -w: retry delay */
 static int	 family = PF_UNSPEC;	/* -[46]: address family to use */
@@ -101,7 +104,6 @@ enum options
 	OPTION_HTTP_REFERER,
 	OPTION_HTTP_USER_AGENT,
 	OPTION_NO_PROXY,
-	OPTION_SSL_ALLOW_SSL2,
 	OPTION_SSL_CA_CERT_FILE,
 	OPTION_SSL_CA_CERT_PATH,
 	OPTION_SSL_CLIENT_CERT_FILE,
@@ -153,7 +155,6 @@ static struct option longopts[] =
 	{ "referer", required_argument, NULL, OPTION_HTTP_REFERER },
 	{ "user-agent", required_argument, NULL, OPTION_HTTP_USER_AGENT },
 	{ "no-proxy", required_argument, NULL, OPTION_NO_PROXY },
-	{ "allow-sslv2", no_argument, NULL, OPTION_SSL_ALLOW_SSL2 },
 	{ "ca-cert", required_argument, NULL, OPTION_SSL_CA_CERT_FILE },
 	{ "ca-path", required_argument, NULL, OPTION_SSL_CA_CERT_PATH },
 	{ "cert", required_argument, NULL, OPTION_SSL_CLIENT_CERT_FILE },
@@ -197,13 +198,34 @@ struct xferstat {
 	off_t		 lastrcvd;	/* bytes received since last update */
 };
 
+/*
+ * Format a number of seconds as either XXdYYh, XXhYYm, XXmYYs, or XXs
+ * depending on its magnitude
+ */
+static void
+stat_seconds(char *str, size_t strsz, long seconds)
+{
+
+	if (seconds > 86400)
+		snprintf(str, strsz, "%02ldd%02ldh",
+		    seconds / 86400, (seconds % 86400) / 3600);
+	else if (seconds > 3600)
+		snprintf(str, strsz, "%02ldh%02ldm",
+		    seconds / 3600, (seconds % 3600) / 60);
+	else if (seconds > 60)
+		snprintf(str, strsz, "%02ldm%02lds",
+		    seconds / 60, seconds % 60);
+	else
+		snprintf(str, strsz, "   %02lds",
+		    seconds);
+}
+
 /*
  * Compute and display ETA
  */
-static const char *
-stat_eta(struct xferstat *xs)
+static void
+stat_eta(char *str, size_t strsz, const struct xferstat *xs)
 {
-	static char str[16];
 	long elapsed, eta;
 	off_t received, expected;
 
@@ -211,55 +233,47 @@ stat_eta(struct xferstat *xs)
 	received = xs->rcvd - xs->offset;
 	expected = xs->size - xs->rcvd;
 	eta = (long)((double)elapsed * expected / received);
-	if (eta > 3600)
-		snprintf(str, sizeof str, "%02ldh%02ldm",
-		    eta / 3600, (eta % 3600) / 60);
-	else if (eta > 0)
-		snprintf(str, sizeof str, "%02ldm%02lds",
-		    eta / 60, eta % 60);
+	if (eta > 0)
+		stat_seconds(str, strsz, eta);
 	else
-		snprintf(str, sizeof str, "%02ldm%02lds",
-		    elapsed / 60, elapsed % 60);
-	return (str);
+		stat_seconds(str, strsz, elapsed);
 }
 
 /*
  * Format a number as "xxxx YB" where Y is ' ', 'k', 'M'...
  */
 static const char *prefixes = " kMGTP";
-static const char *
-stat_bytes(off_t bytes)
+static void
+stat_bytes(char *str, size_t strsz, off_t bytes)
 {
-	static char str[16];
 	const char *prefix = prefixes;
 
 	while (bytes > 9999 && prefix[1] != '\0') {
 		bytes /= 1024;
 		prefix++;
 	}
-	snprintf(str, sizeof str, "%4jd %cB", (intmax_t)bytes, *prefix);
-	return (str);
+	snprintf(str, strsz, "%4ju %cB", (uintmax_t)bytes, *prefix);
 }
 
 /*
  * Compute and display transfer rate
  */
-static const char *
-stat_bps(struct xferstat *xs)
+static void
+stat_bps(char *str, size_t strsz, struct xferstat *xs)
 {
-	static char str[16];
+	char bytes[16];
 	double delta, bps;
 
-	delta = (xs->last.tv_sec + (xs->last.tv_usec / 1.e6))
-	    - (xs->last2.tv_sec + (xs->last2.tv_usec / 1.e6));
+	delta = ((double)xs->last.tv_sec + (xs->last.tv_usec / 1.e6))
+	    - ((double)xs->last2.tv_sec + (xs->last2.tv_usec / 1.e6));
 
 	if (delta == 0.0) {
-		snprintf(str, sizeof str, "?? Bps");
+		snprintf(str, strsz, "?? Bps");
 	} else {
 		bps = (xs->rcvd - xs->lastrcvd) / delta;
-		snprintf(str, sizeof str, "%sps", stat_bytes((off_t)bps));
+		stat_bytes(bytes, sizeof bytes, (off_t)bps);
+		snprintf(str, strsz, "%sps", bytes);
 	}
-	return (str);
 }
 
 /*
@@ -268,11 +282,12 @@ stat_bps(struct xferstat *xs)
 static void
 stat_display(struct xferstat *xs, int force)
 {
+	char bytes[16], bps[16], eta[16];
 	struct timeval now;
 	int ctty_pgrp;
 
 	/* check if we're the foreground process */
-	if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) == -1 ||
+	if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) != 0 ||
 	    (pid_t)ctty_pgrp != pgrp)
 		return;
 
@@ -283,26 +298,31 @@ stat_display(struct xferstat *xs, int force)
 	xs->last = now;
 
 	fprintf(stderr, "\r%-46.46s", xs->name);
-	if (xs->size <= 0) {
-		setproctitle("%s [%s]", xs->name, stat_bytes(xs->rcvd));
-		fprintf(stderr, "        %s", stat_bytes(xs->rcvd));
+	if (xs->rcvd >= xs->size) {
+		stat_bytes(bytes, sizeof bytes, xs->rcvd);
+		setproctitle("%s [%s]", xs->name, bytes);
+		fprintf(stderr, "        %s", bytes);
 	} else {
+		stat_bytes(bytes, sizeof bytes, xs->size);
 		setproctitle("%s [%d%% of %s]", xs->name,
 		    (int)((100.0 * xs->rcvd) / xs->size),
-		    stat_bytes(xs->size));
+		    bytes);
 		fprintf(stderr, "%3d%% of %s",
 		    (int)((100.0 * xs->rcvd) / xs->size),
-		    stat_bytes(xs->size));
+		    bytes);
 	}
 	if (force == 2) {
 		xs->lastrcvd = xs->offset;
 		xs->last2 = xs->start;
 	}
-	fprintf(stderr, " %s", stat_bps(xs));
+	stat_bps(bps, sizeof bps, xs);
+	fprintf(stderr, " %s", bps);
 	if ((xs->size > 0 && xs->rcvd > 0 &&
 	     xs->last.tv_sec >= xs->start.tv_sec + 3) ||
-	    force == 2)
-		fprintf(stderr, " %s", stat_eta(xs));
+	    force == 2) {
+		stat_eta(eta, sizeof eta, xs);
+		fprintf(stderr, " %s", eta);
+	}
 	xs->lastrcvd = xs->rcvd;
 }
 
@@ -312,14 +332,16 @@ stat_display(struct xferstat *xs, int force)
 static void
 stat_start(struct xferstat *xs, const char *name, off_t size, off_t offset)
 {
+
+	memset(xs, 0, sizeof *xs);
 	snprintf(xs->name, sizeof xs->name, "%s", name);
 	gettimeofday(&xs->start, NULL);
-	xs->last.tv_sec = xs->last.tv_usec = 0;
+	xs->last2 = xs->last = xs->start;
 	xs->size = size;
 	xs->offset = offset;
 	xs->rcvd = offset;
 	xs->lastrcvd = offset;
-	if (v_tty && v_level > 0)
+	if (v_progress)
 		stat_display(xs, 1);
 	else if (v_level > 0)
 		fprintf(stderr, "%-46s", xs->name);
@@ -331,8 +353,9 @@ stat_start(struct xferstat *xs, const char *name, off_t size, off_t offset)
 static void
 stat_update(struct xferstat *xs, off_t rcvd)
 {
+
 	xs->rcvd = rcvd;
-	if (v_tty && v_level > 0)
+	if (v_progress)
 		stat_display(xs, 0);
 }
 
@@ -342,13 +365,17 @@ stat_update(struct xferstat *xs, off_t rcvd)
 static void
 stat_end(struct xferstat *xs)
 {
+	char bytes[16], bps[16], eta[16];
+
 	gettimeofday(&xs->last, NULL);
-	if (v_tty && v_level > 0) {
+	if (v_progress) {
 		stat_display(xs, 2);
 		putc('\n', stderr);
 	} else if (v_level > 0) {
-		fprintf(stderr, "        %s %s\n",
-		    stat_bytes(xs->size), stat_bps(xs));
+		stat_bytes(bytes, sizeof bytes, xs->rcvd);
+		stat_bps(bps, sizeof bps, xs);
+		stat_eta(eta, sizeof eta, xs);
+		fprintf(stderr, "        %s %s %s\n", bytes, bps, eta);
 	}
 }
 
@@ -551,9 +578,10 @@ fetch(char *URL, const char *path)
 		goto signal;
 	if (f == NULL) {
 		warnx("%s: %s", URL, fetchLastErrString);
-		if (i_flag && strcmp(url->scheme, SCHEME_HTTP) == 0
-		    && fetchLastErrCode == FETCH_OK
-		    && strcmp(fetchLastErrString, "Not Modified") == 0) {
+		if (i_flag && (strcmp(url->scheme, SCHEME_HTTP) == 0 ||
+		    strcmp(url->scheme, SCHEME_HTTPS) == 0) &&
+		    fetchLastErrCode == FETCH_OK &&
+		    strcmp(fetchLastErrString, "Not Modified") == 0) {
 			/* HTTP Not Modified Response, return OK. */
 			r = 0;
 			goto done;
@@ -844,17 +872,17 @@ static void
 usage(void)
 {
 	fprintf(stderr, "%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n",
-"usage: fetch [-146AadFlMmnPpqRrsUv] [--allow-sslv2] [-B bytes]",
-"       [--bind-address=host] [--ca-cert=file] [--ca-path=dir] [--cert=file]",
-"       [--crl=file] [-i file] [--key=file] [-N file] [--no-passive]",
-"       [--no-proxy=list] [--no-sslv3] [--no-tlsv1] [--no-verify-hostname]",
-"       [--no-verify-peer] [-o file] [--referer=URL] [-S bytes] [-T seconds]",
+"usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [--bind-address=host]",
+"       [--ca-cert=file] [--ca-path=dir] [--cert=file] [--crl=file]",
+"       [-i file] [--key=file] [-N file] [--no-passive] [--no-proxy=list]",
+"       [--no-sslv3] [--no-tlsv1] [--no-verify-hostname] [--no-verify-peer]",
+"       [-o file] [--referer=URL] [-S bytes] [-T seconds]",
 "       [--user-agent=agent-string] [-w seconds] URL ...",
-"       fetch [-146AadFlMmnPpqRrsUv] [--allow-sslv2] [-B bytes]",
-"       [--bind-address=host] [--ca-cert=file] [--ca-path=dir] [--cert=file]",
-"       [--crl=file] [-i file] [--key=file] [-N file] [--no-passive]",
-"       [--no-proxy=list] [--no-sslv3] [--no-tlsv1] [--no-verify-hostname]",
-"       [--no-verify-peer] [-o file] [--referer=URL] [-S bytes] [-T seconds]",
+"       fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [--bind-address=host]",
+"       [--ca-cert=file] [--ca-path=dir] [--cert=file] [--crl=file]",
+"       [-i file] [--key=file] [-N file] [--no-passive] [--no-proxy=list]",
+"       [--no-sslv3] [--no-tlsv1] [--no-verify-hostname] [--no-verify-peer]",
+"       [-o file] [--referer=URL] [-S bytes] [-T seconds]",
 "       [--user-agent=agent-string] [-w seconds] -h host -f file [-c dir]");
 }
 
@@ -1003,9 +1031,6 @@ main(int argc, char *argv[])
 		case OPTION_NO_PROXY:
 			setenv("NO_PROXY", optarg, 1);
 			break;
-		case OPTION_SSL_ALLOW_SSL2:
-			setenv("SSL_ALLOW_SSL2", "", 1);
-			break;
 		case OPTION_SSL_CA_CERT_FILE:
 			setenv("SSL_CA_CERT_FILE", optarg, 1);
 			break;
@@ -1111,7 +1136,8 @@ main(int argc, char *argv[])
 
 	/* check if output is to a tty (for progress report) */
 	v_tty = isatty(STDERR_FILENO);
-	if (v_tty)
+	v_progress = v_tty && v_level > 0;
+	if (v_progress)
 		pgrp = getpgrp();
 
 	r = 0;
-- 
2.21.0

