NetBSD Problem Report #50438

From www@NetBSD.org  Tue Nov 17 01:39:03 2015
Return-Path: <www@NetBSD.org>
Received: from mail.netbsd.org (mail.netbsd.org [149.20.53.66])
	(using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits))
	(Client CN "mail.netbsd.org", Issuer "Postmaster NetBSD.org" (verified OK))
	by mollari.NetBSD.org (Postfix) with ESMTPS id AED23A5B2E
	for <gnats-bugs@gnats.NetBSD.org>; Tue, 17 Nov 2015 01:39:03 +0000 (UTC)
Message-Id: <20151117013901.C656FA65BC@mollari.NetBSD.org>
Date: Tue, 17 Nov 2015 01:39:01 +0000 (UTC)
From: nonakap@gmail.com
Reply-To: nonakap@gmail.com
To: gnats-bugs@NetBSD.org
Subject: ftp(1): CONNECT method support
X-Send-Pr-Version: www-1.0

>Number:         50438
>Category:       bin
>Synopsis:       ftp(1): CONNECT method support
>Confidential:   no
>Severity:       serious
>Priority:       medium
>Responsible:    bin-bug-people
>State:          closed
>Class:          change-request
>Submitter-Id:   net
>Arrival-Date:   Tue Nov 17 01:40:00 +0000 2015
>Closed-Date:    Wed Dec 16 21:24:04 +0000 2015
>Last-Modified:  Sat Jul 04 09:50:03 +0000 2020
>Originator:     NONAKA Kimihiro
>Release:        HEAD (20151113)
>Organization:
>Environment:
NetBSD netbsd 7.99.21 NetBSD 7.99.21 (KOHARU) #0: Tue Nov 17 09:51:25 JST 2015  nonaka@netbsd:/home/snapshot/NetBSD/HEAD.20151113/obj.NetBSD-amd64/amd64/sys/arch/amd64/compile/KOHARU amd64
>Description:
ftp(1) lacks CONNECT method for https connection via proxy.
>How-To-Repeat:
1) Install http proxy software (e.g. pkgsrc/www/tinyproxy)
2) Set https_proxy shell variable
3) ftp https://github.com/index.html

I tested with tinyproxy.

$ ftp https://github.com/index.html
Trying 127.0.0.1:8888 ...
Requesting https://github.com/index.html
  (via 127.0.0.1:8888)
ftp: Error retrieving file `501 Not Implemented'

>Fix:
Please apply the following patch.

Index: usr.bin/ftp/fetch.c
===================================================================
RCS file: /cvsroot/src/usr.bin/ftp/fetch.c,v
retrieving revision 1.207
diff -u -r1.207 fetch.c
--- usr.bin/ftp/fetch.c	12 Sep 2015 19:38:42 -0000	1.207
+++ usr.bin/ftp/fetch.c	17 Nov 2015 00:44:24 -0000
@@ -529,8 +529,11 @@
 	time_t			mtime;
 	url_t			urltype;
 	in_port_t		portnum;
+	int			proxyauthdone;
 #ifdef WITH_SSL
 	void			*ssl;
+	url_t			ourltype;
+	char			*ohost, *oport;
 #endif

 	DPRINTF("%s: `%s' proxyenv `%s'\n", __func__, url, STRorNULL(penv));
@@ -542,9 +545,13 @@
 	s = -1;
 	savefile = NULL;
 	auth = location = message = NULL;
-	ischunked = isproxy = hcode = 0;
+	ischunked = isproxy = hcode = proxyauthdone = 0;
 	rval = 1;
 	uuser = pass = host = path = decodedpath = puser = ppass = NULL;
+#ifdef WITH_SSL
+	ourltype = UNKNOWN_URL_T;
+	ohost = oport = NULL;
+#endif

 	if (sigsetjmp(httpabort, 1))
 		goto cleanup_fetch_url;
@@ -552,6 +559,13 @@
 	if (parse_url(url, "URL", &urltype, &uuser, &pass, &host, &port,
 	    &portnum, &path) == -1)
 		goto cleanup_fetch_url;
+#ifdef WITH_SSL
+	if (urltype == HTTPS_URL_T) {
+		ourltype = urltype;
+		ohost = host;
+		oport = port;
+	}
+#endif

 	if (urltype == FILE_URL_T && ! EMPTYSTRING(host)
 	    && strcasecmp(host, "localhost") != 0) {
@@ -721,8 +735,14 @@
 					goto cleanup_fetch_url;
 				}

+#ifdef WITH_SSL
+				if (ohost != host)
+#endif
 				FREEPTR(host);
 				host = phost;
+#ifdef WITH_SSL
+				if (oport != port)
+#endif
 				FREEPTR(port);
 				port = pport;
 				FREEPTR(path);
@@ -823,10 +843,168 @@
 				leading = ", ";
 				hasleading++;
 			}
+#ifdef WITH_SSL
+			if (ourltype == HTTPS_URL_T) {
+				if (strchr(ohost, ':')) {
+					char *h, *p;
+
+					/*
+					 * strip off IPv6 scope identifier,
+					 * since it is local to the node
+					 */
+					h = ftp_strdup(ohost);
+					if (isipv6addr(h) &&
+					    (p = strchr(h, '%')) != NULL) {
+						*p = '\0';
+					}
+					fetch_printf(fin,
+					    "CONNECT [%s]:%s HTTP/1.1\r\n",
+					    h, oport);
+					fetch_printf(fin, "Host: [%s]:%s\r\n",
+					    h, oport);
+					free(h);
+				} else {
+					fetch_printf(fin,
+					    "CONNECT %s:%s HTTP/1.1\r\n",
+					    ohost, oport);
+					fetch_printf(fin, "Host: %s:%s\r\n",
+					    ohost, oport);
+				}
+				useragent = getenv("FTPUSERAGENT");
+				if (useragent != NULL) {
+					fetch_printf(fin, "User-Agent: %s\r\n",
+					    useragent);
+				} else {
+					fetch_printf(fin,
+					    "User-Agent: %s/%s\r\n",
+					    FTP_PRODUCT, FTP_VERSION);
+				}
+				if (proxyauth) {
+					if (verbose) {
+						fprintf(ttyout,
+						    "%swith proxy authorization"
+						    , leading);
+						leading = ", ";
+						hasleading++;
+					}
+					fetch_printf(fin,
+					    "Proxy-Authorization: %s\r\n",
+					    proxyauth);
+				}
+				if (verbose && hasleading)
+					fputs(")\n", ttyout);
+				leading = "  (";
+				hasleading = 0;
+				fetch_printf(fin, "\r\n");
+				if (fetch_flush(fin) == EOF) {
+					warn("Writing HTTP request");
+					alarmtimer(0);
+					goto cleanup_fetch_url;
+				}
+				alarmtimer(0);
+
+				/* Read the response */
+				alarmtimer(quit_time ? quit_time : 60);
+				len = fetch_getline(fin, buf, sizeof(buf),
+				    &errormsg);
+				alarmtimer(0);
+				if (len < 0) {
+					if (*errormsg == '\n')
+						errormsg++;
+					warnx("Receiving HTTP reply: %s",
+					    errormsg);
+					goto cleanup_fetch_url;
+				}
+				while (len > 0 && (ISLWS(buf[len-1])))
+					buf[--len] = '\0';
+				DPRINTF("%s: received `%s'\n", __func__, buf);
+
+				/* Determine HTTP response code */
+				cp = strchr(buf, ' ');
+				if (cp == NULL)
+					goto improper;
+				else
+					cp++;
+				hcode = strtol(cp, &ep, 10);
+				if (*ep != '\0' && !isspace((unsigned char)*ep))
+					goto improper;
+				message = ftp_strdup(cp);
+
+				while (1) {
+					alarmtimer(quit_time ? quit_time : 60);
+					len = fetch_getline(fin, buf,
+					    sizeof(buf), &errormsg);
+					alarmtimer(0);
+					if (len < 0) {
+						if (*errormsg == '\n')
+							errormsg++;
+						warnx("Receiving HTTP reply: %s"
+						    , errormsg);
+						goto cleanup_fetch_url;
+					}
+					while (len > 0 && (ISLWS(buf[len-1])))
+						buf[--len] = '\0';
+					if (len == 0)
+						break;
+					DPRINTF("%s: received `%s'\n",
+					    __func__, buf);
+
+					if (match_token(&cp,
+					    "Proxy-Authenticate:")) {
+						if (!(token = match_token(&cp,
+						    "Basic"))) {
+							DPRINTF("%s: skipping "
+							    "unknown auth "
+							    "scheme `%s'\n",
+							    __func__, token);
+							continue;
+						}
+						FREEPTR(auth);
+						auth = ftp_strdup(token);
+						DPRINTF("%s: parsed auth as "
+						    "`%s'\n", __func__, cp);
+						proxyauthdone = 1;
+					}
+				}
+
+				/* finished parsing header */
+				switch (hcode) {
+				case 200:
+					break;
+				default:
+					if (message)
+						warnx("Error proxy connect "
+						    "`%s'", message);
+					else
+						warnx("Unknown error proxy "
+						    "connect");
+					goto cleanup_fetch_url;
+				}
+
+				if ((ssl = fetch_start_ssl(s, host)) == NULL)
+					goto cleanup_fetch_url;
+				fetch_set_ssl(fin, ssl);
+				ssl = NULL;
+
+				urltype = ourltype;
+				if (host != ohost) {
+					FREEPTR(host);
+					host = ohost;
+				}
+				if (port != oport) {
+					FREEPTR(port);
+					port = oport;
+				}
+				goto no_proxy;
+			}
+#endif
 			fetch_printf(fin, "GET %s HTTP/1.0\r\n", path);
 			if (flushcache)
 				fetch_printf(fin, "Pragma: no-cache\r\n");
 		} else {
+#ifdef WITH_SSL
+no_proxy:
+#endif
 			fetch_printf(fin, "GET %s HTTP/1.1\r\n", path);
 			if (strchr(host, ':')) {
 				char *h, *p;
@@ -881,7 +1059,7 @@
 			}
 			fetch_printf(fin, "Authorization: %s\r\n", wwwauth);
 		}
-		if (proxyauth) {
+		if (proxyauth && !proxyauthdone) {
 			if (verbose) {
 				fprintf(ttyout,
 				    "%swith proxy authorization", leading);
@@ -1395,6 +1573,12 @@
 	if (pass != NULL)
 		memset(pass, 0, strlen(pass));
 	FREEPTR(pass);
+#ifdef WITH_SSL
+	if (host != ohost)
+		FREEPTR(ohost);
+	if (port != oport)
+		FREEPTR(oport);
+#endif
 	FREEPTR(host);
 	FREEPTR(port);
 	FREEPTR(path);

>Release-Note:

>Audit-Trail:
From: "Christos Zoulas" <christos@netbsd.org>
To: gnats-bugs@gnats.NetBSD.org
Cc: 
Subject: PR/50438 CVS commit: src/usr.bin/ftp
Date: Wed, 16 Dec 2015 16:11:48 -0500

 Module Name:	src
 Committed By:	christos
 Date:		Wed Dec 16 21:11:47 UTC 2015

 Modified Files:
 	src/usr.bin/ftp: fetch.c

 Log Message:
 PR/50438: NONAKA Kimihiro: ftp(1): CONNECT method support
 Please test!


 To generate a diff of this commit:
 cvs rdiff -u -r1.214 -r1.215 src/usr.bin/ftp/fetch.c

 Please note that diffs are not public domain; they are subject to the
 copyright notices on the relevant files.

State-Changed-From-To: open->closed
State-Changed-By: wiz@NetBSD.org
State-Changed-When: Wed, 16 Dec 2015 21:24:04 +0000
State-Changed-Why:
christos committed the patch. Thank you!


From: "Martin Husemann" <martin@netbsd.org>
To: gnats-bugs@gnats.NetBSD.org
Cc: 
Subject: PR/50438 CVS commit: [netbsd-7] src/usr.bin/ftp
Date: Sun, 13 Mar 2016 11:49:14 +0000

 Module Name:	src
 Committed By:	martin
 Date:		Sun Mar 13 11:49:14 UTC 2016

 Modified Files:
 	src/usr.bin/ftp [netbsd-7]: cmds.c fetch.c ftp.c ftp_var.h ssl.c

 Log Message:
 Pull up following revision(s) (requested by nonakap in ticket #1133):
 	usr.bin/ftp/fetch.c: revision 1.208-1.221
 	usr.bin/ftp/cmds.c: revision 1.136-1.137
 	usr.bin/ftp/ssl.c: revision 1.5
 	usr.bin/ftp/ftp.c: revision 1.165-1.166
 	usr.bin/ftp/ftp_var.h: revision 1.84

 Workaround const issues of SSL_set_tlsext_host_name.

 Use the proper format "[IPv6 address]:port" when reporting connection
 attempts to IPv6 endpoints.

 (Hopefully) fix build without IPv6 support

 Try to factor out some code, this is completely out of control.

 Separate no_proxy handling.

 Factor the proxy handling code out.

 Fix compile failure without WITH_SSL.

 PR/50438: NONAKA Kimihiro: ftp(1): CONNECT method support

 make DPRINTF/DWARN always statements.

 Fix to connect https via proxy.

 Fix ttyout message.

 Simplify and factor out connect message

 Split the position/size parsing into a separate function.

 Mark function as only needed with ssl.

 Fix downloads of local files using file:// URLs

 Initialize the token match pointer.

 use sizeof() and array notation.
 CID 1354295: Array overrun.


 To generate a diff of this commit:
 cvs rdiff -u -r1.135 -r1.135.8.1 src/usr.bin/ftp/cmds.c
 cvs rdiff -u -r1.205.4.2 -r1.205.4.3 src/usr.bin/ftp/fetch.c
 cvs rdiff -u -r1.164 -r1.164.10.1 src/usr.bin/ftp/ftp.c
 cvs rdiff -u -r1.82.8.1 -r1.82.8.2 src/usr.bin/ftp/ftp_var.h
 cvs rdiff -u -r1.2.14.1 -r1.2.14.2 src/usr.bin/ftp/ssl.c

 Please note that diffs are not public domain; they are subject to the
 copyright notices on the relevant files.

From: "Luke Mewburn" <lukem@netbsd.org>
To: gnats-bugs@gnats.NetBSD.org
Cc: 
Subject: PR/50438 CVS commit: othersrc/usr.bin/tnftp/src
Date: Sat, 4 Jul 2020 09:48:28 +0000

 Module Name:	othersrc
 Committed By:	lukem
 Date:		Sat Jul  4 09:48:28 UTC 2020

 Update of /cvsroot/othersrc/usr.bin/tnftp/src
 In directory ivanova.netbsd.org:/tmp/cvs-serv25717

 Log Message:
 Import NetBSD-ftp 20200608

 Notable changes since NetBSD-ftp 20151003:
 * Avoid crashies by exit if lostpeer due to a signal
   (e.g., remote server disconnection).
 * Issue PWD commands to the server only when we actually
   need the results, not speculatively, just in case we might.
   Works around broken servers, and is quicker too.
 * Fix error reporting when handling TLS connections.
 * Use "anonymous" instead of the local username for anonymous ftp.
   Avoids unnecesary information leak.
 * Correct format of IPv6 endpoint reporting.
 * Refactoring and build fixes.
 * Support using CONNECT for https:// via proxy. PR/50438, PR/51043.
 * Fix downloads of local files using file:// URLs
 * Use the first name we requested the http/https URL for, not any name
   we ended up with after random redirects.

 Status:

 Vendor Tag:	NetBSD
 Release Tags:	NetBSD-20200608

 U othersrc/usr.bin/tnftp/src/ruserpass.c
 C othersrc/usr.bin/tnftp/src/ftp.1
 C othersrc/usr.bin/tnftp/src/fetch.c
 C othersrc/usr.bin/tnftp/src/util.c
 C othersrc/usr.bin/tnftp/src/ftp.c
 U othersrc/usr.bin/tnftp/src/cmdtab.c
 C othersrc/usr.bin/tnftp/src/ssl.h
 C othersrc/usr.bin/tnftp/src/main.c
 C othersrc/usr.bin/tnftp/src/ssl.c
 C othersrc/usr.bin/tnftp/src/extern.h
 U othersrc/usr.bin/tnftp/src/progressbar.h
 C othersrc/usr.bin/tnftp/src/ftp_var.h
 C othersrc/usr.bin/tnftp/src/version.h
 C othersrc/usr.bin/tnftp/src/progressbar.c
 C othersrc/usr.bin/tnftp/src/Makefile
 C othersrc/usr.bin/tnftp/src/complete.c
 C othersrc/usr.bin/tnftp/src/domacro.c
 C othersrc/usr.bin/tnftp/src/cmds.c

 15 conflicts created by this import.
 Use the following command to help the merge:

 	cvs checkout -jNetBSD:yesterday -jNetBSD othersrc/usr.bin/tnftp/src

>Unformatted:

Home	PR Database Search