1 /*
2 * $LynxId: HTTP.c,v 1.128 2013/05/05 19:36:45 tom Exp $
3 *
4 * HyperText Tranfer Protocol - Client implementation HTTP.c
5 * ==========================
6 * Modified:
7 * 27 Jan 1994 PDM Added Ari Luotonen's Fix for Reload when using proxy
8 * servers.
9 * 28 Apr 1997 AJL,FM Do Proxy Authorisation.
10 */
11
12 #include <HTUtils.h>
13 #include <HTTP.h>
14 #include <LYUtils.h>
15
16 #ifdef USE_SSL
17 #include <HTNews.h>
18 #endif
19
20 #define HTTP_VERSION "HTTP/1.0"
21
22 #define HTTP_PORT 80
23 #define HTTPS_PORT 443
24 #define SNEWS_PORT 563
25
26 #define INIT_LINE_SIZE 1536 /* Start with line buffer this big */
27 #define LINE_EXTEND_THRESH 256 /* Minimum read size */
28 #define VERSION_LENGTH 20 /* for returned protocol version */
29
30 #include <HTParse.h>
31 #include <HTTCP.h>
32 #include <HTFormat.h>
33 #include <HTFile.h>
34 #include <HTAlert.h>
35 #include <HTMIME.h>
36 #include <HTML.h>
37 #include <HTInit.h>
38 #include <HTAABrow.h>
39 #include <HTAccess.h> /* Are we using an HTTP gateway? */
40
41 #include <LYCookie.h>
42 #include <LYGlobalDefs.h>
43 #include <GridText.h>
44 #include <LYStrings.h>
45 #include <LYUtils.h>
46 #include <LYrcFile.h>
47 #include <LYLeaks.h>
48
49 #ifdef USE_SSL
50 #ifdef USE_OPENSSL_INCL
51 #include <openssl/x509v3.h>
52 #endif
53 #ifdef USE_GNUTLS_INCL
54 #include <gnutls/x509.h>
55 #endif
56 #endif
57
58 BOOLEAN reloading = FALSE; /* Reloading => send no-cache pragma to proxy */
59 char *redirecting_url = NULL; /* Location: value. */
60 BOOL permanent_redirection = FALSE; /* Got 301 status? */
61 BOOL redirect_post_content = FALSE; /* Don't convert to GET? */
62
63 #ifdef USE_SSL
64 SSL_CTX *ssl_ctx = NULL; /* SSL ctx */
65 SSL *SSL_handle = NULL;
66 static int ssl_okay;
67
free_ssl_ctx(void)68 static void free_ssl_ctx(void)
69 {
70 if (ssl_ctx != NULL)
71 SSL_CTX_free(ssl_ctx);
72 }
73
HTSSLCallback(int preverify_ok,X509_STORE_CTX * x509_ctx GCC_UNUSED)74 static int HTSSLCallback(int preverify_ok, X509_STORE_CTX * x509_ctx GCC_UNUSED)
75 {
76 char *msg = NULL;
77 int result = 1;
78
79 #ifdef USE_X509_SUPPORT
80 HTSprintf0(&msg,
81 gettext("SSL callback:%s, preverify_ok=%d, ssl_okay=%d"),
82 X509_verify_cert_error_string((long) X509_STORE_CTX_get_error(x509_ctx)),
83 preverify_ok, ssl_okay);
84 _HTProgress(msg);
85 FREE(msg);
86 #endif
87
88 #ifndef USE_NSS_COMPAT_INCL
89 if (!(preverify_ok || ssl_okay || ssl_noprompt)) {
90 #ifdef USE_X509_SUPPORT
91 HTSprintf0(&msg, SSL_FORCED_PROMPT,
92 X509_verify_cert_error_string((long)
93 X509_STORE_CTX_get_error(x509_ctx)));
94 if (HTForcedPrompt(ssl_noprompt, msg, YES))
95 ssl_okay = 1;
96 else
97 result = 0;
98 #endif
99
100 FREE(msg);
101 }
102 #endif
103 return result;
104 }
105
HTGetSSLHandle(void)106 SSL *HTGetSSLHandle(void)
107 {
108 #ifdef USE_GNUTLS_INCL
109 static char *certfile = NULL;
110 #endif
111
112 if (ssl_ctx == NULL) {
113 /*
114 * First time only.
115 */
116 #if SSLEAY_VERSION_NUMBER < 0x0800
117 ssl_ctx = SSL_CTX_new();
118 X509_set_default_verify_paths(ssl_ctx->cert);
119 #else
120 long ssl_opts;
121
122 ssl_opts = SSL_OP_ALL;
123 #ifdef SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
124 ssl_opts &= ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS;
125 #endif
126 SSLeay_add_ssl_algorithms();
127 ssl_ctx = SSL_CTX_new(SSLv23_client_method());
128 SSL_CTX_set_options(ssl_ctx, ssl_opts);
129 SSL_CTX_set_default_verify_paths(ssl_ctx);
130 SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, HTSSLCallback);
131 #endif /* SSLEAY_VERSION_NUMBER < 0x0800 */
132 #if defined(USE_PROGRAM_DIR) & !defined(USE_GNUTLS_INCL)
133 {
134 X509_LOOKUP *lookup;
135
136 lookup = X509_STORE_add_lookup(ssl_ctx->cert_store,
137 X509_LOOKUP_file());
138 if (lookup != NULL) {
139 char *certfile = NULL;
140
141 HTSprintf0(&certfile, "%s\\cert.pem", program_dir);
142 X509_LOOKUP_load_file(lookup, certfile, X509_FILETYPE_PEM);
143 FREE(certfile);
144 }
145 }
146 #endif
147 #ifdef USE_GNUTLS_INCL
148 if ((certfile = LYGetEnv("SSL_CERT_FILE")) != NULL) {
149 CTRACE((tfp,
150 "HTGetSSLHandle: certfile is set to %s by SSL_CERT_FILE\n",
151 certfile));
152 } else {
153 if (non_empty(SSL_cert_file)) {
154 certfile = SSL_cert_file;
155 CTRACE((tfp,
156 "HTGetSSLHandle: certfile is set to %s by config SSL_CERT_FILE\n",
157 certfile));
158 }
159 #if defined(USE_PROGRAM_DIR)
160 else {
161 HTSprintf0(&(certfile), "%s\\cert.pem", program_dir);
162 CTRACE((tfp,
163 "HTGetSSLHandle: certfile is set to %s by installed directory\n", certfile));
164 }
165 #endif
166 }
167 #endif
168 atexit(free_ssl_ctx);
169 }
170 #ifdef USE_GNUTLS_INCL
171 ssl_ctx->certfile = certfile;
172 ssl_ctx->certfile_type = GNUTLS_X509_FMT_PEM;
173 #endif
174 ssl_okay = 0;
175 return (SSL_new(ssl_ctx));
176 }
177
HTSSLInitPRNG(void)178 void HTSSLInitPRNG(void)
179 {
180 #ifndef __MirBSD__
181 #if SSLEAY_VERSION_NUMBER >= 0x00905100
182 if (RAND_status() == 0) {
183 char rand_file[256];
184 time_t t;
185 long l, seed;
186
187 #ifndef _WINDOWS
188 pid_t pid;
189
190 #else
191 DWORD pid;
192 #endif
193
194 t = time(NULL);
195
196 #ifndef _WINDOWS
197 pid = getpid();
198 #else
199 pid = GetCurrentThreadId();
200 #endif
201
202 RAND_file_name(rand_file, 256L);
203 CTRACE((tfp, "HTTP: Seeding PRNG\n"));
204 /* Seed as much as 1024 bytes from RAND_file_name */
205 RAND_load_file(rand_file, 1024L);
206 /* Seed in time (mod_ssl does this) */
207 RAND_seed((unsigned char *) &t, (int) sizeof(time_t));
208
209 /* Seed in pid (mod_ssl does this) */
210 RAND_seed((unsigned char *) &pid, (int) sizeof(pid));
211 /* Initialize system's random number generator */
212 RAND_bytes((unsigned char *) &seed, (int) sizeof(long));
213
214 lynx_srand((unsigned) seed);
215 while (RAND_status() == 0) {
216 /* Repeatedly seed the PRNG using the system's random number generator until it has been seeded with enough data */
217 l = lynx_rand();
218 RAND_seed((unsigned char *) &l, (int) sizeof(long));
219 }
220 /* Write a rand_file */
221 RAND_write_file(rand_file);
222 }
223 #endif /* SSLEAY_VERSION_NUMBER >= 0x00905100 */
224 #endif
225 return;
226 }
227
228 #define HTTP_NETREAD(sock, buff, size, handle) \
229 (handle \
230 ? SSL_read(handle, buff, size) \
231 : NETREAD(sock, buff, size))
232
233 #define HTTP_NETWRITE(sock, buff, size, handle) \
234 (handle \
235 ? SSL_write(handle, buff, size) \
236 : NETWRITE(sock, buff, size))
237
238 #define HTTP_NETCLOSE(sock, handle) \
239 { (void)NETCLOSE(sock); \
240 if (handle) \
241 SSL_free(handle); \
242 SSL_handle = handle = NULL; \
243 }
244
245 #else
246 #define HTTP_NETREAD(a, b, c, d) NETREAD(a, b, c)
247 #define HTTP_NETWRITE(a, b, c, d) NETWRITE(a, b, c)
248 #define HTTP_NETCLOSE(a, b) (void)NETCLOSE(a)
249 #endif /* USE_SSL */
250
251 #ifdef _WINDOWS /* 1997/11/06 (Thu) 13:00:08 */
252
253 #define BOX_TITLE "Lynx " __FILE__
254 #define BOX_FLAG (MB_ICONINFORMATION | MB_SETFOREGROUND)
255
256 typedef struct {
257 int fd;
258 char *buf;
259 int len;
260 } recv_data_t;
261
262 int ws_read_per_sec = 0;
263 static int ws_errno = 0;
264
265 static DWORD g_total_times = 0;
266 static DWORD g_total_bytes = 0;
267
268 /* The same like read, but takes care of EINTR and uses select to
269 timeout the stale connections. */
270
ws_read(int fd,char * buf,int len)271 static int ws_read(int fd, char *buf, int len)
272 {
273 int res;
274 int retry = 3;
275
276 do {
277 res = recv(fd, buf, len, 0);
278 if (WSAEWOULDBLOCK == WSAGetLastError()) {
279 Sleep(100);
280 if (retry-- > 0)
281 continue;
282 }
283 } while (res == SOCKET_ERROR && SOCKET_ERRNO == EINTR);
284
285 return res;
286 }
287
288 #define DWORD_ERR ((DWORD)-1)
289
_thread_func(void * p)290 static DWORD __stdcall _thread_func(void *p)
291 {
292 DWORD result;
293 int i, val;
294 recv_data_t *q = (recv_data_t *) p;
295
296 i = 0;
297 i++;
298 val = ws_read(q->fd, q->buf, q->len);
299
300 if (val == SOCKET_ERROR) {
301 ws_errno = WSAGetLastError();
302 #if 0
303 char buff[256];
304
305 sprintf(buff, "Thread read: %d, error (%ld), fd = %d, len = %d",
306 i, ws_errno, q->fd, q->len);
307 MessageBox(NULL, buff, BOX_TITLE, BOX_FLAG);
308 #endif
309 result = DWORD_ERR;
310 } else {
311 result = val;
312 }
313
314 return result;
315 }
316
317 /* The same like read, but takes care of EINTR and uses select to
318 timeout the stale connections. */
319
ws_netread(int fd,char * buf,int len)320 int ws_netread(int fd, char *buf, int len)
321 {
322 int i;
323 char buff[256];
324
325 /* 1998/03/30 (Mon) 09:01:21 */
326 HANDLE hThread;
327 DWORD dwThreadID;
328 DWORD exitcode = 0;
329 DWORD ret_val = DWORD_ERR;
330 DWORD val, process_time, now_TickCount, save_TickCount;
331
332 static recv_data_t para;
333
334 #define TICK 5
335 #define STACK_SIZE 0x2000uL
336
337 InitializeCriticalSection(&critSec_READ);
338
339 para.fd = fd;
340 para.buf = buf;
341 para.len = len;
342
343 ws_read_per_sec = 0;
344 save_TickCount = GetTickCount();
345
346 hThread = CreateThread(NULL, STACK_SIZE,
347 _thread_func,
348 (void *) ¶, 0UL, &dwThreadID);
349
350 if (hThread == 0) {
351 HTInfoMsg("CreateThread Failed (read)");
352 goto read_exit;
353 }
354
355 i = 0;
356 while (1) {
357 val = WaitForSingleObject(hThread, 1000 / TICK);
358 i++;
359 if (val == WAIT_FAILED) {
360 HTInfoMsg("Wait Failed");
361 ret_val = DWORD_ERR;
362 break;
363 } else if (val == WAIT_TIMEOUT) {
364 i++;
365 if (i / TICK > (AlertSecs + 2)) {
366 sprintf(buff, "Read Waiting (%2d.%01d) for %d Bytes",
367 i / TICK, (i % TICK) * 10 / TICK, len);
368 SetConsoleTitle(buff);
369 }
370 if (win32_check_interrupt() || ((i / TICK) > lynx_timeout)) {
371 if (CloseHandle(hThread) == FALSE) {
372 HTInfoMsg("Thread terminate Failed");
373 }
374 WSASetLastError(ETIMEDOUT);
375 ret_val = HT_INTERRUPTED;
376 break;
377 }
378 } else if (val == WAIT_OBJECT_0) {
379 if (GetExitCodeThread(hThread, &exitcode) == FALSE) {
380 exitcode = DWORD_ERR;
381 }
382 if (CloseHandle(hThread) == FALSE) {
383 HTInfoMsg("Thread terminate Failed");
384 }
385 now_TickCount = GetTickCount();
386 if (now_TickCount >= save_TickCount)
387 process_time = now_TickCount - save_TickCount;
388 else
389 process_time = now_TickCount + (0xffffffff - save_TickCount);
390
391 if (process_time == 0)
392 process_time = 1;
393 g_total_times += process_time;
394
395 /*
396 * DWORD is unsigned, and could be an error code which is signed.
397 */
398 if ((long) exitcode > 0)
399 g_total_bytes += exitcode;
400
401 ws_read_per_sec = g_total_bytes;
402 if (ws_read_per_sec > 2000000) {
403 if (g_total_times > 1000)
404 ws_read_per_sec /= (g_total_times / 1000);
405 } else {
406 ws_read_per_sec *= 1000;
407 ws_read_per_sec /= g_total_times;
408 }
409
410 ret_val = exitcode;
411 break;
412 }
413 } /* end while(1) */
414
415 read_exit:
416 LeaveCriticalSection(&critSec_READ);
417 return ret_val;
418 }
419 #endif /* _WINDOWS */
420
421 /*
422 * Strip any username from the given string so we retain only the host.
423 */
strip_userid(char * host)424 static void strip_userid(char *host)
425 {
426 char *p1 = host;
427 char *p2 = strchr(host, '@');
428 char *fake;
429
430 if (p2 != 0) {
431 *p2++ = '\0';
432 if ((fake = HTParse(host, "", PARSE_HOST)) != NULL) {
433 char *msg = NULL;
434
435 CTRACE((tfp, "parsed:%s\n", fake));
436 HTSprintf0(&msg, gettext("Address contains a username: %s"), host);
437 HTAlert(msg);
438 FREE(msg);
439 }
440 while ((*p1++ = *p2++) != '\0') {
441 ;
442 }
443 }
444 }
445
446 /*
447 * Check if the user's options specified to use the given encoding. Normally
448 * all encodings with compiled-in support are specified (encodingALL).
449 */
acceptEncoding(int code)450 static BOOL acceptEncoding(int code)
451 {
452 BOOL result = FALSE;
453
454 if ((code & LYAcceptEncoding) != 0) {
455 const char *program = 0;
456
457 switch (code) {
458 case encodingGZIP:
459 program = HTGetProgramPath(ppGZIP);
460 break;
461 case encodingDEFLATE:
462 program = HTGetProgramPath(ppINFLATE);
463 break;
464 case encodingCOMPRESS:
465 program = HTGetProgramPath(ppCOMPRESS);
466 break;
467 case encodingBZIP2:
468 program = HTGetProgramPath(ppBZIP2);
469 break;
470 default:
471 break;
472 }
473 /*
474 * FIXME: if lynx did not rely upon external programs to decompress
475 * files for external viewers, this check could be relaxed.
476 */
477 result = (BOOL) (program != 0);
478 }
479 return result;
480 }
481
482 #ifdef USE_SSL
show_cert_issuer(X509 * peer_cert GCC_UNUSED)483 static void show_cert_issuer(X509 * peer_cert GCC_UNUSED)
484 {
485 #if defined(USE_OPENSSL_INCL) || defined(USE_GNUTLS_FUNCS)
486 char ssl_dn[1024];
487 char *msg = NULL;
488
489 X509_NAME_oneline(X509_get_issuer_name(peer_cert), ssl_dn, (int) sizeof(ssl_dn));
490 HTSprintf0(&msg, gettext("Certificate issued by: %s"), ssl_dn);
491 _HTProgress(msg);
492 FREE(msg);
493 #elif defined(USE_GNUTLS_INCL)
494 /* the OpenSSL "compat" code compiles but dumps core with GNU TLS */
495 #endif
496 }
497 #endif
498
499 /*
500 * Remove IPv6 brackets (and any port-number) from the given host-string.
501 */
502 #ifdef USE_SSL
StripIpv6Brackets(char * host)503 static char *StripIpv6Brackets(char *host)
504 {
505 int port_number;
506 char *p;
507
508 if ((p = HTParsePort(host, &port_number)) != 0)
509 *p = '\0';
510
511 if (*host == '[') {
512 p = host + strlen(host) - 1;
513 if (*p == ']') {
514 *p = '\0';
515 ++host;
516 }
517 }
518 return host;
519 }
520 #endif
521
522 /* Load Document from HTTP Server HTLoadHTTP()
523 * ==============================
524 *
525 * Given a hypertext address, this routine loads a document.
526 *
527 *
528 * On entry,
529 * arg is the hypertext reference of the article to be loaded.
530 *
531 * On exit,
532 * returns >=0 If no error, a good socket number
533 * <0 Error.
534 *
535 * The socket must be closed by the caller after the document has been
536 * read.
537 *
538 */
HTLoadHTTP(const char * arg,HTParentAnchor * anAnchor,HTFormat format_out,HTStream * sink)539 static int HTLoadHTTP(const char *arg,
540 HTParentAnchor *anAnchor,
541 HTFormat format_out,
542 HTStream *sink)
543 {
544 static char empty[1];
545 int s; /* Socket number for returned data */
546 const char *url = arg; /* The URL which get_physical() returned */
547 bstring *command = NULL; /* The whole command */
548 char *eol; /* End of line if found */
549 char *start_of_data; /* Start of body of reply */
550 int status; /* tcp return */
551 off_t bytes_already_read;
552 char crlf[3]; /* A CR LF equivalent string */
553 HTStream *target; /* Unconverted data */
554 HTFormat format_in; /* Format arriving in the message */
555 BOOL do_head = FALSE; /* Whether or not we should do a head */
556 BOOL do_post = FALSE; /* ARE WE posting ? */
557 const char *METHOD;
558
559 char *line_buffer = NULL;
560 char *line_kept_clean = NULL;
561
562 #ifdef SH_EX /* FIX BUG by kaz@maczuka.hitachi.ibaraki.jp */
563 int real_length_of_line = 0;
564 #endif
565 BOOL extensions; /* Assume good HTTP server */
566 char *linebuf = NULL;
567 char temp[80];
568 BOOL first_Accept = TRUE;
569 BOOL show_401 = FALSE;
570 BOOL show_407 = FALSE;
571 BOOL auth_proxy = NO; /* Generate a proxy authorization. - AJL */
572
573 int length, rawlength, rv;
574 int server_status = 0;
575 BOOL doing_redirect, already_retrying = FALSE;
576 int len = 0;
577
578 #ifdef USE_SSL
579 unsigned long SSLerror;
580 BOOL do_connect = FALSE; /* ARE WE going to use a proxy tunnel ? */
581 BOOL did_connect = FALSE; /* ARE WE actually using a proxy tunnel ? */
582 const char *connect_url = NULL; /* The URL being proxied */
583 char *connect_host = NULL; /* The host being proxied */
584 SSL *handle = NULL; /* The SSL handle */
585 X509 *peer_cert; /* The peer certificate */
586 char ssl_dn[1024];
587 char *cert_host;
588 char *ssl_host;
589 char *p;
590 char *msg = NULL;
591 int status_sslcertcheck;
592 char *ssl_dn_start;
593 char *ssl_all_cns = NULL;
594
595 #ifdef USE_GNUTLS_INCL
596 int ret;
597 unsigned tls_status;
598 #endif
599
600 #if SSLEAY_VERSION_NUMBER >= 0x0900
601 BOOL try_tls = TRUE;
602 #endif /* SSLEAY_VERSION_NUMBER >= 0x0900 */
603 SSL_handle = NULL;
604 #else
605 void *handle = NULL;
606 #endif /* USE_SSL */
607
608 if (anAnchor->isHEAD)
609 do_head = TRUE;
610 else if (anAnchor->post_data)
611 do_post = TRUE;
612
613 if (!url) {
614 status = -3;
615 _HTProgress(BAD_REQUEST);
616 goto done;
617 }
618 if (!*url) {
619 status = -2;
620 _HTProgress(BAD_REQUEST);
621 goto done;
622 }
623 #ifdef USE_SSL
624 if (using_proxy && !StrNCmp(url, "http://", 7)) {
625 int portnumber;
626
627 if ((connect_url = strstr((url + 7), "https://"))) {
628 do_connect = TRUE;
629 connect_host = HTParse(connect_url, "https", PARSE_HOST);
630 if (!HTParsePort(connect_host, &portnumber)) {
631 sprintf(temp, ":%d", HTTPS_PORT);
632 StrAllocCat(connect_host, temp);
633 }
634 CTRACE((tfp, "HTTP: connect_url = '%s'\n", connect_url));
635 CTRACE((tfp, "HTTP: connect_host = '%s'\n", connect_host));
636 } else if ((connect_url = strstr((url + 7), "snews://"))) {
637 do_connect = TRUE;
638 connect_host = HTParse(connect_url, "snews", PARSE_HOST);
639 if (!HTParsePort(connect_host, &portnumber)) {
640 sprintf(temp, ":%d", SNEWS_PORT);
641 StrAllocCat(connect_host, temp);
642 }
643 CTRACE((tfp, "HTTP: connect_url = '%s'\n", connect_url));
644 CTRACE((tfp, "HTTP: connect_host = '%s'\n", connect_host));
645 }
646 }
647 #endif /* USE_SSL */
648
649 sprintf(crlf, "%c%c", CR, LF);
650
651 /*
652 * At this point, we're talking HTTP/1.0.
653 */
654 extensions = YES;
655
656 try_again:
657 /*
658 * All initializations are moved down here from up above, so we can start
659 * over here...
660 */
661 eol = 0;
662 length = 0;
663 doing_redirect = FALSE;
664 permanent_redirection = FALSE;
665 redirect_post_content = FALSE;
666 target = NULL;
667 line_buffer = NULL;
668 line_kept_clean = NULL;
669
670 #ifdef USE_SSL
671 if (!StrNCmp(url, "https", 5))
672 status = HTDoConnect(url, "HTTPS", HTTPS_PORT, &s);
673 else
674 status = HTDoConnect(url, "HTTP", HTTP_PORT, &s);
675 #else
676 if (!StrNCmp(url, "https", 5)) {
677 HTAlert(gettext("This client does not contain support for HTTPS URLs."));
678 status = HT_NOT_LOADED;
679 goto done;
680 }
681 status = HTDoConnect(arg, "HTTP", HTTP_PORT, &s);
682 #endif /* USE_SSL */
683 if (status == HT_INTERRUPTED) {
684 /*
685 * Interrupt cleanly.
686 */
687 CTRACE((tfp, "HTTP: Interrupted on connect; recovering cleanly.\n"));
688 _HTProgress(CONNECTION_INTERRUPTED);
689 status = HT_NOT_LOADED;
690 goto done;
691 }
692 if (status < 0) {
693 #ifdef _WINDOWS
694 CTRACE((tfp, "HTTP: Unable to connect to remote host for `%s'\n"
695 " (status = %d, sock_errno = %d).\n",
696 url, status, SOCKET_ERRNO));
697 #else
698 CTRACE((tfp,
699 "HTTP: Unable to connect to remote host for `%s' (errno = %d).\n",
700 url, SOCKET_ERRNO));
701 #endif
702 HTAlert(gettext("Unable to connect to remote host."));
703 status = HT_NOT_LOADED;
704 goto done;
705 }
706 #ifdef USE_SSL
707 use_tunnel:
708 /*
709 * If this is an https document, then do the SSL stuff here.
710 */
711 if (did_connect || !StrNCmp(url, "https", 5)) {
712 SSL_handle = handle = HTGetSSLHandle();
713 SSL_set_fd(handle, s);
714 /* get host we're connecting to */
715 ssl_host = HTParse(url, "", PARSE_HOST);
716 ssl_host = StripIpv6Brackets(ssl_host);
717 #if defined(USE_GNUTLS_FUNCS)
718 ret = gnutls_server_name_set(handle->gnutls_state,
719 GNUTLS_NAME_DNS,
720 ssl_host, strlen(ssl_host));
721 CTRACE((tfp, "...called gnutls_server_name_set(%s) ->%d\n", ssl_host, ret));
722 #elif SSLEAY_VERSION_NUMBER >= 0x0900
723 #ifndef USE_NSS_COMPAT_INCL
724 if (!try_tls) {
725 handle->options |= SSL_OP_NO_TLSv1;
726 #if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT)
727 } else {
728 SSL_set_tlsext_host_name(handle, ssl_host);
729 #endif
730 }
731 #endif
732 #endif /* SSLEAY_VERSION_NUMBER >= 0x0900 */
733 HTSSLInitPRNG();
734 status = SSL_connect(handle);
735
736 if (status <= 0) {
737 #if SSLEAY_VERSION_NUMBER >= 0x0900
738 if (try_tls) {
739 _HTProgress(gettext("Retrying connection without TLS."));
740 try_tls = FALSE;
741 if (did_connect)
742 HTTP_NETCLOSE(s, handle);
743 goto try_again;
744 } else {
745 CTRACE((tfp,
746 "HTTP: Unable to complete SSL handshake for '%s', SSL_connect=%d, SSL error stack dump follows\n",
747 url, status));
748 SSL_load_error_strings();
749 while ((SSLerror = ERR_get_error()) != 0) {
750 CTRACE((tfp, "HTTP: SSL: %s\n", ERR_error_string(SSLerror, NULL)));
751 }
752 HTAlert("Unable to make secure connection to remote host.");
753 if (did_connect)
754 HTTP_NETCLOSE(s, handle);
755 status = HT_NOT_LOADED;
756 goto done;
757 }
758 #else
759 unsigned long SSLerror;
760
761 CTRACE((tfp,
762 "HTTP: Unable to complete SSL handshake for '%s', SSL_connect=%d, SSL error stack dump follows\n",
763 url, status));
764 SSL_load_error_strings();
765 while ((SSLerror = ERR_get_error()) != 0) {
766 CTRACE((tfp, "HTTP: SSL: %s\n", ERR_error_string(SSLerror, NULL)));
767 }
768 HTAlert("Unable to make secure connection to remote host.");
769 if (did_connect)
770 HTTP_NETCLOSE(s, handle);
771 status = HT_NOT_LOADED;
772 goto done;
773 #endif /* SSLEAY_VERSION_NUMBER >= 0x0900 */
774 }
775 #ifdef USE_GNUTLS_INCL
776 gnutls_certificate_set_verify_flags(handle->gnutls_cred,
777 GNUTLS_VERIFY_DO_NOT_ALLOW_SAME |
778 GNUTLS_VERIFY_ALLOW_X509_V1_CA_CRT);
779 ret = gnutls_certificate_verify_peers2(handle->gnutls_state, &tls_status);
780 if (ret < 0 || (ret == 0 &&
781 tls_status & GNUTLS_CERT_SIGNER_NOT_FOUND)) {
782 int flag_continue = 1;
783 char *msg2;
784
785 if (ret == 0 && tls_status & GNUTLS_CERT_SIGNER_NOT_FOUND) {
786 msg2 = gettext("self signed certificate");
787 } else if (tls_status & GNUTLS_CERT_SIGNER_NOT_FOUND) {
788 msg2 = gettext("no issuer was found");
789 } else if (tls_status & GNUTLS_CERT_SIGNER_NOT_CA) {
790 msg2 = gettext("issuer is not a CA");
791 } else if (tls_status & GNUTLS_CERT_REVOKED) {
792 msg2 = gettext("the certificate has been revoked");
793 } else {
794 msg2 = gettext("the certificate is not trusted");
795 }
796 HTSprintf0(&msg, SSL_FORCED_PROMPT, msg2);
797 CTRACE((tfp, "HTLoadHTTP: %s\n", msg));
798 if (!ssl_noprompt) {
799 if (!HTForcedPrompt(ssl_noprompt, msg, YES)) {
800 flag_continue = 0;
801 }
802 } else if (ssl_noprompt == FORCE_PROMPT_NO) {
803 flag_continue = 0;
804 }
805 FREE(msg);
806 if (flag_continue == 0) {
807 status = HT_NOT_LOADED;
808 FREE(msg);
809 goto done;
810 }
811 }
812 #endif
813
814 peer_cert = SSL_get_peer_certificate(handle);
815 #if defined(USE_OPENSSL_INCL) || defined(USE_GNUTLS_FUNCS)
816 X509_NAME_oneline(X509_get_subject_name(peer_cert),
817 ssl_dn, (int) sizeof(ssl_dn));
818 #elif defined(USE_GNUTLS_INCL)
819 X509_NAME_oneline(X509_get_subject_name(peer_cert),
820 ssl_dn + 1, (int) sizeof(ssl_dn) - 1);
821
822 /* Iterate over DN in incompatible GnuTLS format to bring it into OpenSSL format */
823 ssl_dn[0] = '/';
824 ssl_dn_start = ssl_dn;
825 while (*ssl_dn_start) {
826 if ((*ssl_dn_start == ',') && (*(ssl_dn_start + 1) == ' ')) {
827 *ssl_dn_start++ = '/';
828 if (*(p = ssl_dn_start) != 0) {
829 while ((p[0] = p[1]) != 0)
830 ++p;
831 }
832 } else {
833 ssl_dn_start++;
834 }
835 }
836 #endif
837
838 /*
839 * X.509 DN validation taking ALL CN fields into account
840 * (c) 2006 Thorsten Glaser <tg@mirbsd.de>
841 */
842
843 /* initialise status information */
844 status_sslcertcheck = 0; /* 0 = no CN found in DN */
845 ssl_dn_start = ssl_dn;
846
847 /* validate all CNs found in DN */
848 CTRACE((tfp, "Validating CNs in '%s'\n", ssl_dn_start));
849 while ((cert_host = strstr(ssl_dn_start, "/CN=")) != NULL) {
850 status_sslcertcheck = 1; /* 1 = could not verify CN */
851 /* start of CommonName */
852 cert_host += 4;
853 /* find next part of DistinguishedName */
854 if ((p = strchr(cert_host, '/')) != NULL) {
855 *p = '\0';
856 ssl_dn_start = p; /* yes this points to the NUL byte */
857 } else
858 ssl_dn_start = NULL;
859 cert_host = StripIpv6Brackets(cert_host);
860
861 /* verify this CN */
862 CTRACE((tfp, "Matching\n\tssl_host '%s'\n\tcert_host '%s'\n",
863 ssl_host, cert_host));
864 if (!strcasecomp_asterisk(ssl_host, cert_host)) {
865 status_sslcertcheck = 2; /* 2 = verified peer */
866 /* I think this is cool to have in the logs -TG */
867 HTSprintf0(&msg,
868 gettext("Verified connection to %s (cert=%s)"),
869 ssl_host, cert_host);
870 _HTProgress(msg);
871 FREE(msg);
872 /* no need to continue the verification loop */
873 break;
874 }
875
876 /* add this CN to list of failed CNs */
877 if (ssl_all_cns == NULL)
878 StrAllocCopy(ssl_all_cns, "CN<");
879 else
880 StrAllocCat(ssl_all_cns, ":CN<");
881 StrAllocCat(ssl_all_cns, cert_host);
882 StrAllocCat(ssl_all_cns, ">");
883 /* if we cannot retry, don't try it */
884 if (ssl_dn_start == NULL)
885 break;
886 /* now retry next CN found in DN */
887 *ssl_dn_start = '/'; /* formerly NUL byte */
888 }
889
890 /* check the X.509v3 Subject Alternative Name */
891 #ifdef USE_GNUTLS_INCL
892 if (status_sslcertcheck < 2) {
893 int i;
894 size_t size;
895 gnutls_x509_crt cert;
896 static char buf[2048];
897
898 /* import the certificate to the x509_crt format */
899 if (gnutls_x509_crt_init(&cert) == 0) {
900
901 if (gnutls_x509_crt_import(cert, peer_cert,
902 GNUTLS_X509_FMT_DER) < 0) {
903 gnutls_x509_crt_deinit(cert);
904 goto done;
905 }
906
907 ret = 0;
908 for (i = 0; !(ret < 0); i++) {
909 size = sizeof(buf);
910 ret = gnutls_x509_crt_get_subject_alt_name(cert, i, buf,
911 &size, NULL);
912
913 if (strcasecomp_asterisk(ssl_host, buf) == 0) {
914 status_sslcertcheck = 2;
915 HTSprintf0(&msg,
916 gettext("Verified connection to %s (subj=%s)"),
917 ssl_host, buf);
918 _HTProgress(msg);
919 FREE(msg);
920 break;
921 }
922
923 }
924 }
925 }
926 #endif
927 #ifdef USE_OPENSSL_INCL
928 if (status_sslcertcheck < 2) {
929 STACK_OF(GENERAL_NAME) * gens;
930 int i, numalts;
931 const GENERAL_NAME *gn;
932
933 gens = (STACK_OF(GENERAL_NAME) *)
934 X509_get_ext_d2i(peer_cert, NID_subject_alt_name, NULL, NULL);
935
936 if (gens != NULL) {
937 numalts = sk_GENERAL_NAME_num(gens);
938 for (i = 0; i < numalts; ++i) {
939 gn = sk_GENERAL_NAME_value(gens, i);
940 if (gn->type == GEN_DNS)
941 cert_host = (char *) ASN1_STRING_data(gn->d.ia5);
942 else if (gn->type == GEN_IPADD) {
943 /* XXX untested -TG */
944 size_t j = (size_t) ASN1_STRING_length(gn->d.ia5);
945
946 cert_host = (char *) malloc(j + 1);
947 MemCpy(cert_host, ASN1_STRING_data(gn->d.ia5), j);
948 cert_host[j] = '\0';
949 } else
950 continue;
951 status_sslcertcheck = 1; /* got at least one */
952 /* verify this SubjectAltName (see above) */
953 cert_host = StripIpv6Brackets(cert_host);
954 if (!(gn->type == GEN_IPADD ? strcasecomp :
955 strcasecomp_asterisk) (ssl_host, cert_host)) {
956 status_sslcertcheck = 2;
957 HTSprintf0(&msg,
958 gettext("Verified connection to %s (subj=%s)"),
959 ssl_host, cert_host);
960 _HTProgress(msg);
961 FREE(msg);
962 if (gn->type == GEN_IPADD)
963 free(cert_host);
964 break;
965 }
966 /* add to list of failed CNs */
967 if (ssl_all_cns == NULL)
968 StrAllocCopy(ssl_all_cns, "SAN<");
969 else
970 StrAllocCat(ssl_all_cns, ":SAN<");
971 if (gn->type == GEN_DNS)
972 StrAllocCat(ssl_all_cns, "DNS=");
973 else if (gn->type == GEN_IPADD)
974 StrAllocCat(ssl_all_cns, "IP=");
975 StrAllocCat(ssl_all_cns, cert_host);
976 StrAllocCat(ssl_all_cns, ">");
977 if (gn->type == GEN_IPADD)
978 free(cert_host);
979 }
980 sk_GENERAL_NAME_free(gens);
981 }
982 }
983 #endif /* USE_OPENSSL_INCL */
984
985 /* if an error occurred, format the appropriate message */
986 if (status_sslcertcheck == 0) {
987 HTSprintf0(&msg, SSL_FORCED_PROMPT,
988 gettext("Can't find common name in certificate"));
989 } else if (status_sslcertcheck == 1) {
990 HTSprintf0(&msg,
991 gettext("SSL error:host(%s)!=cert(%s)-Continue?"),
992 ssl_host, ssl_all_cns);
993 }
994
995 /* if an error occurred, let the user decide how much he trusts */
996 if (status_sslcertcheck < 2) {
997 if (!HTForcedPrompt(ssl_noprompt, msg, YES)) {
998 status = HT_NOT_LOADED;
999 FREE(msg);
1000 FREE(ssl_all_cns);
1001 goto done;
1002 }
1003 HTSprintf0(&msg,
1004 gettext("UNVERIFIED connection to %s (cert=%s)"),
1005 ssl_host, ssl_all_cns ? ssl_all_cns : "NONE");
1006 _HTProgress(msg);
1007 FREE(msg);
1008 }
1009
1010 show_cert_issuer(peer_cert);
1011
1012 HTSprintf0(&msg,
1013 gettext("Secure %d-bit %s (%s) HTTP connection"),
1014 SSL_get_cipher_bits(handle, NULL),
1015 SSL_get_cipher_version(handle),
1016 SSL_get_cipher(handle));
1017 _HTProgress(msg);
1018 FREE(msg);
1019 }
1020 #endif /* USE_SSL */
1021
1022 /* Ask that node for the document, omitting the host name & anchor
1023 */
1024 {
1025 char *p1 = (HTParse(url, "", PARSE_PATH | PARSE_PUNCTUATION));
1026
1027 #ifdef USE_SSL
1028 if (do_connect) {
1029 METHOD = "CONNECT";
1030 BStrCopy0(command, "CONNECT ");
1031 } else
1032 #endif /* USE_SSL */
1033 if (do_post) {
1034 METHOD = "POST";
1035 BStrCopy0(command, "POST ");
1036 } else if (do_head) {
1037 METHOD = "HEAD";
1038 BStrCopy0(command, "HEAD ");
1039 } else {
1040 METHOD = "GET";
1041 BStrCopy0(command, "GET ");
1042 }
1043
1044 /*
1045 * If we are using a proxy gateway don't copy in the first slash of
1046 * say: /gopher://a;lkdjfl;ajdf;lkj/;aldk/adflj so that just
1047 * gopher://.... is sent.
1048 */
1049 #ifdef USE_SSL
1050 if (using_proxy && !did_connect) {
1051 if (do_connect)
1052 BStrCat0(command, connect_host);
1053 else
1054 BStrCat0(command, p1 + 1);
1055 }
1056 #else
1057 if (using_proxy)
1058 BStrCat0(command, p1 + 1);
1059 #endif /* USE_SSL */
1060 else
1061 BStrCat0(command, p1);
1062 FREE(p1);
1063 }
1064 if (extensions) {
1065 BStrCat0(command, " ");
1066 BStrCat0(command, HTTP_VERSION);
1067 }
1068
1069 BStrCat0(command, crlf); /* CR LF, as in rfc 977 */
1070
1071 if (extensions) {
1072 int n, i;
1073 char *host = NULL;
1074
1075 if ((host = HTParse(anAnchor->address, "", PARSE_HOST)) != NULL) {
1076 strip_userid(host);
1077 HTBprintf(&command, "Host: %s%c%c", host, CR, LF);
1078 FREE(host);
1079 }
1080
1081 if (!HTPresentations)
1082 HTFormatInit();
1083 n = HTList_count(HTPresentations);
1084
1085 first_Accept = TRUE;
1086 len = 0;
1087 for (i = 0; i < n; i++) {
1088 HTPresentation *pres =
1089 (HTPresentation *) HTList_objectAt(HTPresentations, i);
1090
1091 if (pres->get_accept) {
1092 if (pres->quality < 1.0) {
1093 if (pres->maxbytes > 0) {
1094 sprintf(temp, ";q=%4.3f;mxb=%" PRI_off_t "",
1095 pres->quality, CAST_off_t (pres->maxbytes));
1096 } else {
1097 sprintf(temp, ";q=%4.3f", pres->quality);
1098 }
1099 } else if (pres->maxbytes > 0) {
1100 sprintf(temp, ";mxb=%" PRI_off_t "", CAST_off_t (pres->maxbytes));
1101 } else {
1102 temp[0] = '\0';
1103 }
1104 HTSprintf0(&linebuf, "%s%s%s",
1105 (first_Accept ?
1106 "Accept: " : ", "),
1107 HTAtom_name(pres->rep),
1108 temp);
1109 len += (int) strlen(linebuf);
1110 if (len > 252 && !first_Accept) {
1111 BStrCat0(command, crlf);
1112 HTSprintf0(&linebuf, "Accept: %s%s",
1113 HTAtom_name(pres->rep),
1114 temp);
1115 len = (int) strlen(linebuf);
1116 }
1117 BStrCat0(command, linebuf);
1118 first_Accept = FALSE;
1119 }
1120 }
1121 HTBprintf(&command, "%s*/*;q=0.01%c%c",
1122 (first_Accept ?
1123 "Accept: " : ", "), CR, LF);
1124
1125 /*
1126 * FIXME: suppressing the "Accept-Encoding" in this case is done to
1127 * work around limitations of the presentation logic used for the
1128 * command-line "-base" option. The remote site may transmit the
1129 * document gzip'd, but the ensuing logic in HTSaveToFile() would see
1130 * the mime-type as gzip rather than text/html, and not prepend the
1131 * base URL. This is less efficient than accepting the compressed data
1132 * and uncompressing it, adding the base URL but is simpler than
1133 * augmenting the dump's presentation logic -TD
1134 */
1135 if (LYPrependBaseToSource && dump_output_immediately) {
1136 CTRACE((tfp,
1137 "omit Accept-Encoding to work-around interaction with -source\n"));
1138 } else {
1139 char *list = 0;
1140 int j, k;
1141
1142 for (j = 1; j < encodingALL; j <<= 1) {
1143 if (acceptEncoding(j)) {
1144 for (k = 0; tbl_preferred_encoding[k].name != 0; ++k) {
1145 if (tbl_preferred_encoding[k].value == j) {
1146 if (list != 0)
1147 StrAllocCat(list, ", ");
1148 StrAllocCat(list, tbl_preferred_encoding[k].name);
1149 break;
1150 }
1151 }
1152 }
1153 }
1154
1155 if (list != 0) {
1156 HTBprintf(&command, "Accept-Encoding: %s%c%c", list, CR, LF);
1157 free(list);
1158 }
1159 }
1160
1161 if (language && *language) {
1162 HTBprintf(&command, "Accept-Language: %s%c%c", language, CR, LF);
1163 }
1164
1165 if (pref_charset && *pref_charset) {
1166 BStrCat0(command, "Accept-Charset: ");
1167 StrAllocCopy(linebuf, pref_charset);
1168 if (linebuf[strlen(linebuf) - 1] == ',')
1169 linebuf[strlen(linebuf) - 1] = '\0';
1170 LYLowerCase(linebuf);
1171 if (strstr(linebuf, "iso-8859-1") == NULL)
1172 StrAllocCat(linebuf, ", iso-8859-1;q=0.01");
1173 if (strstr(linebuf, "us-ascii") == NULL)
1174 StrAllocCat(linebuf, ", us-ascii;q=0.01");
1175 BStrCat0(command, linebuf);
1176 HTBprintf(&command, "%c%c", CR, LF);
1177 }
1178 #if 0
1179 /*
1180 * Promote 300 (Multiple Choices) replies, if supported, over 406 (Not
1181 * Acceptable) replies. - FM
1182 *
1183 * This used to be done in versions 2.7 and 2.8*, but violates the
1184 * specs for transparent content negotiation and has the effect that
1185 * servers supporting those specs will send 300 (Multiple Choices)
1186 * instead of a normal response (e.g. 200 OK), since they will assume
1187 * that the client wants to make the choice. It is not clear whether
1188 * there are any servers or sites for which sending this header really
1189 * improves anything.
1190 *
1191 * If there ever is a need to send "Negotiate: trans" and really mean
1192 * it, we should send "Negotiate: trans,trans" or similar, since that
1193 * is semantically equivalent and some servers may ignore "Negotiate:
1194 * trans" as a special case when it comes from Lynx (to work around the
1195 * old faulty behavior). - kw
1196 *
1197 * References:
1198 * RFC 2295 (see also RFC 2296), and mail to lynx-dev and
1199 * new-httpd@apache.org from Koen Holtman, Jan 1999.
1200 */
1201 if (!do_post) {
1202 HTBprintf(&command, "Negotiate: trans%c%c", CR, LF);
1203 }
1204 #endif /* 0 */
1205
1206 /*
1207 * When reloading give no-cache pragma to proxy server to make it
1208 * refresh its cache. -- Ari L. <luotonen@dxcern.cern.ch>
1209 *
1210 * Also send it as a Cache-Control header for HTTP/1.1. - FM
1211 */
1212 if (reloading) {
1213 HTBprintf(&command, "Pragma: no-cache%c%c", CR, LF);
1214 HTBprintf(&command, "Cache-Control: no-cache%c%c", CR, LF);
1215 }
1216
1217 if (LYSendUserAgent || no_useragent) {
1218 if (non_empty(LYUserAgent)) {
1219 char *cp = LYSkipBlanks(LYUserAgent);
1220
1221 /* Won't send it at all if all blank - kw */
1222 if (*cp != '\0')
1223 HTBprintf(&command, "User-Agent: %.*s%c%c",
1224 INIT_LINE_SIZE - 15, LYUserAgent, CR, LF);
1225 } else {
1226 HTBprintf(&command, "User-Agent: %s/%s libwww-FM/%s%c%c",
1227 HTAppName ? HTAppName : "unknown",
1228 HTAppVersion ? HTAppVersion : "0.0",
1229 HTLibraryVersion, CR, LF);
1230 }
1231 }
1232
1233 if (personal_mail_address && !LYNoFromHeader) {
1234 HTBprintf(&command, "From: %s%c%c", personal_mail_address, CR, LF);
1235 }
1236
1237 if (!(LYUserSpecifiedURL ||
1238 LYNoRefererHeader || LYNoRefererForThis) &&
1239 strcmp(HTLoadedDocumentURL(), "")) {
1240 const char *cp = LYRequestReferer;
1241
1242 if (!cp)
1243 cp = HTLoadedDocumentURL(); /* @@@ Try both? - kw */
1244 BStrCat0(command, "Referer: ");
1245 if (isLYNXIMGMAP(cp)) {
1246 char *pound = findPoundSelector(cp);
1247 int nn = (pound ? (int) (pound - cp) : (int) strlen(cp));
1248
1249 HTSABCat(&command, cp + LEN_LYNXIMGMAP, nn);
1250 } else {
1251 BStrCat0(command, cp);
1252 }
1253 HTBprintf(&command, "%c%c", CR, LF);
1254 } {
1255 char *abspath;
1256 char *docname;
1257 char *hostname;
1258 char *colon;
1259 int portnumber;
1260 char *auth, *cookie = NULL;
1261 BOOL secure = (BOOL) (StrNCmp(anAnchor->address, "https", 5)
1262 ? FALSE
1263 : TRUE);
1264
1265 abspath = HTParse(arg, "", PARSE_PATH | PARSE_PUNCTUATION);
1266 docname = HTParse(arg, "", PARSE_PATH);
1267 hostname = HTParse(arg, "", PARSE_HOST);
1268 if (hostname &&
1269 NULL != (colon = HTParsePort(hostname, &portnumber))) {
1270 *colon = '\0'; /* Chop off port number */
1271 } else if (!StrNCmp(arg, "https", 5)) {
1272 portnumber = HTTPS_PORT;
1273 } else {
1274 portnumber = HTTP_PORT;
1275 }
1276
1277 /*
1278 * Add Authorization, Proxy-Authorization, and/or Cookie headers,
1279 * if applicable.
1280 */
1281 if (using_proxy) {
1282 /*
1283 * If we are using a proxy, first determine if we should
1284 * include an Authorization header and/or Cookie header for the
1285 * ultimate target of this request. - FM & AJL
1286 */
1287 char *host2 = NULL, *path2 = NULL;
1288 int port2 = (StrNCmp(docname, "https", 5) ?
1289 HTTP_PORT : HTTPS_PORT);
1290
1291 host2 = HTParse(docname, "", PARSE_HOST);
1292 path2 = HTParse(docname, "", PARSE_PATH | PARSE_PUNCTUATION);
1293 if (host2) {
1294 if ((colon = HTParsePort(host2, &port2)) != NULL) {
1295 /* Use non-default port number */
1296 *colon = '\0';
1297 }
1298 }
1299 /*
1300 * This composeAuth() does file access, i.e., for the ultimate
1301 * target of the request. - AJL
1302 */
1303 auth_proxy = NO;
1304 if ((auth = HTAA_composeAuth(host2, port2, path2,
1305 auth_proxy)) != NULL &&
1306 *auth != '\0') {
1307 /*
1308 * If auth is not NULL nor zero-length, it's an
1309 * Authorization header to be included. - FM
1310 */
1311 HTBprintf(&command, "%s%c%c", auth, CR, LF);
1312 CTRACE((tfp, "HTTP: Sending authorization: %s\n", auth));
1313 } else if (auth && *auth == '\0') {
1314 /*
1315 * If auth is a zero-length string, the user either
1316 * cancelled or goofed at the username and password prompt.
1317 * - FM
1318 */
1319 if (!(traversal || dump_output_immediately) &&
1320 HTConfirm(CONFIRM_WO_PASSWORD)) {
1321 show_401 = TRUE;
1322 } else {
1323 if (traversal || dump_output_immediately)
1324 HTAlert(FAILED_NEED_PASSWD);
1325 #ifdef USE_SSL
1326 if (did_connect)
1327 HTTP_NETCLOSE(s, handle);
1328 #endif /* USE_SSL */
1329 BStrFree(command);
1330 FREE(hostname);
1331 FREE(docname);
1332 FREE(abspath);
1333 FREE(host2);
1334 FREE(path2);
1335 status = HT_NOT_LOADED;
1336 goto done;
1337 }
1338 } else {
1339 CTRACE((tfp, "HTTP: Not sending authorization (yet).\n"));
1340 }
1341 /*
1342 * Add 'Cookie:' header, if it's HTTP or HTTPS document being
1343 * proxied.
1344 */
1345 if (!StrNCmp(docname, "http", 4)) {
1346 cookie = LYAddCookieHeader(host2, path2, port2, secure);
1347 }
1348 FREE(host2);
1349 FREE(path2);
1350 /*
1351 * The next composeAuth() will be for the proxy. - AJL
1352 */
1353 auth_proxy = YES;
1354 } else {
1355 /*
1356 * Add cookie for a non-proxied request. - FM
1357 */
1358 cookie = LYAddCookieHeader(hostname, abspath, portnumber, secure);
1359 auth_proxy = NO;
1360 }
1361 /*
1362 * If we do have a cookie set, add it to the request buffer. - FM
1363 */
1364 if (cookie != NULL) {
1365 if (*cookie != '$') {
1366 /*
1367 * It's a historical cookie, so signal to the server that
1368 * we support modern cookies. - FM
1369 */
1370 BStrCat0(command, "Cookie2: $Version=\"1\"");
1371 BStrCat0(command, crlf);
1372 CTRACE((tfp, "HTTP: Sending Cookie2: $Version =\"1\"\n"));
1373 }
1374 if (*cookie != '\0') {
1375 /*
1376 * It's not a zero-length string, so add the header. Note
1377 * that any folding of long strings has been done already
1378 * in LYCookie.c. - FM
1379 */
1380 BStrCat0(command, "Cookie: ");
1381 BStrCat0(command, cookie);
1382 BStrCat0(command, crlf);
1383 CTRACE((tfp, "HTTP: Sending Cookie: %s\n", cookie));
1384 }
1385 FREE(cookie);
1386 }
1387 FREE(abspath);
1388
1389 /*
1390 * If we are using a proxy, auth_proxy should be YES, and we check
1391 * here whether we want a Proxy-Authorization header for it. If we
1392 * are not using a proxy, auth_proxy should still be NO, and we
1393 * check here for whether we want an Authorization header. - FM &
1394 * AJL
1395 */
1396 if ((auth = HTAA_composeAuth(hostname,
1397 portnumber,
1398 docname,
1399 auth_proxy)) != NULL &&
1400 *auth != '\0') {
1401 /*
1402 * If auth is not NULL nor zero-length, it's an Authorization
1403 * or Proxy-Authorization header to be included. - FM
1404 */
1405 HTBprintf(&command, "%s%c%c", auth, CR, LF);
1406 CTRACE((tfp, (auth_proxy ?
1407 "HTTP: Sending proxy authorization: %s\n" :
1408 "HTTP: Sending authorization: %s\n"),
1409 auth));
1410 } else if (auth && *auth == '\0') {
1411 /*
1412 * If auth is a zero-length string, the user either cancelled
1413 * or goofed at the username and password prompt. - FM
1414 */
1415 if (!(traversal || dump_output_immediately) && HTConfirm(CONFIRM_WO_PASSWORD)) {
1416 if (auth_proxy == TRUE) {
1417 show_407 = TRUE;
1418 } else {
1419 show_401 = TRUE;
1420 }
1421 } else {
1422 if (traversal || dump_output_immediately)
1423 HTAlert(FAILED_NEED_PASSWD);
1424 BStrFree(command);
1425 FREE(hostname);
1426 FREE(docname);
1427 status = HT_NOT_LOADED;
1428 goto done;
1429 }
1430 } else {
1431 CTRACE((tfp, (auth_proxy ?
1432 "HTTP: Not sending proxy authorization (yet).\n" :
1433 "HTTP: Not sending authorization (yet).\n")));
1434 }
1435 FREE(hostname);
1436 FREE(docname);
1437 }
1438 }
1439
1440 if (
1441 #ifdef USE_SSL
1442 !do_connect &&
1443 #endif /* USE_SSL */
1444 do_post) {
1445 CTRACE((tfp, "HTTP: Doing post, content-type '%s'\n",
1446 anAnchor->post_content_type
1447 ? anAnchor->post_content_type
1448 : "lose"));
1449 HTBprintf(&command, "Content-type: %s%c%c",
1450 anAnchor->post_content_type
1451 ? anAnchor->post_content_type
1452 : "lose",
1453 CR, LF);
1454
1455 HTBprintf(&command, "Content-length: %d%c%c",
1456 !isBEmpty(anAnchor->post_data)
1457 ? BStrLen(anAnchor->post_data)
1458 : 0,
1459 CR, LF);
1460
1461 BStrCat0(command, crlf); /* Blank line means "end" of headers */
1462
1463 BStrCat(command, anAnchor->post_data);
1464 } else
1465 BStrCat0(command, crlf); /* Blank line means "end" of headers */
1466
1467 if (TRACE) {
1468 CTRACE((tfp, "Writing:\n"));
1469 trace_bstring(command);
1470 #ifdef USE_SSL
1471 CTRACE((tfp, "%s",
1472 (anAnchor->post_data && !do_connect ? crlf : "")));
1473 #else
1474 CTRACE((tfp, "%s",
1475 (anAnchor->post_data ? crlf : "")));
1476 #endif /* USE_SSL */
1477 CTRACE((tfp, "----------------------------------\n"));
1478 }
1479
1480 _HTProgress(gettext("Sending HTTP request."));
1481
1482 #ifdef NOT_ASCII /* S/390 -- gil -- 0548 */
1483 {
1484 char *p2;
1485
1486 for (p2 = BStrData(command);
1487 p2 < BStrData(command) + BStrLen(command);
1488 p2++)
1489 *p2 = TOASCII(*p2);
1490 }
1491 #endif /* NOT_ASCII */
1492 status = (int) HTTP_NETWRITE(s,
1493 BStrData(command),
1494 BStrLen(command),
1495 handle);
1496 BStrFree(command);
1497 FREE(linebuf);
1498 if (status <= 0) {
1499 if (status == 0) {
1500 CTRACE((tfp, "HTTP: Got status 0 in initial write\n"));
1501 /* Do nothing. */
1502 } else if ((SOCKET_ERRNO == ENOTCONN ||
1503 SOCKET_ERRNO == ECONNRESET ||
1504 SOCKET_ERRNO == EPIPE) &&
1505 !already_retrying &&
1506 /* Don't retry if we're posting. */ !do_post) {
1507 /*
1508 * Arrrrgh, HTTP 0/1 compatibility problem, maybe.
1509 */
1510 CTRACE((tfp,
1511 "HTTP: BONZO ON WRITE Trying again with HTTP0 request.\n"));
1512 _HTProgress(RETRYING_AS_HTTP0);
1513 HTTP_NETCLOSE(s, handle);
1514 extensions = NO;
1515 already_retrying = TRUE;
1516 goto try_again;
1517 } else {
1518 CTRACE((tfp,
1519 "HTTP: Hit unexpected network WRITE error; aborting connection.\n"));
1520 HTTP_NETCLOSE(s, handle);
1521 status = -1;
1522 HTAlert(gettext("Unexpected network write error; connection aborted."));
1523 goto done;
1524 }
1525 }
1526
1527 CTRACE((tfp, "HTTP: WRITE delivered OK\n"));
1528 _HTProgress(gettext("HTTP request sent; waiting for response."));
1529
1530 /* Read the first line of the response
1531 * -----------------------------------
1532 */
1533 {
1534 /* Get numeric status etc */
1535 BOOL end_of_file = NO;
1536 int buffer_length = INIT_LINE_SIZE;
1537
1538 line_buffer = typecallocn(char, (size_t) buffer_length);
1539
1540 if (line_buffer == NULL)
1541 outofmem(__FILE__, "HTLoadHTTP");
1542
1543 HTReadProgress(bytes_already_read = 0, (off_t) 0);
1544 do { /* Loop to read in the first line */
1545 /*
1546 * Extend line buffer if necessary for those crazy WAIS URLs ;-)
1547 */
1548 if (buffer_length - length < LINE_EXTEND_THRESH) {
1549 buffer_length = buffer_length + buffer_length;
1550 line_buffer =
1551 (char *) realloc(line_buffer, ((unsigned) buffer_length *
1552 sizeof(char)));
1553
1554 if (line_buffer == NULL)
1555 outofmem(__FILE__, "HTLoadHTTP");
1556 }
1557 CTRACE((tfp, "HTTP: Trying to read %d\n", buffer_length - length - 1));
1558 status = HTTP_NETREAD(s,
1559 line_buffer + length,
1560 (buffer_length - length - 1),
1561 handle);
1562 CTRACE((tfp, "HTTP: Read %d\n", status));
1563 if (status <= 0) {
1564 /*
1565 * Retry if we get nothing back too.
1566 * Bomb out if we get nothing twice.
1567 */
1568 if (status == HT_INTERRUPTED) {
1569 CTRACE((tfp, "HTTP: Interrupted initial read.\n"));
1570 _HTProgress(CONNECTION_INTERRUPTED);
1571 HTTP_NETCLOSE(s, handle);
1572 status = HT_NO_DATA;
1573 goto clean_up;
1574 } else if (status < 0 &&
1575 (SOCKET_ERRNO == ENOTCONN ||
1576 #ifdef _WINDOWS /* 1997/11/09 (Sun) 16:59:58 */
1577 SOCKET_ERRNO == ETIMEDOUT ||
1578 #endif
1579 SOCKET_ERRNO == ECONNRESET ||
1580 SOCKET_ERRNO == EPIPE) &&
1581 !already_retrying && !do_post) {
1582 /*
1583 * Arrrrgh, HTTP 0/1 compability problem, maybe.
1584 */
1585 CTRACE((tfp,
1586 "HTTP: BONZO Trying again with HTTP0 request.\n"));
1587 HTTP_NETCLOSE(s, handle);
1588 FREE(line_buffer);
1589 FREE(line_kept_clean);
1590
1591 extensions = NO;
1592 already_retrying = TRUE;
1593 _HTProgress(RETRYING_AS_HTTP0);
1594 goto try_again;
1595 }
1596 #ifdef USE_SSL
1597 else if ((SSLerror = ERR_get_error()) != 0) {
1598 CTRACE((tfp,
1599 "HTTP: Hit unexpected network read error; aborting connection; status %d:%s.\n",
1600 status, ERR_error_string(SSLerror, NULL)));
1601 HTAlert(gettext("Unexpected network read error; connection aborted."));
1602 HTTP_NETCLOSE(s, handle);
1603 status = -1;
1604 goto clean_up;
1605 }
1606 #endif
1607 else {
1608 CTRACE((tfp,
1609 "HTTP: Hit unexpected network read error; aborting connection; status %d.\n",
1610 status));
1611 HTAlert(gettext("Unexpected network read error; connection aborted."));
1612 HTTP_NETCLOSE(s, handle);
1613 status = -1;
1614 goto clean_up;
1615 }
1616 }
1617 #ifdef NOT_ASCII /* S/390 -- gil -- 0564 */
1618 {
1619 char *p2;
1620
1621 for (p2 = line_buffer + length;
1622 p2 < line_buffer + length + status;
1623 p2++)
1624 *p2 = FROMASCII(*p2);
1625 }
1626 #endif /* NOT_ASCII */
1627
1628 bytes_already_read += status;
1629 HTReadProgress(bytes_already_read, (off_t) 0);
1630
1631 #ifdef UCX /* UCX returns -1 on EOF */
1632 if (status == 0 || status == -1)
1633 #else
1634 if (status == 0)
1635 #endif
1636 {
1637 break;
1638 }
1639 line_buffer[length + status] = 0;
1640
1641 if (line_buffer) {
1642 FREE(line_kept_clean);
1643 line_kept_clean = (char *) malloc((unsigned) buffer_length *
1644 sizeof(char));
1645
1646 if (line_kept_clean == NULL)
1647 outofmem(__FILE__, "HTLoadHTTP");
1648 MemCpy(line_kept_clean, line_buffer, buffer_length);
1649 #ifdef SH_EX /* FIX BUG by kaz@maczuka.hitachi.ibaraki.jp */
1650 real_length_of_line = length + status;
1651 #endif
1652 }
1653
1654 eol = strchr(line_buffer + length, LF);
1655 /* Do we *really* want to do this? */
1656 if (eol && eol != line_buffer && *(eol - 1) == CR)
1657 *(eol - 1) = ' ';
1658
1659 length = length + status;
1660
1661 /* Do we really want to do *this*? */
1662 if (eol)
1663 *eol = 0; /* Terminate the line */
1664 }
1665 /* All we need is the first line of the response. If it's a HTTP/1.0
1666 * response, then the first line will be absurdly short and therefore
1667 * we can safely gate the number of bytes read through this code (as
1668 * opposed to below) to ~1000.
1669 *
1670 * Well, let's try 100.
1671 */
1672 while (!eol && !end_of_file && bytes_already_read < 100);
1673 } /* Scope of loop variables */
1674
1675 /* save total length, in case we decide later to show it all - kw */
1676 rawlength = length;
1677
1678 /* We now have a terminated unfolded line. Parse it.
1679 * --------------------------------------------------
1680 */
1681 CTRACE((tfp, "HTTP: Rx: %s\n", line_buffer));
1682
1683 /*
1684 * Kludge to work with old buggy servers and the VMS Help gateway. They
1685 * can't handle the third word, so we try again without it.
1686 */
1687 if (extensions && /* Old buggy server or Help gateway? */
1688 (0 == StrNCmp(line_buffer, "<TITLE>Bad File Request</TITLE>", 31) ||
1689 0 == StrNCmp(line_buffer, "Address should begin with", 25) ||
1690 0 == StrNCmp(line_buffer, "<TITLE>Help ", 12) ||
1691 0 == strcmp(line_buffer,
1692 "Document address invalid or access not authorised"))) {
1693 FREE(line_buffer);
1694 FREE(line_kept_clean);
1695 extensions = NO;
1696 already_retrying = TRUE;
1697 CTRACE((tfp, "HTTP: close socket %d to retry with HTTP0\n", s));
1698 HTTP_NETCLOSE(s, handle);
1699 /* print a progress message */
1700 _HTProgress(RETRYING_AS_HTTP0);
1701 goto try_again;
1702 } {
1703 int fields;
1704 char server_version[VERSION_LENGTH + 1];
1705
1706 server_version[0] = 0;
1707
1708 fields = sscanf(line_buffer, "%20s %d",
1709 server_version,
1710 &server_status);
1711
1712 CTRACE((tfp, "HTTP: Scanned %d fields from line_buffer\n", fields));
1713
1714 if (http_error_file) { /* Make the status code externally available */
1715 FILE *error_file;
1716
1717 #ifdef SERVER_STATUS_ONLY
1718 error_file = fopen(http_error_file, TXT_W);
1719 if (error_file) { /* Managed to open the file */
1720 fprintf(error_file, "error=%d\n", server_status);
1721 fclose(error_file);
1722 }
1723 #else
1724 error_file = fopen(http_error_file, TXT_A);
1725 if (error_file) { /* Managed to open the file */
1726 fprintf(error_file, " URL=%s (%s)\n", url, METHOD);
1727 fprintf(error_file, "STATUS=%s\n", line_buffer);
1728 fclose(error_file);
1729 }
1730 #endif /* SERVER_STATUS_ONLY */
1731 }
1732
1733 /*
1734 * Rule out a non-HTTP/1.n reply as best we can.
1735 */
1736 if (fields < 2 || !server_version[0] || server_version[0] != 'H' ||
1737 server_version[1] != 'T' || server_version[2] != 'T' ||
1738 server_version[3] != 'P' || server_version[4] != '/' ||
1739 server_version[6] != '.') {
1740 /*
1741 * Ugh! An HTTP0 reply,
1742 */
1743 HTAtom *encoding;
1744
1745 CTRACE((tfp, "--- Talking HTTP0.\n"));
1746
1747 format_in = HTFileFormat(url, &encoding, NULL);
1748 /*
1749 * Treat all plain text as HTML. This sucks but its the only
1750 * solution without without looking at content.
1751 */
1752 if (!StrNCmp(HTAtom_name(format_in), "text/plain", 10)) {
1753 CTRACE((tfp, "HTTP: format_in being changed to text/HTML\n"));
1754 format_in = WWW_HTML;
1755 }
1756 if (!IsUnityEnc(encoding)) {
1757 /*
1758 * Change the format to that for "www/compressed".
1759 */
1760 CTRACE((tfp, "HTTP: format_in is '%s',\n", HTAtom_name(format_in)));
1761 StrAllocCopy(anAnchor->content_type, HTAtom_name(format_in));
1762 StrAllocCopy(anAnchor->content_encoding, HTAtom_name(encoding));
1763 format_in = HTAtom_for("www/compressed");
1764 CTRACE((tfp, " Treating as '%s' with encoding '%s'\n",
1765 "www/compressed", HTAtom_name(encoding)));
1766 }
1767
1768 start_of_data = line_kept_clean;
1769 } else {
1770 /*
1771 * Set up to decode full HTTP/1.n response. - FM
1772 */
1773 format_in = HTAtom_for("www/mime");
1774 CTRACE((tfp, "--- Talking HTTP1.\n"));
1775
1776 /*
1777 * We set start_of_data to "" when !eol here because there will be
1778 * a put_block done below; we do *not* use the value of
1779 * start_of_data (as a pointer) in the computation of length (or
1780 * anything else) when !eol. Otherwise, set the value of length to
1781 * what we have beyond eol (i.e., beyond the status line). - FM
1782 */
1783 if (eol != 0) {
1784 start_of_data = (eol + 1);
1785 } else {
1786 start_of_data = empty;
1787 }
1788 length = (eol
1789 ? length - (int) (start_of_data - line_buffer)
1790 : 0);
1791
1792 /*
1793 * Trim trailing spaces in line_buffer so that we can use it in
1794 * messages which include the status line. - FM
1795 */
1796 while (line_buffer[strlen(line_buffer) - 1] == ' ')
1797 line_buffer[strlen(line_buffer) - 1] = '\0';
1798
1799 /*
1800 * Take appropriate actions based on the status. - FM
1801 */
1802 switch (server_status / 100) {
1803 case 1:
1804 /*
1805 * HTTP/1.1 Informational statuses.
1806 * 100 Continue.
1807 * 101 Switching Protocols.
1808 * > 101 is unknown.
1809 * We should never get these, and they have only the status
1810 * line and possibly other headers, so we'll deal with them by
1811 * showing the full header to the user as text/plain. - FM
1812 */
1813 HTAlert(gettext("Got unexpected Informational Status."));
1814 do_head = TRUE;
1815 break;
1816
1817 case 2:
1818 /*
1819 * Good: Got MIME object! (Successful) - FM
1820 */
1821 if (do_head) {
1822 /*
1823 * If HEAD was requested, show headers (and possibly bogus
1824 * body) for all 2xx status codes as text/plain - KW
1825 */
1826 HTProgress(line_buffer);
1827 break;
1828 }
1829 switch (server_status) {
1830 case 204:
1831 /*
1832 * No Content.
1833 */
1834 HTAlert(line_buffer);
1835 HTTP_NETCLOSE(s, handle);
1836 HTNoDataOK = 1;
1837 status = HT_NO_DATA;
1838 goto clean_up;
1839
1840 case 205:
1841 /*
1842 * Reset Content. The server has fulfilled the request but
1843 * nothing is returned and we should reset any form
1844 * content. We'll instruct the user to do that, and
1845 * restore the current document. - FM
1846 */
1847 HTAlert(gettext("Request fulfilled. Reset Content."));
1848 HTTP_NETCLOSE(s, handle);
1849 status = HT_NO_DATA;
1850 goto clean_up;
1851
1852 case 206:
1853 /*
1854 * Partial Content. We didn't send a Range so something
1855 * went wrong somewhere. Show the status message and
1856 * restore the current document. - FM
1857 */
1858 HTAlert(line_buffer);
1859 HTTP_NETCLOSE(s, handle);
1860 status = HT_NO_DATA;
1861 goto clean_up;
1862
1863 default:
1864 /*
1865 * 200 OK.
1866 * 201 Created.
1867 * 202 Accepted.
1868 * 203 Non-Authoritative Information.
1869 * > 206 is unknown.
1870 * All should return something to display.
1871 */
1872 #if defined(USE_SSL) && !defined(DISABLE_NEWS)
1873 if (do_connect) {
1874 CTRACE((tfp,
1875 "HTTP: Proxy tunnel to '%s' established.\n",
1876 connect_host));
1877 do_connect = FALSE;
1878 url = connect_url;
1879 FREE(line_buffer);
1880 FREE(line_kept_clean);
1881 if (!StrNCmp(connect_url, "snews", 5)) {
1882 CTRACE((tfp,
1883 " Will attempt handshake and snews connection.\n"));
1884 status = HTNewsProxyConnect(s, url, anAnchor,
1885 format_out, sink);
1886 goto done;
1887 }
1888 did_connect = TRUE;
1889 already_retrying = TRUE;
1890 eol = 0;
1891 length = 0;
1892 doing_redirect = FALSE;
1893 permanent_redirection = FALSE;
1894 target = NULL;
1895 CTRACE((tfp,
1896 " Will attempt handshake and resubmit headers.\n"));
1897 goto use_tunnel;
1898 }
1899 #endif /* USE_SSL */
1900 HTProgress(line_buffer);
1901 } /* case 2 switch */
1902 break;
1903
1904 case 3:
1905 /*
1906 * Various forms of Redirection. - FM
1907 * 300 Multiple Choices.
1908 * 301 Moved Permanently.
1909 * 302 Found (temporary; we can, and do, use GET).
1910 * 303 See Other (temporary; always use GET).
1911 * 304 Not Modified.
1912 * 305 Use Proxy.
1913 * 306 Set Proxy.
1914 * 307 Temporary Redirect with method retained.
1915 * > 308 is unknown.
1916 */
1917 if (no_url_redirection || do_head || keep_mime_headers) {
1918 /*
1919 * If any of these flags are set, we do not redirect, but
1920 * instead show what was returned to the user as
1921 * text/plain. - FM
1922 */
1923 HTProgress(line_buffer);
1924 break;
1925 }
1926
1927 if (server_status == 300) { /* Multiple Choices */
1928 /*
1929 * For client driven content negotiation. The server
1930 * should be sending some way for the user-agent to make a
1931 * selection, so we'll show the user whatever the server
1932 * returns. There might be a Location: header with the
1933 * server's preference present, but the choice should be up
1934 * to the user, someday based on an Alternates: header,
1935 * and a body always should be present with descriptions
1936 * and links for the choices (i.e., we use the latter, for
1937 * now). - FM
1938 */
1939 HTAlert(line_buffer);
1940 if (traversal) {
1941 HTTP_NETCLOSE(s, handle);
1942 status = -1;
1943 goto clean_up;
1944 }
1945 if (!dump_output_immediately &&
1946 format_out == HTAtom_for("www/download")) {
1947 /*
1948 * Convert a download request to a presentation request
1949 * for interactive users. - FM
1950 */
1951 format_out = WWW_PRESENT;
1952 }
1953 break;
1954 }
1955
1956 if (server_status == 304) { /* Not Modified */
1957 /*
1958 * We didn't send an "If-Modified-Since" header, so this
1959 * status is inappropriate. We'll deal with it by showing
1960 * the full header to the user as text/plain. - FM
1961 */
1962 HTAlert(gettext("Got unexpected 304 Not Modified status."));
1963 do_head = TRUE;
1964 break;
1965 }
1966
1967 if (server_status == 305 ||
1968 server_status == 306 ||
1969 server_status > 307) {
1970 /*
1971 * Show user the content, if any, for 305, 306, or unknown
1972 * status. - FM
1973 */
1974 HTAlert(line_buffer);
1975 if (traversal) {
1976 HTTP_NETCLOSE(s, handle);
1977 status = -1;
1978 goto clean_up;
1979 }
1980 if (!dump_output_immediately &&
1981 format_out == HTAtom_for("www/download")) {
1982 /*
1983 * Convert a download request to a presentation request
1984 * for interactive users. - FM
1985 */
1986 format_out = WWW_PRESENT;
1987 }
1988 break;
1989 }
1990
1991 /*
1992 * We do not load the file, but read the headers for the
1993 * "Location:", check out that redirecting_url and if it's
1994 * acceptible (e.g., not a telnet URL when we have that
1995 * disabled), initiate a new fetch. If that's another
1996 * redirecting_url, we'll repeat the checks, and fetch
1997 * initiations if acceptible, until we reach the actual URL, or
1998 * the redirection limit set in HTAccess.c is exceeded. If the
1999 * status was 301 indicating that the relocation is permanent,
2000 * we set the permanent_redirection flag to make it permanent
2001 * for the current anchor tree (i.e., will persist until the
2002 * tree is freed or the client exits). If the redirection
2003 * would include POST content, we seek confirmation from an
2004 * interactive user, with option to use 303 for 301 (but not
2005 * for 307), and otherwise refuse the redirection. We also
2006 * don't allow permanent redirection if we keep POST content.
2007 * If we don't find the Location header or its value is
2008 * zero-length, we display whatever the server returned, and
2009 * the user should RELOAD that to try again, or make a
2010 * selection from it if it contains links, or Left-Arrow to the
2011 * previous document. - FM
2012 */
2013 {
2014 if ((dump_output_immediately || traversal) &&
2015 do_post &&
2016 server_status != 303 &&
2017 server_status != 302 &&
2018 server_status != 301) {
2019 /*
2020 * Don't redirect POST content without approval from an
2021 * interactive user. - FM
2022 */
2023 HTTP_NETCLOSE(s, handle);
2024 status = -1;
2025 HTAlert(gettext("Redirection of POST content requires user approval."));
2026 if (traversal)
2027 HTProgress(line_buffer);
2028 goto clean_up;
2029 }
2030
2031 HTProgress(line_buffer);
2032 if (server_status == 301) { /* Moved Permanently */
2033 if (do_post) {
2034 /*
2035 * Don't make the redirection permanent if we have
2036 * POST content. - FM
2037 */
2038 CTRACE((tfp,
2039 "HTTP: Have POST content. Treating 301 (Permanent) as Temporary.\n"));
2040 HTAlert(gettext("Have POST content. Treating Permanent Redirection as Temporary.\n"));
2041 } else {
2042 permanent_redirection = TRUE;
2043 }
2044 }
2045 doing_redirect = TRUE;
2046
2047 break;
2048 }
2049
2050 case 4:
2051 /*
2052 * "I think I goofed!" (Client Error) - FM
2053 */
2054 switch (server_status) {
2055 case 401: /* Unauthorized */
2056 /*
2057 * Authorization for origin server required. If show_401
2058 * is set, proceed to showing the 401 body. Otherwise, if
2059 * we can set up authorization based on the
2060 * WWW-Authenticate header, and the user provides a
2061 * username and password, try again. Otherwise, check
2062 * whether to show the 401 body or restore the current
2063 * document - FM
2064 */
2065 if (show_401)
2066 break;
2067 if (HTAA_shouldRetryWithAuth(start_of_data, (size_t)
2068 length, s, NO)) {
2069
2070 HTTP_NETCLOSE(s, handle);
2071 if (dump_output_immediately && !authentication_info[0]) {
2072 fprintf(stderr,
2073 "HTTP: Access authorization required.\n");
2074 fprintf(stderr,
2075 " Use the -auth=id:pw parameter.\n");
2076 status = HT_NO_DATA;
2077 goto clean_up;
2078 }
2079
2080 CTRACE((tfp, "%s %d %s\n",
2081 "HTTP: close socket", s,
2082 "to retry with Access Authorization"));
2083
2084 _HTProgress(gettext("Retrying with access authorization information."));
2085 FREE(line_buffer);
2086 FREE(line_kept_clean);
2087 #ifdef USE_SSL
2088 if (using_proxy && !StrNCmp(url, "https://", 8)) {
2089 url = arg;
2090 do_connect = TRUE;
2091 did_connect = FALSE;
2092 }
2093 #endif /* USE_SSL */
2094 goto try_again;
2095 } else if (!(traversal || dump_output_immediately) &&
2096 HTConfirm(gettext("Show the 401 message body?"))) {
2097 break;
2098 } else {
2099 if (traversal || dump_output_immediately)
2100 HTAlert(FAILED_RETRY_WITH_AUTH);
2101 HTTP_NETCLOSE(s, handle);
2102 status = -1;
2103 goto clean_up;
2104 }
2105
2106 case 407:
2107 /*
2108 * Authorization for proxy server required. If we are not
2109 * in fact using a proxy, or show_407 is set, proceed to
2110 * showing the 407 body. Otherwise, if we can set up
2111 * authorization based on the Proxy-Authenticate header,
2112 * and the user provides a username and password, try
2113 * again. Otherwise, check whether to show the 401 body or
2114 * restore the current document. - FM & AJL
2115 */
2116 if (!using_proxy || show_407)
2117 break;
2118 if (HTAA_shouldRetryWithAuth(start_of_data, (size_t)
2119 length, s, YES)) {
2120
2121 HTTP_NETCLOSE(s, handle);
2122 if (dump_output_immediately && !proxyauth_info[0]) {
2123 fprintf(stderr,
2124 "HTTP: Proxy authorization required.\n");
2125 fprintf(stderr,
2126 " Use the -pauth=id:pw parameter.\n");
2127 status = HT_NO_DATA;
2128 goto clean_up;
2129 }
2130
2131 CTRACE((tfp, "%s %d %s\n",
2132 "HTTP: close socket", s,
2133 "to retry with Proxy Authorization"));
2134
2135 _HTProgress(HTTP_RETRY_WITH_PROXY);
2136 FREE(line_buffer);
2137 FREE(line_kept_clean);
2138 goto try_again;
2139 } else if (!(traversal || dump_output_immediately) &&
2140 HTConfirm(gettext("Show the 407 message body?"))) {
2141 if (!dump_output_immediately &&
2142 format_out == HTAtom_for("www/download")) {
2143 /*
2144 * Convert a download request to a presentation
2145 * request for interactive users. - FM
2146 */
2147 format_out = WWW_PRESENT;
2148 }
2149 break;
2150 } else {
2151 if (traversal || dump_output_immediately)
2152 HTAlert(FAILED_RETRY_WITH_PROXY);
2153 HTTP_NETCLOSE(s, handle);
2154 status = -1;
2155 goto clean_up;
2156 }
2157
2158 case 408:
2159 /*
2160 * Request Timeout. Show the status message and restore
2161 * the current document. - FM
2162 */
2163 HTAlert(line_buffer);
2164 HTTP_NETCLOSE(s, handle);
2165 status = HT_NO_DATA;
2166 goto clean_up;
2167
2168 default:
2169 /*
2170 * 400 Bad Request.
2171 * 402 Payment Required.
2172 * 403 Forbidden.
2173 * 404 Not Found.
2174 * 405 Method Not Allowed.
2175 * 406 Not Acceptable.
2176 * 409 Conflict.
2177 * 410 Gone.
2178 * 411 Length Required.
2179 * 412 Precondition Failed.
2180 * 413 Request Entity Too Large.
2181 * 414 Request-URI Too Long.
2182 * 415 Unsupported Media Type.
2183 * 416 List Response (for content negotiation).
2184 * > 416 is unknown.
2185 * Show the status message, and display the returned text
2186 * if we are not doing a traversal. - FM
2187 */
2188 HTAlert(line_buffer);
2189 if (traversal) {
2190 HTTP_NETCLOSE(s, handle);
2191 status = -1;
2192 goto clean_up;
2193 }
2194 if (!dump_output_immediately &&
2195 format_out == HTAtom_for("www/download")) {
2196 /*
2197 * Convert a download request to a presentation request
2198 * for interactive users. - FM
2199 */
2200 format_out = WWW_PRESENT;
2201 }
2202 break;
2203 } /* case 4 switch */
2204 break;
2205
2206 case 5:
2207 /*
2208 * "I think YOU goofed!" (server error)
2209 * 500 Internal Server Error
2210 * 501 Not Implemented
2211 * 502 Bad Gateway
2212 * 503 Service Unavailable
2213 * 504 Gateway Timeout
2214 * 505 HTTP Version Not Supported
2215 * > 505 is unknown.
2216 * Should always include a message, which we always should
2217 * display. - FM
2218 */
2219 HTAlert(line_buffer);
2220 if (traversal) {
2221 HTTP_NETCLOSE(s, handle);
2222 status = -1;
2223 goto clean_up;
2224 }
2225 if (!dump_output_immediately &&
2226 format_out == HTAtom_for("www/download")) {
2227 /*
2228 * Convert a download request to a presentation request for
2229 * interactive users. - FM
2230 */
2231 format_out = WWW_PRESENT;
2232 }
2233 break;
2234
2235 default:
2236 /*
2237 * Bad or unknown server_status number. Take a chance and hope
2238 * there is something to display. - FM
2239 */
2240 HTAlert(gettext("Unknown status reply from server!"));
2241 HTAlert(line_buffer);
2242 if (traversal) {
2243 HTTP_NETCLOSE(s, handle);
2244 status = -1;
2245 goto clean_up;
2246 }
2247 if (!dump_output_immediately &&
2248 format_out == HTAtom_for("www/download")) {
2249 /*
2250 * Convert a download request to a presentation request for
2251 * interactive users. - FM
2252 */
2253 format_out = WWW_PRESENT;
2254 }
2255 break;
2256 } /* Switch on server_status/100 */
2257
2258 } /* Full HTTP reply */
2259 } /* scope of fields */
2260
2261 /*
2262 * The user may have pressed the 'z'ap key during the pause caused by one
2263 * of the HTAlerts above if the server reported an error, to skip loading
2264 * of the error response page. Checking here before setting up the stream
2265 * stack and feeding it data avoids doing unnecessary work, it also can
2266 * avoid unnecessarily pushing a loaded document out of the cache to make
2267 * room for the unwanted error page. - kw
2268 */
2269 if (HTCheckForInterrupt()) {
2270 HTTP_NETCLOSE(s, handle);
2271 if (doing_redirect) {
2272 /*
2273 * Impatient user. - FM
2274 */
2275 CTRACE((tfp, "HTTP: Interrupted followup read.\n"));
2276 _HTProgress(CONNECTION_INTERRUPTED);
2277 }
2278 status = HT_INTERRUPTED;
2279 goto clean_up;
2280 }
2281 /*
2282 * Set up the stream stack to handle the body of the message.
2283 */
2284 if (do_head || keep_mime_headers) {
2285 /*
2286 * It was a HEAD request, or we want the headers and source.
2287 */
2288 start_of_data = line_kept_clean;
2289 #ifdef SH_EX /* FIX BUG by kaz@maczuka.hitachi.ibaraki.jp */
2290 /* GIF file contains \0, so strlen does not return the data length */
2291 length = real_length_of_line;
2292 #else
2293 length = rawlength;
2294 #endif
2295 format_in = HTAtom_for("text/plain");
2296
2297 } else if (doing_redirect) {
2298
2299 format_in = HTAtom_for("message/x-http-redirection");
2300 StrAllocCopy(anAnchor->content_type, HTAtom_name(format_in));
2301 if (traversal) {
2302 format_out = WWW_DEBUG;
2303 if (!sink)
2304 sink = HTErrorStream();
2305 } else if (!dump_output_immediately &&
2306 format_out == HTAtom_for("www/download")) {
2307 /*
2308 * Convert a download request to a presentation request for
2309 * interactive users. - FM
2310 */
2311 format_out = WWW_PRESENT;
2312 }
2313 }
2314
2315 target = HTStreamStack(format_in,
2316 format_out,
2317 sink, anAnchor);
2318
2319 if (target == NULL) {
2320 char *buffer = NULL;
2321
2322 HTTP_NETCLOSE(s, handle);
2323 HTSprintf0(&buffer, CANNOT_CONVERT_I_TO_O,
2324 HTAtom_name(format_in), HTAtom_name(format_out));
2325 _HTProgress(buffer);
2326 FREE(buffer);
2327 status = -1;
2328 goto clean_up;
2329 }
2330
2331 /*
2332 * Recycle the first chunk of data, in all cases.
2333 */
2334 (*target->isa->put_block) (target, start_of_data, length);
2335
2336 /*
2337 * Go pull the bulk of the data down.
2338 */
2339 rv = HTCopy(anAnchor, s, (void *) handle, target);
2340
2341 /*
2342 * If we get here with doing_redirect set, it means that we were looking
2343 * for a Location header. We either have got it now in redirecting_url -
2344 * in that case the stream should not have loaded any data. Or we didn't
2345 * get it, in that case the stream may have presented the message body
2346 * normally. - kw
2347 */
2348
2349 if (rv == -1) {
2350 /*
2351 * Intentional interrupt before data were received, not an error
2352 */
2353 if (doing_redirect && traversal)
2354 status = -1;
2355 else
2356 status = HT_INTERRUPTED;
2357 HTTP_NETCLOSE(s, handle);
2358 goto clean_up;
2359 }
2360
2361 if (rv == -2) {
2362 /*
2363 * Aw hell, a REAL error, maybe cuz it's a dumb HTTP0 server
2364 */
2365 (*target->isa->_abort) (target, NULL);
2366 if (doing_redirect && redirecting_url) {
2367 /*
2368 * Got a location before the error occurred? Then consider it an
2369 * interrupt but proceed below as normal. - kw
2370 */
2371 /* do nothing here */
2372 } else {
2373 HTTP_NETCLOSE(s, handle);
2374 if (!doing_redirect && !already_retrying && !do_post) {
2375 CTRACE((tfp, "HTTP: Trying again with HTTP0 request.\n"));
2376 /*
2377 * May as well consider it an interrupt -- right?
2378 */
2379 FREE(line_buffer);
2380 FREE(line_kept_clean);
2381 extensions = NO;
2382 already_retrying = TRUE;
2383 _HTProgress(RETRYING_AS_HTTP0);
2384 goto try_again;
2385 } else {
2386 status = HT_NOT_LOADED;
2387 goto clean_up;
2388 }
2389 }
2390 }
2391
2392 /*
2393 * Free if complete transmission (socket was closed before return). Close
2394 * socket if partial transmission (was freed on abort).
2395 */
2396 if (rv != HT_INTERRUPTED && rv != -2) {
2397 (*target->isa->_free) (target);
2398 } else {
2399 HTTP_NETCLOSE(s, handle);
2400 }
2401
2402 if (doing_redirect) {
2403 if (redirecting_url) {
2404 /*
2405 * Set up for checking redirecting_url in LYGetFile.c for
2406 * restrictions before we seek the document at that Location. - FM
2407 */
2408 CTRACE((tfp, "HTTP: Picked up location '%s'\n",
2409 redirecting_url));
2410 if (rv == HT_INTERRUPTED) {
2411 /*
2412 * Intentional interrupt after data were received, not an error
2413 * (probably). We take it as a user request to abandon the
2414 * redirection chain.
2415 *
2416 * This could reasonably be changed (by just removing this
2417 * block), it would make sense if there are redirecting
2418 * resources that "hang" after sending the headers. - kw
2419 */
2420 FREE(redirecting_url);
2421 CTRACE((tfp, "HTTP: Interrupted followup read.\n"));
2422 status = HT_INTERRUPTED;
2423 goto clean_up;
2424 }
2425 HTProgress(line_buffer);
2426 if (server_status == 305) { /* Use Proxy */
2427 /*
2428 * Make sure the proxy field ends with a slash. - FM
2429 */
2430 if (redirecting_url[strlen(redirecting_url) - 1]
2431 != '/')
2432 StrAllocCat(redirecting_url, "/");
2433 /*
2434 * Append our URL. - FM
2435 */
2436 StrAllocCat(redirecting_url, anAnchor->address);
2437 CTRACE((tfp, "HTTP: Proxy URL is '%s'\n",
2438 redirecting_url));
2439 }
2440 if (!do_post ||
2441 server_status == 303 ||
2442 server_status == 302) {
2443 /*
2444 * We don't have POST content (nor support PUT or DELETE), or
2445 * the status is "See Other" or "General Redirection" and we
2446 * can convert to GET, so go back and check out the new URL. -
2447 * FM
2448 */
2449 status = HT_REDIRECTING;
2450 goto clean_up;
2451 }
2452 /*
2453 * Make sure the user wants to redirect the POST content, or treat
2454 * as GET - FM & DK
2455 */
2456 switch (HTConfirmPostRedirect(redirecting_url,
2457 server_status)) {
2458 /*
2459 * User failed to confirm. Abort the fetch.
2460 */
2461 case 0:
2462 FREE(redirecting_url);
2463 status = HT_NO_DATA;
2464 goto clean_up;
2465
2466 /*
2467 * User wants to treat as GET with no content. Go back to
2468 * check out the URL.
2469 */
2470 case 303:
2471 break;
2472
2473 /*
2474 * Set the flag to retain the POST content and go back to check
2475 * out the URL. - FM
2476 */
2477 default:
2478 redirect_post_content = TRUE;
2479 }
2480
2481 /* Lou's old comment: - FM */
2482 /* OK, now we've got the redirection URL temporarily stored
2483 in external variable redirecting_url, exported from HTMIME.c,
2484 since there's no straightforward way to do this in the library
2485 currently. Do the right thing. */
2486
2487 status = HT_REDIRECTING;
2488
2489 } else {
2490 status = traversal ? -1 : HT_LOADED;
2491 }
2492
2493 } else {
2494 /*
2495 * If any data were received, treat as a complete transmission
2496 */
2497 status = HT_LOADED;
2498 }
2499
2500 /*
2501 * Clean up
2502 */
2503 clean_up:
2504 FREE(line_buffer);
2505 FREE(line_kept_clean);
2506
2507 done:
2508 /*
2509 * Clear out on exit, just in case.
2510 */
2511 reloading = FALSE;
2512 #ifdef USE_SSL
2513 FREE(connect_host);
2514 if (handle) {
2515 SSL_free(handle);
2516 SSL_handle = handle = NULL;
2517 }
2518 #endif /* USE_SSL */
2519 dump_server_status = server_status;
2520 return status;
2521 }
2522
2523 /* Protocol descriptor
2524 */
2525 #ifdef GLOBALDEF_IS_MACRO
2526 #define _HTTP_C_GLOBALDEF_1_INIT { "http", HTLoadHTTP, 0}
2527 GLOBALDEF(HTProtocol, HTTP, _HTTP_C_GLOBALDEF_1_INIT);
2528 #define _HTTP_C_GLOBALDEF_2_INIT { "https", HTLoadHTTP, 0}
2529 GLOBALDEF(HTProtocol, HTTPS, _HTTP_C_GLOBALDEF_2_INIT);
2530 #else
2531 GLOBALDEF HTProtocol HTTP =
2532 {"http", HTLoadHTTP, 0};
2533 GLOBALDEF HTProtocol HTTPS =
2534 {"https", HTLoadHTTP, 0};
2535 #endif /* GLOBALDEF_IS_MACRO */
2536