http.c

   1 #include "git-compat-util.h"
   2 #include "git-curl-compat.h"
   3 #include "hex.h"
   4 #include "http.h"
   5 #include "config.h"
   6 #include "pack.h"
   7 #include "run-command.h"
   8 #include "url.h"
   9 #include "urlmatch.h"
  10 #include "credential.h"
  11 #include "version.h"
  12 #include "pkt-line.h"
  13 #include "gettext.h"
  14 #include "trace.h"
  15 #include "transport.h"
  16 #include "packfile.h"
  17 #include "string-list.h"
  18 #include "object-file.h"
  19 #include "object-store-ll.h"
  20
  21 static struct trace_key trace_curl = TRACE_KEY_INIT(CURL);
  22 static int trace_curl_data = 1;
  23 static int trace_curl_redact = 1;
  24 long int git_curl_ipresolve = CURL_IPRESOLVE_WHATEVER;
  25 int active_requests;
  26 int http_is_verbose;
  27 ssize_t http_post_buffer = 16 * LARGE_PACKET_MAX;
  28
  29 static int min_curl_sessions = 1;
  30 static int curl_session_count;
  31 static int max_requests = -1;
  32 static CURLM *curlm;
  33 static CURL *curl_default;
  34
  35 #define PREV_BUF_SIZE 4096
  36
  37 char curl_errorstr[CURL_ERROR_SIZE];
  38
  39 static int curl_ssl_verify = -1;
  40 static int curl_ssl_try;
  41 static char *curl_http_version;
  42 static char *ssl_cert;
  43 static char *ssl_cert_type;
  44 static char *ssl_cipherlist;
  45 static char *ssl_version;
  46 static struct {
  47         const char *name;
  48         long ssl_version;
  49 } sslversions[] = {
  50         { "sslv2", CURL_SSLVERSION_SSLv2 },
  51         { "sslv3", CURL_SSLVERSION_SSLv3 },
  52         { "tlsv1", CURL_SSLVERSION_TLSv1 },
  53 #ifdef GIT_CURL_HAVE_CURL_SSLVERSION_TLSv1_0
  54         { "tlsv1.0", CURL_SSLVERSION_TLSv1_0 },
  55         { "tlsv1.1", CURL_SSLVERSION_TLSv1_1 },
  56         { "tlsv1.2", CURL_SSLVERSION_TLSv1_2 },
  57 #endif
  58 #ifdef GIT_CURL_HAVE_CURL_SSLVERSION_TLSv1_3
  59         { "tlsv1.3", CURL_SSLVERSION_TLSv1_3 },
  60 #endif
  61 };
  62 static char *ssl_key;
  63 static char *ssl_key_type;
  64 static char *ssl_capath;
  65 static char *curl_no_proxy;
  66 #ifdef GIT_CURL_HAVE_CURLOPT_PINNEDPUBLICKEY
  67 static char *ssl_pinnedkey;
  68 #endif
  69 static char *ssl_cainfo;
  70 static long curl_low_speed_limit = -1;
  71 static long curl_low_speed_time = -1;
  72 static int curl_ftp_no_epsv;
  73 static char *curl_http_proxy;
  74 static char *http_proxy_authmethod;
  75
  76 static char *http_proxy_ssl_cert;
  77 static char *http_proxy_ssl_key;
  78 static char *http_proxy_ssl_ca_info;
  79 static struct credential proxy_cert_auth = CREDENTIAL_INIT;
  80 static int proxy_ssl_cert_password_required;
  81
  82 static struct {
  83         const char *name;
  84         long curlauth_param;
  85 } proxy_authmethods[] = {
  86         { "basic", CURLAUTH_BASIC },
  87         { "digest", CURLAUTH_DIGEST },
  88         { "negotiate", CURLAUTH_GSSNEGOTIATE },
  89         { "ntlm", CURLAUTH_NTLM },
  90         { "anyauth", CURLAUTH_ANY },
  91         /*
  92          * CURLAUTH_DIGEST_IE has no corresponding command-line option in
  93          * curl(1) and is not included in CURLAUTH_ANY, so we leave it out
  94          * here, too
  95          */
  96 };
  97 #ifdef CURLGSSAPI_DELEGATION_FLAG
  98 static char *curl_deleg;
  99 static struct {
 100         const char *name;
 101         long curl_deleg_param;
 102 } curl_deleg_levels[] = {
 103         { "none", CURLGSSAPI_DELEGATION_NONE },
 104         { "policy", CURLGSSAPI_DELEGATION_POLICY_FLAG },
 105         { "always", CURLGSSAPI_DELEGATION_FLAG },
 106 };
 107 #endif
 108
 109 static struct credential proxy_auth = CREDENTIAL_INIT;
 110 static const char *curl_proxyuserpwd;
 111 static char *curl_cookie_file;
 112 static int curl_save_cookies;
 113 struct credential http_auth = CREDENTIAL_INIT;
 114 static int http_proactive_auth;
 115 static char *user_agent;
 116 static int curl_empty_auth = -1;
 117
 118 enum http_follow_config http_follow_config = HTTP_FOLLOW_INITIAL;
 119
 120 static struct credential cert_auth = CREDENTIAL_INIT;
 121 static int ssl_cert_password_required;
 122 static unsigned long http_auth_methods = CURLAUTH_ANY;
 123 static int http_auth_methods_restricted;
 124 /* Modes for which empty_auth cannot actually help us. */
 125 static unsigned long empty_auth_useless =
 126         CURLAUTH_BASIC
 127         | CURLAUTH_DIGEST_IE
 128         | CURLAUTH_DIGEST;
 129
 130 static struct curl_slist *pragma_header;
 131 static struct string_list extra_http_headers = STRING_LIST_INIT_DUP;
 132
 133 static struct curl_slist *host_resolutions;
 134
 135 static struct active_request_slot *active_queue_head;
 136
 137 static char *cached_accept_language;
 138
 139 static char *http_ssl_backend;
 140
 141 static int http_schannel_check_revoke = 1;
 142 /*
 143  * With the backend being set to `schannel`, setting sslCAinfo would override
 144  * the Certificate Store in cURL v7.60.0 and later, which is not what we want
 145  * by default.
 146  */
 147 static int http_schannel_use_ssl_cainfo;
 148
 149 size_t fread_buffer(char *ptr, size_t eltsize, size_t nmemb, void *buffer_)
 150 {
 151         size_t size = eltsize * nmemb;
 152         struct buffer *buffer = buffer_;
 153
 154         if (size > buffer->buf.len - buffer->posn)
 155                 size = buffer->buf.len - buffer->posn;
 156         memcpy(ptr, buffer->buf.buf + buffer->posn, size);
 157         buffer->posn += size;
 158
 159         return size / eltsize;
 160 }
 161
 162 int seek_buffer(void *clientp, curl_off_t offset, int origin)
 163 {
 164         struct buffer *buffer = clientp;
 165
 166         if (origin != SEEK_SET)
 167                 BUG("seek_buffer only handles SEEK_SET");
 168         if (offset < 0 || offset >= buffer->buf.len) {
 169                 error("curl seek would be outside of buffer");
 170                 return CURL_SEEKFUNC_FAIL;
 171         }
 172
 173         buffer->posn = offset;
 174         return CURL_SEEKFUNC_OK;
 175 }
 176
 177 size_t fwrite_buffer(char *ptr, size_t eltsize, size_t nmemb, void *buffer_)
 178 {
 179         size_t size = eltsize * nmemb;
 180         struct strbuf *buffer = buffer_;
 181
 182         strbuf_add(buffer, ptr, size);
 183         return nmemb;
 184 }
 185
 186 /*
 187  * A folded header continuation line starts with any number of spaces or
 188  * horizontal tab characters (SP or HTAB) as per RFC 7230 section 3.2.
 189  * It is not a continuation line if the line starts with any other character.
 190  */
 191 static inline int is_hdr_continuation(const char *ptr, const size_t size)
 192 {
 193         return size && (*ptr == ' ' || *ptr == '\t');
 194 }
 195
 196 static size_t fwrite_wwwauth(char *ptr, size_t eltsize, size_t nmemb, void *p UNUSED)
 197 {
 198         size_t size = eltsize * nmemb;
 199         struct strvec *values = &http_auth.wwwauth_headers;
 200         struct strbuf buf = STRBUF_INIT;
 201         const char *val;
 202         size_t val_len;
 203
 204         /*
 205          * Header lines may not come NULL-terminated from libcurl so we must
 206          * limit all scans to the maximum length of the header line, or leverage
 207          * strbufs for all operations.
 208          *
 209          * In addition, it is possible that header values can be split over
 210          * multiple lines as per RFC 7230. 'Line folding' has been deprecated
 211          * but older servers may still emit them. A continuation header field
 212          * value is identified as starting with a space or horizontal tab.
 213          *
 214          * The formal definition of a header field as given in RFC 7230 is:
 215          *
 216          * header-field   = field-name ":" OWS field-value OWS
 217          *
 218          * field-name     = token
 219          * field-value    = *( field-content / obs-fold )
 220          * field-content  = field-vchar [ 1*( SP / HTAB ) field-vchar ]
 221          * field-vchar    = VCHAR / obs-text
 222          *
 223          * obs-fold       = CRLF 1*( SP / HTAB )
 224          *                ; obsolete line folding
 225          *                ; see Section 3.2.4
 226          */
 227
 228         /* Start of a new WWW-Authenticate header */
 229         if (skip_iprefix_mem(ptr, size, "www-authenticate:", &val, &val_len)) {
 230                 strbuf_add(&buf, val, val_len);
 231
 232                 /*
 233                  * Strip the CRLF that should be present at the end of each
 234                  * field as well as any trailing or leading whitespace from the
 235                  * value.
 236                  */
 237                 strbuf_trim(&buf);
 238
 239                 strvec_push(values, buf.buf);
 240                 http_auth.header_is_last_match = 1;
 241                 goto exit;
 242         }
 243
 244         /*
 245          * This line could be a continuation of the previously matched header
 246          * field. If this is the case then we should append this value to the
 247          * end of the previously consumed value.
 248          */
 249         if (http_auth.header_is_last_match && is_hdr_continuation(ptr, size)) {
 250                 /*
 251                  * Trim the CRLF and any leading or trailing from this line.
 252                  */
 253                 strbuf_add(&buf, ptr, size);
 254                 strbuf_trim(&buf);
 255
 256                 /*
 257                  * At this point we should always have at least one existing
 258                  * value, even if it is empty. Do not bother appending the new
 259                  * value if this continuation header is itself empty.
 260                  */
 261                 if (!values->nr) {
 262                         BUG("should have at least one existing header value");
 263                 } else if (buf.len) {
 264                         char *prev = xstrdup(values->v[values->nr - 1]);
 265
 266                         /* Join two non-empty values with a single space. */
 267                         const char *const sp = *prev ? " " : "";
 268
 269                         strvec_pop(values);
 270                         strvec_pushf(values, "%s%s%s", prev, sp, buf.buf);
 271                         free(prev);
 272                 }
 273
 274                 goto exit;
 275         }
 276
 277         /* Not a continuation of a previously matched auth header line. */
 278         http_auth.header_is_last_match = 0;
 279
 280         /*
 281          * If this is a HTTP status line and not a header field, this signals
 282          * a different HTTP response. libcurl writes all the output of all
 283          * response headers of all responses, including redirects.
 284          * We only care about the last HTTP request response's headers so clear
 285          * the existing array.
 286          */
 287         if (skip_iprefix_mem(ptr, size, "http/", &val, &val_len))
 288                 strvec_clear(values);
 289
 290 exit:
 291         strbuf_release(&buf);
 292         return size;
 293 }
 294
 295 size_t fwrite_null(char *ptr UNUSED, size_t eltsize UNUSED, size_t nmemb,
 296                    void *data UNUSED)
 297 {
 298         return nmemb;
 299 }
 300
 301 static struct curl_slist *object_request_headers(void)
 302 {
 303         return curl_slist_append(http_copy_default_headers(), "Pragma:");
 304 }
 305
 306 static void closedown_active_slot(struct active_request_slot *slot)
 307 {
 308         active_requests--;
 309         slot->in_use = 0;
 310 }
 311
 312 static void finish_active_slot(struct active_request_slot *slot)
 313 {
 314         closedown_active_slot(slot);
 315         curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code);
 316
 317         if (slot->finished)
 318                 (*slot->finished) = 1;
 319
 320         /* Store slot results so they can be read after the slot is reused */
 321         if (slot->results) {
 322                 slot->results->curl_result = slot->curl_result;
 323                 slot->results->http_code = slot->http_code;
 324                 curl_easy_getinfo(slot->curl, CURLINFO_HTTPAUTH_AVAIL,
 325                                   &slot->results->auth_avail);
 326
 327                 curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CONNECTCODE,
 328                         &slot->results->http_connectcode);
 329         }
 330
 331         /* Run callback if appropriate */
 332         if (slot->callback_func)
 333                 slot->callback_func(slot->callback_data);
 334 }
 335
 336 static void xmulti_remove_handle(struct active_request_slot *slot)
 337 {
 338         curl_multi_remove_handle(curlm, slot->curl);
 339 }
 340
 341 static void process_curl_messages(void)
 342 {
 343         int num_messages;
 344         struct active_request_slot *slot;
 345         CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
 346
 347         while (curl_message != NULL) {
 348                 if (curl_message->msg == CURLMSG_DONE) {
 349                         int curl_result = curl_message->data.result;
 350                         slot = active_queue_head;
 351                         while (slot != NULL &&
 352                                slot->curl != curl_message->easy_handle)
 353                                 slot = slot->next;
 354                         if (slot) {
 355                                 xmulti_remove_handle(slot);
 356                                 slot->curl_result = curl_result;
 357                                 finish_active_slot(slot);
 358                         } else {
 359                                 fprintf(stderr, "Received DONE message for unknown request!\n");
 360                         }
 361                 } else {
 362                         fprintf(stderr, "Unknown CURL message received: %d\n",
 363                                 (int)curl_message->msg);
 364                 }
 365                 curl_message = curl_multi_info_read(curlm, &num_messages);
 366         }
 367 }
 368
 369 static int http_options(const char *var, const char *value,
 370                         const struct config_context *ctx, void *data)
 371 {
 372         if (!strcmp("http.version", var)) {
 373                 return git_config_string(&curl_http_version, var, value);
 374         }
 375         if (!strcmp("http.sslverify", var)) {
 376                 curl_ssl_verify = git_config_bool(var, value);
 377                 return 0;
 378         }
 379         if (!strcmp("http.sslcipherlist", var))
 380                 return git_config_string(&ssl_cipherlist, var, value);
 381         if (!strcmp("http.sslversion", var))
 382                 return git_config_string(&ssl_version, var, value);
 383         if (!strcmp("http.sslcert", var))
 384                 return git_config_pathname(&ssl_cert, var, value);
 385         if (!strcmp("http.sslcerttype", var))
 386                 return git_config_string(&ssl_cert_type, var, value);
 387         if (!strcmp("http.sslkey", var))
 388                 return git_config_pathname(&ssl_key, var, value);
 389         if (!strcmp("http.sslkeytype", var))
 390                 return git_config_string(&ssl_key_type, var, value);
 391         if (!strcmp("http.sslcapath", var))
 392                 return git_config_pathname(&ssl_capath, var, value);
 393         if (!strcmp("http.sslcainfo", var))
 394                 return git_config_pathname(&ssl_cainfo, var, value);
 395         if (!strcmp("http.sslcertpasswordprotected", var)) {
 396                 ssl_cert_password_required = git_config_bool(var, value);
 397                 return 0;
 398         }
 399         if (!strcmp("http.ssltry", var)) {
 400                 curl_ssl_try = git_config_bool(var, value);
 401                 return 0;
 402         }
 403         if (!strcmp("http.sslbackend", var)) {
 404                 free(http_ssl_backend);
 405                 http_ssl_backend = xstrdup_or_null(value);
 406                 return 0;
 407         }
 408
 409         if (!strcmp("http.schannelcheckrevoke", var)) {
 410                 http_schannel_check_revoke = git_config_bool(var, value);
 411                 return 0;
 412         }
 413
 414         if (!strcmp("http.schannelusesslcainfo", var)) {
 415                 http_schannel_use_ssl_cainfo = git_config_bool(var, value);
 416                 return 0;
 417         }
 418
 419         if (!strcmp("http.minsessions", var)) {
 420                 min_curl_sessions = git_config_int(var, value, ctx->kvi);
 421                 if (min_curl_sessions > 1)
 422                         min_curl_sessions = 1;
 423                 return 0;
 424         }
 425         if (!strcmp("http.maxrequests", var)) {
 426                 max_requests = git_config_int(var, value, ctx->kvi);
 427                 return 0;
 428         }
 429         if (!strcmp("http.lowspeedlimit", var)) {
 430                 curl_low_speed_limit = (long)git_config_int(var, value, ctx->kvi);
 431                 return 0;
 432         }
 433         if (!strcmp("http.lowspeedtime", var)) {
 434                 curl_low_speed_time = (long)git_config_int(var, value, ctx->kvi);
 435                 return 0;
 436         }
 437
 438         if (!strcmp("http.noepsv", var)) {
 439                 curl_ftp_no_epsv = git_config_bool(var, value);
 440                 return 0;
 441         }
 442         if (!strcmp("http.proxy", var))
 443                 return git_config_string(&curl_http_proxy, var, value);
 444
 445         if (!strcmp("http.proxyauthmethod", var))
 446                 return git_config_string(&http_proxy_authmethod, var, value);
 447
 448         if (!strcmp("http.proxysslcert", var))
 449                 return git_config_string(&http_proxy_ssl_cert, var, value);
 450
 451         if (!strcmp("http.proxysslkey", var))
 452                 return git_config_string(&http_proxy_ssl_key, var, value);
 453
 454         if (!strcmp("http.proxysslcainfo", var))
 455                 return git_config_string(&http_proxy_ssl_ca_info, var, value);
 456
 457         if (!strcmp("http.proxysslcertpasswordprotected", var)) {
 458                 proxy_ssl_cert_password_required = git_config_bool(var, value);
 459                 return 0;
 460         }
 461
 462         if (!strcmp("http.cookiefile", var))
 463                 return git_config_pathname(&curl_cookie_file, var, value);
 464         if (!strcmp("http.savecookies", var)) {
 465                 curl_save_cookies = git_config_bool(var, value);
 466                 return 0;
 467         }
 468
 469         if (!strcmp("http.postbuffer", var)) {
 470                 http_post_buffer = git_config_ssize_t(var, value, ctx->kvi);
 471                 if (http_post_buffer < 0)
 472                         warning(_("negative value for http.postBuffer; defaulting to %d"), LARGE_PACKET_MAX);
 473                 if (http_post_buffer < LARGE_PACKET_MAX)
 474                         http_post_buffer = LARGE_PACKET_MAX;
 475                 return 0;
 476         }
 477
 478         if (!strcmp("http.useragent", var))
 479                 return git_config_string(&user_agent, var, value);
 480
 481         if (!strcmp("http.emptyauth", var)) {
 482                 if (value && !strcmp("auto", value))
 483                         curl_empty_auth = -1;
 484                 else
 485                         curl_empty_auth = git_config_bool(var, value);
 486                 return 0;
 487         }
 488
 489         if (!strcmp("http.delegation", var)) {
 490 #ifdef CURLGSSAPI_DELEGATION_FLAG
 491                 return git_config_string(&curl_deleg, var, value);
 492 #else
 493                 warning(_("Delegation control is not supported with cURL < 7.22.0"));
 494                 return 0;
 495 #endif
 496         }
 497
 498         if (!strcmp("http.pinnedpubkey", var)) {
 499 #ifdef GIT_CURL_HAVE_CURLOPT_PINNEDPUBLICKEY
 500                 return git_config_pathname(&ssl_pinnedkey, var, value);
 501 #else
 502                 warning(_("Public key pinning not supported with cURL < 7.39.0"));
 503                 return 0;
 504 #endif
 505         }
 506
 507         if (!strcmp("http.extraheader", var)) {
 508                 if (!value) {
 509                         return config_error_nonbool(var);
 510                 } else if (!*value) {
 511                         string_list_clear(&extra_http_headers, 0);
 512                 } else {
 513                         string_list_append(&extra_http_headers, value);
 514                 }
 515                 return 0;
 516         }
 517
 518         if (!strcmp("http.curloptresolve", var)) {
 519                 if (!value) {
 520                         return config_error_nonbool(var);
 521                 } else if (!*value) {
 522                         curl_slist_free_all(host_resolutions);
 523                         host_resolutions = NULL;
 524                 } else {
 525                         host_resolutions = curl_slist_append(host_resolutions, value);
 526                 }
 527                 return 0;
 528         }
 529
 530         if (!strcmp("http.followredirects", var)) {
 531                 if (value && !strcmp(value, "initial"))
 532                         http_follow_config = HTTP_FOLLOW_INITIAL;
 533                 else if (git_config_bool(var, value))
 534                         http_follow_config = HTTP_FOLLOW_ALWAYS;
 535                 else
 536                         http_follow_config = HTTP_FOLLOW_NONE;
 537                 return 0;
 538         }
 539
 540         /* Fall back on the default ones */
 541         return git_default_config(var, value, ctx, data);
 542 }
 543
 544 static int curl_empty_auth_enabled(void)
 545 {
 546         if (curl_empty_auth >= 0)
 547                 return curl_empty_auth;
 548
 549         /*
 550          * In the automatic case, kick in the empty-auth
 551          * hack as long as we would potentially try some
 552          * method more exotic than "Basic" or "Digest".
 553          *
 554          * But only do this when this is our second or
 555          * subsequent request, as by then we know what
 556          * methods are available.
 557          */
 558         if (http_auth_methods_restricted &&
 559             (http_auth_methods & ~empty_auth_useless))
 560                 return 1;
 561         return 0;
 562 }
 563
 564 struct curl_slist *http_append_auth_header(const struct credential *c,
 565                                            struct curl_slist *headers)
 566 {
 567         if (c->authtype && c->credential) {
 568                 struct strbuf auth = STRBUF_INIT;
 569                 strbuf_addf(&auth, "Authorization: %s %s",
 570                             c->authtype, c->credential);
 571                 headers = curl_slist_append(headers, auth.buf);
 572                 strbuf_release(&auth);
 573         }
 574         return headers;
 575 }
 576
 577 static void init_curl_http_auth(CURL *result)
 578 {
 579         if ((!http_auth.username || !*http_auth.username) &&
 580             (!http_auth.credential || !*http_auth.credential)) {
 581                 if (curl_empty_auth_enabled())
 582                         curl_easy_setopt(result, CURLOPT_USERPWD, ":");
 583                 return;
 584         }
 585
 586         credential_fill(&http_auth, 1);
 587
 588         if (http_auth.password) {
 589                 curl_easy_setopt(result, CURLOPT_USERNAME, http_auth.username);
 590                 curl_easy_setopt(result, CURLOPT_PASSWORD, http_auth.password);
 591         }
 592 }
 593
 594 /* *var must be free-able */
 595 static void var_override(char **var, char *value)
 596 {
 597         if (value) {
 598                 free(*var);
 599                 *var = xstrdup(value);
 600         }
 601 }
 602
 603 static void set_proxyauth_name_password(CURL *result)
 604 {
 605         if (proxy_auth.password) {
 606                 curl_easy_setopt(result, CURLOPT_PROXYUSERNAME,
 607                         proxy_auth.username);
 608                 curl_easy_setopt(result, CURLOPT_PROXYPASSWORD,
 609                         proxy_auth.password);
 610         } else if (proxy_auth.authtype && proxy_auth.credential) {
 611                 curl_easy_setopt(result, CURLOPT_PROXYHEADER,
 612                                  http_append_auth_header(&proxy_auth, NULL));
 613         }
 614 }
 615
 616 static void init_curl_proxy_auth(CURL *result)
 617 {
 618         if (proxy_auth.username) {
 619                 if (!proxy_auth.password && !proxy_auth.credential)
 620                         credential_fill(&proxy_auth, 1);
 621                 set_proxyauth_name_password(result);
 622         }
 623
 624         var_override(&http_proxy_authmethod, getenv("GIT_HTTP_PROXY_AUTHMETHOD"));
 625
 626         if (http_proxy_authmethod) {
 627                 int i;
 628                 for (i = 0; i < ARRAY_SIZE(proxy_authmethods); i++) {
 629                         if (!strcmp(http_proxy_authmethod, proxy_authmethods[i].name)) {
 630                                 curl_easy_setopt(result, CURLOPT_PROXYAUTH,
 631                                                 proxy_authmethods[i].curlauth_param);
 632                                 break;
 633                         }
 634                 }
 635                 if (i == ARRAY_SIZE(proxy_authmethods)) {
 636                         warning("unsupported proxy authentication method %s: using anyauth",
 637                                         http_proxy_authmethod);
 638                         curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
 639                 }
 640         }
 641         else
 642                 curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
 643 }
 644
 645 static int has_cert_password(void)
 646 {
 647         if (ssl_cert == NULL || ssl_cert_password_required != 1)
 648                 return 0;
 649         if (!cert_auth.password) {
 650                 cert_auth.protocol = xstrdup("cert");
 651                 cert_auth.host = xstrdup("");
 652                 cert_auth.username = xstrdup("");
 653                 cert_auth.path = xstrdup(ssl_cert);
 654                 credential_fill(&cert_auth, 0);
 655         }
 656         return 1;
 657 }
 658
 659 #ifdef GIT_CURL_HAVE_CURLOPT_PROXY_KEYPASSWD
 660 static int has_proxy_cert_password(void)
 661 {
 662         if (http_proxy_ssl_cert == NULL || proxy_ssl_cert_password_required != 1)
 663                 return 0;
 664         if (!proxy_cert_auth.password) {
 665                 proxy_cert_auth.protocol = xstrdup("cert");
 666                 proxy_cert_auth.host = xstrdup("");
 667                 proxy_cert_auth.username = xstrdup("");
 668                 proxy_cert_auth.path = xstrdup(http_proxy_ssl_cert);
 669                 credential_fill(&proxy_cert_auth, 0);
 670         }
 671         return 1;
 672 }
 673 #endif
 674
 675 #ifdef GITCURL_HAVE_CURLOPT_TCP_KEEPALIVE
 676 static void set_curl_keepalive(CURL *c)
 677 {
 678         curl_easy_setopt(c, CURLOPT_TCP_KEEPALIVE, 1);
 679 }
 680
 681 #else
 682 static int sockopt_callback(void *client, curl_socket_t fd, curlsocktype type)
 683 {
 684         int ka = 1;
 685         int rc;
 686         socklen_t len = (socklen_t)sizeof(ka);
 687
 688         if (type != CURLSOCKTYPE_IPCXN)
 689                 return 0;
 690
 691         rc = setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, (void *)&ka, len);
 692         if (rc < 0)
 693                 warning_errno("unable to set SO_KEEPALIVE on socket");
 694
 695         return CURL_SOCKOPT_OK;
 696 }
 697
 698 static void set_curl_keepalive(CURL *c)
 699 {
 700         curl_easy_setopt(c, CURLOPT_SOCKOPTFUNCTION, sockopt_callback);
 701 }
 702 #endif
 703
 704 /* Return 1 if redactions have been made, 0 otherwise. */
 705 static int redact_sensitive_header(struct strbuf *header, size_t offset)
 706 {
 707         int ret = 0;
 708         const char *sensitive_header;
 709
 710         if (trace_curl_redact &&
 711             (skip_iprefix(header->buf + offset, "Authorization:", &sensitive_header) ||
 712              skip_iprefix(header->buf + offset, "Proxy-Authorization:", &sensitive_header))) {
 713                 /* The first token is the type, which is OK to log */
 714                 while (isspace(*sensitive_header))
 715                         sensitive_header++;
 716                 while (*sensitive_header && !isspace(*sensitive_header))
 717                         sensitive_header++;
 718                 /* Everything else is opaque and possibly sensitive */
 719                 strbuf_setlen(header,  sensitive_header - header->buf);
 720                 strbuf_addstr(header, " <redacted>");
 721                 ret = 1;
 722         } else if (trace_curl_redact &&
 723                    skip_iprefix(header->buf + offset, "Cookie:", &sensitive_header)) {
 724                 struct strbuf redacted_header = STRBUF_INIT;
 725                 const char *cookie;
 726
 727                 while (isspace(*sensitive_header))
 728                         sensitive_header++;
 729
 730                 cookie = sensitive_header;
 731
 732                 while (cookie) {
 733                         char *equals;
 734                         char *semicolon = strstr(cookie, "; ");
 735                         if (semicolon)
 736                                 *semicolon = 0;
 737                         equals = strchrnul(cookie, '=');
 738                         if (!equals) {
 739                                 /* invalid cookie, just append and continue */
 740                                 strbuf_addstr(&redacted_header, cookie);
 741                                 continue;
 742                         }
 743                         strbuf_add(&redacted_header, cookie, equals - cookie);
 744                         strbuf_addstr(&redacted_header, "=<redacted>");
 745                         if (semicolon) {
 746                                 /*
 747                                  * There are more cookies. (Or, for some
 748                                  * reason, the input string ends in "; ".)
 749                                  */
 750                                 strbuf_addstr(&redacted_header, "; ");
 751                                 cookie = semicolon + strlen("; ");
 752                         } else {
 753                                 cookie = NULL;
 754                         }
 755                 }
 756
 757                 strbuf_setlen(header, sensitive_header - header->buf);
 758                 strbuf_addbuf(header, &redacted_header);
 759                 ret = 1;
 760         }
 761         return ret;
 762 }
 763
 764 static int match_curl_h2_trace(const char *line, const char **out)
 765 {
 766         const char *p;
 767
 768         /*
 769          * curl prior to 8.1.0 gives us:
 770          *
 771          *     h2h3 [<header-name>: <header-val>]
 772          *
 773          * Starting in 8.1.0, the first token became just "h2".
 774          */
 775         if (skip_iprefix(line, "h2h3 [", out) ||
 776             skip_iprefix(line, "h2 [", out))
 777                 return 1;
 778
 779         /*
 780          * curl 8.3.0 uses:
 781          *   [HTTP/2] [<stream-id>] [<header-name>: <header-val>]
 782          * where <stream-id> is numeric.
 783          */
 784         if (skip_iprefix(line, "[HTTP/2] [", &p)) {
 785                 while (isdigit(*p))
 786                         p++;
 787                 if (skip_prefix(p, "] [", out))
 788                         return 1;
 789         }
 790
 791         return 0;
 792 }
 793
 794 /* Redact headers in info */
 795 static void redact_sensitive_info_header(struct strbuf *header)
 796 {
 797         const char *sensitive_header;
 798
 799         if (trace_curl_redact &&
 800             match_curl_h2_trace(header->buf, &sensitive_header)) {
 801                 if (redact_sensitive_header(header, sensitive_header - header->buf)) {
 802                         /* redaction ate our closing bracket */
 803                         strbuf_addch(header, ']');
 804                 }
 805         }
 806 }
 807
 808 static void curl_dump_header(const char *text, unsigned char *ptr, size_t size, int hide_sensitive_header)
 809 {
 810         struct strbuf out = STRBUF_INIT;
 811         struct strbuf **headers, **header;
 812
 813         strbuf_addf(&out, "%s, %10.10ld bytes (0x%8.8lx)\n",
 814                 text, (long)size, (long)size);
 815         trace_strbuf(&trace_curl, &out);
 816         strbuf_reset(&out);
 817         strbuf_add(&out, ptr, size);
 818         headers = strbuf_split_max(&out, '\n', 0);
 819
 820         for (header = headers; *header; header++) {
 821                 if (hide_sensitive_header)
 822                         redact_sensitive_header(*header, 0);
 823                 strbuf_insertstr((*header), 0, text);
 824                 strbuf_insertstr((*header), strlen(text), ": ");
 825                 strbuf_rtrim((*header));
 826                 strbuf_addch((*header), '\n');
 827                 trace_strbuf(&trace_curl, (*header));
 828         }
 829         strbuf_list_free(headers);
 830         strbuf_release(&out);
 831 }
 832
 833 static void curl_dump_data(const char *text, unsigned char *ptr, size_t size)
 834 {
 835         size_t i;
 836         struct strbuf out = STRBUF_INIT;
 837         unsigned int width = 60;
 838
 839         strbuf_addf(&out, "%s, %10.10ld bytes (0x%8.8lx)\n",
 840                 text, (long)size, (long)size);
 841         trace_strbuf(&trace_curl, &out);
 842
 843         for (i = 0; i < size; i += width) {
 844                 size_t w;
 845
 846                 strbuf_reset(&out);
 847                 strbuf_addf(&out, "%s: ", text);
 848                 for (w = 0; (w < width) && (i + w < size); w++) {
 849                         unsigned char ch = ptr[i + w];
 850
 851                         strbuf_addch(&out,
 852                                        (ch >= 0x20) && (ch < 0x80)
 853                                        ? ch : '.');
 854                 }
 855                 strbuf_addch(&out, '\n');
 856                 trace_strbuf(&trace_curl, &out);
 857         }
 858         strbuf_release(&out);
 859 }
 860
 861 static void curl_dump_info(char *data, size_t size)
 862 {
 863         struct strbuf buf = STRBUF_INIT;
 864
 865         strbuf_add(&buf, data, size);
 866
 867         redact_sensitive_info_header(&buf);
 868         trace_printf_key(&trace_curl, "== Info: %s", buf.buf);
 869
 870         strbuf_release(&buf);
 871 }
 872
 873 static int curl_trace(CURL *handle UNUSED, curl_infotype type,
 874                       char *data, size_t size,
 875                       void *userp UNUSED)
 876 {
 877         const char *text;
 878         enum { NO_FILTER = 0, DO_FILTER = 1 };
 879
 880         switch (type) {
 881         case CURLINFO_TEXT:
 882                 curl_dump_info(data, size);
 883                 break;
 884         case CURLINFO_HEADER_OUT:
 885                 text = "=> Send header";
 886                 curl_dump_header(text, (unsigned char *)data, size, DO_FILTER);
 887                 break;
 888         case CURLINFO_DATA_OUT:
 889                 if (trace_curl_data) {
 890                         text = "=> Send data";
 891                         curl_dump_data(text, (unsigned char *)data, size);
 892                 }
 893                 break;
 894         case CURLINFO_SSL_DATA_OUT:
 895                 if (trace_curl_data) {
 896                         text = "=> Send SSL data";
 897                         curl_dump_data(text, (unsigned char *)data, size);
 898                 }
 899                 break;
 900         case CURLINFO_HEADER_IN:
 901                 text = "<= Recv header";
 902                 curl_dump_header(text, (unsigned char *)data, size, NO_FILTER);
 903                 break;
 904         case CURLINFO_DATA_IN:
 905                 if (trace_curl_data) {
 906                         text = "<= Recv data";
 907                         curl_dump_data(text, (unsigned char *)data, size);
 908                 }
 909                 break;
 910         case CURLINFO_SSL_DATA_IN:
 911                 if (trace_curl_data) {
 912                         text = "<= Recv SSL data";
 913                         curl_dump_data(text, (unsigned char *)data, size);
 914                 }
 915                 break;
 916
 917         default:                /* we ignore unknown types by default */
 918                 return 0;
 919         }
 920         return 0;
 921 }
 922
 923 void http_trace_curl_no_data(void)
 924 {
 925         trace_override_envvar(&trace_curl, "1");
 926         trace_curl_data = 0;
 927 }
 928
 929 void setup_curl_trace(CURL *handle)
 930 {
 931         if (!trace_want(&trace_curl))
 932                 return;
 933         curl_easy_setopt(handle, CURLOPT_VERBOSE, 1L);
 934         curl_easy_setopt(handle, CURLOPT_DEBUGFUNCTION, curl_trace);
 935         curl_easy_setopt(handle, CURLOPT_DEBUGDATA, NULL);
 936 }
 937
 938 static void proto_list_append(struct strbuf *list, const char *proto)
 939 {
 940         if (!list)
 941                 return;
 942         if (list->len)
 943                 strbuf_addch(list, ',');
 944         strbuf_addstr(list, proto);
 945 }
 946
 947 static long get_curl_allowed_protocols(int from_user, struct strbuf *list)
 948 {
 949         long bits = 0;
 950
 951         if (is_transport_allowed("http", from_user)) {
 952                 bits |= CURLPROTO_HTTP;
 953                 proto_list_append(list, "http");
 954         }
 955         if (is_transport_allowed("https", from_user)) {
 956                 bits |= CURLPROTO_HTTPS;
 957                 proto_list_append(list, "https");
 958         }
 959         if (is_transport_allowed("ftp", from_user)) {
 960                 bits |= CURLPROTO_FTP;
 961                 proto_list_append(list, "ftp");
 962         }
 963         if (is_transport_allowed("ftps", from_user)) {
 964                 bits |= CURLPROTO_FTPS;
 965                 proto_list_append(list, "ftps");
 966         }
 967
 968         return bits;
 969 }
 970
 971 #ifdef GIT_CURL_HAVE_CURL_HTTP_VERSION_2
 972 static int get_curl_http_version_opt(const char *version_string, long *opt)
 973 {
 974         int i;
 975         static struct {
 976                 const char *name;
 977                 long opt_token;
 978         } choice[] = {
 979                 { "HTTP/1.1", CURL_HTTP_VERSION_1_1 },
 980                 { "HTTP/2", CURL_HTTP_VERSION_2 }
 981         };
 982
 983         for (i = 0; i < ARRAY_SIZE(choice); i++) {
 984                 if (!strcmp(version_string, choice[i].name)) {
 985                         *opt = choice[i].opt_token;
 986                         return 0;
 987                 }
 988         }
 989
 990         warning("unknown value given to http.version: '%s'", version_string);
 991         return -1; /* not found */
 992 }
 993
 994 #endif
 995
 996 static CURL *get_curl_handle(void)
 997 {
 998         CURL *result = curl_easy_init();
 999
1000         if (!result)
1001                 die("curl_easy_init failed");
1002
1003         if (!curl_ssl_verify) {
1004                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 0);
1005                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 0);
1006         } else {
1007                 /* Verify authenticity of the peer's certificate */
1008                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 1);
1009                 /* The name in the cert must match whom we tried to connect */
1010                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 2);
1011         }
1012
1013 #ifdef GIT_CURL_HAVE_CURL_HTTP_VERSION_2
1014     if (curl_http_version) {
1015                 long opt;
1016                 if (!get_curl_http_version_opt(curl_http_version, &opt)) {
1017                         /* Set request use http version */
1018                         curl_easy_setopt(result, CURLOPT_HTTP_VERSION, opt);
1019                 }
1020     }
1021 #endif
1022
1023         curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
1024         curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
1025
1026 #ifdef CURLGSSAPI_DELEGATION_FLAG
1027         if (curl_deleg) {
1028                 int i;
1029                 for (i = 0; i < ARRAY_SIZE(curl_deleg_levels); i++) {
1030                         if (!strcmp(curl_deleg, curl_deleg_levels[i].name)) {
1031                                 curl_easy_setopt(result, CURLOPT_GSSAPI_DELEGATION,
1032                                                 curl_deleg_levels[i].curl_deleg_param);
1033                                 break;
1034                         }
1035                 }
1036                 if (i == ARRAY_SIZE(curl_deleg_levels))
1037                         warning("Unknown delegation method '%s': using default",
1038                                 curl_deleg);
1039         }
1040 #endif
1041
1042         if (http_ssl_backend && !strcmp("schannel", http_ssl_backend) &&
1043             !http_schannel_check_revoke) {
1044 #ifdef GIT_CURL_HAVE_CURLSSLOPT_NO_REVOKE
1045                 curl_easy_setopt(result, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NO_REVOKE);
1046 #else
1047                 warning(_("CURLSSLOPT_NO_REVOKE not supported with cURL < 7.44.0"));
1048 #endif
1049         }
1050
1051         if (http_proactive_auth)
1052                 init_curl_http_auth(result);
1053
1054         if (getenv("GIT_SSL_VERSION"))
1055                 ssl_version = getenv("GIT_SSL_VERSION");
1056         if (ssl_version && *ssl_version) {
1057                 int i;
1058                 for (i = 0; i < ARRAY_SIZE(sslversions); i++) {
1059                         if (!strcmp(ssl_version, sslversions[i].name)) {
1060                                 curl_easy_setopt(result, CURLOPT_SSLVERSION,
1061                                                  sslversions[i].ssl_version);
1062                                 break;
1063                         }
1064                 }
1065                 if (i == ARRAY_SIZE(sslversions))
1066                         warning("unsupported ssl version %s: using default",
1067                                 ssl_version);
1068         }
1069
1070         if (getenv("GIT_SSL_CIPHER_LIST"))
1071                 ssl_cipherlist = getenv("GIT_SSL_CIPHER_LIST");
1072         if (ssl_cipherlist != NULL && *ssl_cipherlist)
1073                 curl_easy_setopt(result, CURLOPT_SSL_CIPHER_LIST,
1074                                 ssl_cipherlist);
1075
1076         if (ssl_cert)
1077                 curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
1078         if (ssl_cert_type)
1079                 curl_easy_setopt(result, CURLOPT_SSLCERTTYPE, ssl_cert_type);
1080         if (has_cert_password())
1081                 curl_easy_setopt(result, CURLOPT_KEYPASSWD, cert_auth.password);
1082         if (ssl_key)
1083                 curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
1084         if (ssl_key_type)
1085                 curl_easy_setopt(result, CURLOPT_SSLKEYTYPE, ssl_key_type);
1086         if (ssl_capath)
1087                 curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
1088 #ifdef GIT_CURL_HAVE_CURLOPT_PINNEDPUBLICKEY
1089         if (ssl_pinnedkey)
1090                 curl_easy_setopt(result, CURLOPT_PINNEDPUBLICKEY, ssl_pinnedkey);
1091 #endif
1092         if (http_ssl_backend && !strcmp("schannel", http_ssl_backend) &&
1093             !http_schannel_use_ssl_cainfo) {
1094                 curl_easy_setopt(result, CURLOPT_CAINFO, NULL);
1095 #ifdef GIT_CURL_HAVE_CURLOPT_PROXY_CAINFO
1096                 curl_easy_setopt(result, CURLOPT_PROXY_CAINFO, NULL);
1097 #endif
1098         } else if (ssl_cainfo != NULL || http_proxy_ssl_ca_info != NULL) {
1099                 if (ssl_cainfo)
1100                         curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
1101 #ifdef GIT_CURL_HAVE_CURLOPT_PROXY_CAINFO
1102                 if (http_proxy_ssl_ca_info)
1103                         curl_easy_setopt(result, CURLOPT_PROXY_CAINFO, http_proxy_ssl_ca_info);
1104 #endif
1105         }
1106
1107         if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
1108                 curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
1109                                  curl_low_speed_limit);
1110                 curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
1111                                  curl_low_speed_time);
1112         }
1113
1114         curl_easy_setopt(result, CURLOPT_MAXREDIRS, 20);
1115         curl_easy_setopt(result, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
1116
1117 #ifdef GIT_CURL_HAVE_CURLOPT_PROTOCOLS_STR
1118         {
1119                 struct strbuf buf = STRBUF_INIT;
1120
1121                 get_curl_allowed_protocols(0, &buf);
1122                 curl_easy_setopt(result, CURLOPT_REDIR_PROTOCOLS_STR, buf.buf);
1123                 strbuf_reset(&buf);
1124
1125                 get_curl_allowed_protocols(-1, &buf);
1126                 curl_easy_setopt(result, CURLOPT_PROTOCOLS_STR, buf.buf);
1127                 strbuf_release(&buf);
1128         }
1129 #else
1130         curl_easy_setopt(result, CURLOPT_REDIR_PROTOCOLS,
1131                          get_curl_allowed_protocols(0, NULL));
1132         curl_easy_setopt(result, CURLOPT_PROTOCOLS,
1133                          get_curl_allowed_protocols(-1, NULL));
1134 #endif
1135
1136         if (getenv("GIT_CURL_VERBOSE"))
1137                 http_trace_curl_no_data();
1138         setup_curl_trace(result);
1139         if (getenv("GIT_TRACE_CURL_NO_DATA"))
1140                 trace_curl_data = 0;
1141         if (!git_env_bool("GIT_TRACE_REDACT", 1))
1142                 trace_curl_redact = 0;
1143
1144         curl_easy_setopt(result, CURLOPT_USERAGENT,
1145                 user_agent ? user_agent : git_user_agent());
1146
1147         if (curl_ftp_no_epsv)
1148                 curl_easy_setopt(result, CURLOPT_FTP_USE_EPSV, 0);
1149
1150         if (curl_ssl_try)
1151                 curl_easy_setopt(result, CURLOPT_USE_SSL, CURLUSESSL_TRY);
1152
1153         /*
1154          * CURL also examines these variables as a fallback; but we need to query
1155          * them here in order to decide whether to prompt for missing password (cf.
1156          * init_curl_proxy_auth()).
1157          *
1158          * Unlike many other common environment variables, these are historically
1159          * lowercase only. It appears that CURL did not know this and implemented
1160          * only uppercase variants, which was later corrected to take both - with
1161          * the exception of http_proxy, which is lowercase only also in CURL. As
1162          * the lowercase versions are the historical quasi-standard, they take
1163          * precedence here, as in CURL.
1164          */
1165         if (!curl_http_proxy) {
1166                 if (http_auth.protocol && !strcmp(http_auth.protocol, "https")) {
1167                         var_override(&curl_http_proxy, getenv("HTTPS_PROXY"));
1168                         var_override(&curl_http_proxy, getenv("https_proxy"));
1169                 } else {
1170                         var_override(&curl_http_proxy, getenv("http_proxy"));
1171                 }
1172                 if (!curl_http_proxy) {
1173                         var_override(&curl_http_proxy, getenv("ALL_PROXY"));
1174                         var_override(&curl_http_proxy, getenv("all_proxy"));
1175                 }
1176         }
1177
1178         if (curl_http_proxy && curl_http_proxy[0] == '\0') {
1179                 /*
1180                  * Handle case with the empty http.proxy value here to keep
1181                  * common code clean.
1182                  * NB: empty option disables proxying at all.
1183                  */
1184                 curl_easy_setopt(result, CURLOPT_PROXY, "");
1185         } else if (curl_http_proxy) {
1186                 if (starts_with(curl_http_proxy, "socks5h"))
1187                         curl_easy_setopt(result,
1188                                 CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME);
1189                 else if (starts_with(curl_http_proxy, "socks5"))
1190                         curl_easy_setopt(result,
1191                                 CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
1192                 else if (starts_with(curl_http_proxy, "socks4a"))
1193                         curl_easy_setopt(result,
1194                                 CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4A);
1195                 else if (starts_with(curl_http_proxy, "socks"))
1196                         curl_easy_setopt(result,
1197                                 CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4);
1198 #ifdef GIT_CURL_HAVE_CURLOPT_PROXY_KEYPASSWD
1199                 else if (starts_with(curl_http_proxy, "https")) {
1200                         curl_easy_setopt(result, CURLOPT_PROXYTYPE, CURLPROXY_HTTPS);
1201
1202                         if (http_proxy_ssl_cert)
1203                                 curl_easy_setopt(result, CURLOPT_PROXY_SSLCERT, http_proxy_ssl_cert);
1204
1205                         if (http_proxy_ssl_key)
1206                                 curl_easy_setopt(result, CURLOPT_PROXY_SSLKEY, http_proxy_ssl_key);
1207
1208                         if (has_proxy_cert_password())
1209                                 curl_easy_setopt(result, CURLOPT_PROXY_KEYPASSWD, proxy_cert_auth.password);
1210                 }
1211 #endif
1212                 if (strstr(curl_http_proxy, "://"))
1213                         credential_from_url(&proxy_auth, curl_http_proxy);
1214                 else {
1215                         struct strbuf url = STRBUF_INIT;
1216                         strbuf_addf(&url, "http://%s", curl_http_proxy);
1217                         credential_from_url(&proxy_auth, url.buf);
1218                         strbuf_release(&url);
1219                 }
1220
1221                 if (!proxy_auth.host)
1222                         die("Invalid proxy URL '%s'", curl_http_proxy);
1223
1224                 curl_easy_setopt(result, CURLOPT_PROXY, proxy_auth.host);
1225                 var_override(&curl_no_proxy, getenv("NO_PROXY"));
1226                 var_override(&curl_no_proxy, getenv("no_proxy"));
1227                 curl_easy_setopt(result, CURLOPT_NOPROXY, curl_no_proxy);
1228         }
1229         init_curl_proxy_auth(result);
1230
1231         set_curl_keepalive(result);
1232
1233         return result;
1234 }
1235
1236 static void set_from_env(char **var, const char *envname)
1237 {
1238         const char *val = getenv(envname);
1239         if (val) {
1240                 FREE_AND_NULL(*var);
1241                 *var = xstrdup(val);
1242         }
1243 }
1244
1245 void http_init(struct remote *remote, const char *url, int proactive_auth)
1246 {
1247         char *low_speed_limit;
1248         char *low_speed_time;
1249         char *normalized_url;
1250         struct urlmatch_config config = URLMATCH_CONFIG_INIT;
1251
1252         config.section = "http";
1253         config.key = NULL;
1254         config.collect_fn = http_options;
1255         config.cascade_fn = git_default_config;
1256         config.cb = NULL;
1257
1258         http_is_verbose = 0;
1259         normalized_url = url_normalize(url, &config.url);
1260
1261         git_config(urlmatch_config_entry, &config);
1262         free(normalized_url);
1263         string_list_clear(&config.vars, 1);
1264
1265 #ifdef GIT_CURL_HAVE_CURLSSLSET_NO_BACKENDS
1266         if (http_ssl_backend) {
1267                 const curl_ssl_backend **backends;
1268                 struct strbuf buf = STRBUF_INIT;
1269                 int i;
1270
1271                 switch (curl_global_sslset(-1, http_ssl_backend, &backends)) {
1272                 case CURLSSLSET_UNKNOWN_BACKEND:
1273                         strbuf_addf(&buf, _("Unsupported SSL backend '%s'. "
1274                                             "Supported SSL backends:"),
1275                                             http_ssl_backend);
1276                         for (i = 0; backends[i]; i++)
1277                                 strbuf_addf(&buf, "\n\t%s", backends[i]->name);
1278                         die("%s", buf.buf);
1279                 case CURLSSLSET_NO_BACKENDS:
1280                         die(_("Could not set SSL backend to '%s': "
1281                               "cURL was built without SSL backends"),
1282                             http_ssl_backend);
1283                 case CURLSSLSET_TOO_LATE:
1284                         die(_("Could not set SSL backend to '%s': already set"),
1285                             http_ssl_backend);
1286                 case CURLSSLSET_OK:
1287                         break; /* Okay! */
1288                 }
1289         }
1290 #endif
1291
1292         if (curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK)
1293                 die("curl_global_init failed");
1294
1295         http_proactive_auth = proactive_auth;
1296
1297         if (remote && remote->http_proxy)
1298                 curl_http_proxy = xstrdup(remote->http_proxy);
1299
1300         if (remote)
1301                 var_override(&http_proxy_authmethod, remote->http_proxy_authmethod);
1302
1303         pragma_header = curl_slist_append(http_copy_default_headers(),
1304                 "Pragma: no-cache");
1305
1306         {
1307                 char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
1308                 if (http_max_requests)
1309                         max_requests = atoi(http_max_requests);
1310         }
1311
1312         curlm = curl_multi_init();
1313         if (!curlm)
1314                 die("curl_multi_init failed");
1315
1316         if (getenv("GIT_SSL_NO_VERIFY"))
1317                 curl_ssl_verify = 0;
1318
1319         set_from_env(&ssl_cert, "GIT_SSL_CERT");
1320         set_from_env(&ssl_cert_type, "GIT_SSL_CERT_TYPE");
1321         set_from_env(&ssl_key, "GIT_SSL_KEY");
1322         set_from_env(&ssl_key_type, "GIT_SSL_KEY_TYPE");
1323         set_from_env(&ssl_capath, "GIT_SSL_CAPATH");
1324         set_from_env(&ssl_cainfo, "GIT_SSL_CAINFO");
1325
1326         set_from_env(&user_agent, "GIT_HTTP_USER_AGENT");
1327
1328         low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
1329         if (low_speed_limit)
1330                 curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
1331         low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
1332         if (low_speed_time)
1333                 curl_low_speed_time = strtol(low_speed_time, NULL, 10);
1334
1335         if (curl_ssl_verify == -1)
1336                 curl_ssl_verify = 1;
1337
1338         curl_session_count = 0;
1339         if (max_requests < 1)
1340                 max_requests = DEFAULT_MAX_REQUESTS;
1341
1342         set_from_env(&http_proxy_ssl_cert, "GIT_PROXY_SSL_CERT");
1343         set_from_env(&http_proxy_ssl_key, "GIT_PROXY_SSL_KEY");
1344         set_from_env(&http_proxy_ssl_ca_info, "GIT_PROXY_SSL_CAINFO");
1345
1346         if (getenv("GIT_PROXY_SSL_CERT_PASSWORD_PROTECTED"))
1347                 proxy_ssl_cert_password_required = 1;
1348
1349         if (getenv("GIT_CURL_FTP_NO_EPSV"))
1350                 curl_ftp_no_epsv = 1;
1351
1352         if (url) {
1353                 credential_from_url(&http_auth, url);
1354                 if (!ssl_cert_password_required &&
1355                     getenv("GIT_SSL_CERT_PASSWORD_PROTECTED") &&
1356                     starts_with(url, "https://"))
1357                         ssl_cert_password_required = 1;
1358         }
1359
1360         curl_default = get_curl_handle();
1361 }
1362
1363 void http_cleanup(void)
1364 {
1365         struct active_request_slot *slot = active_queue_head;
1366
1367         while (slot != NULL) {
1368                 struct active_request_slot *next = slot->next;
1369                 if (slot->curl) {
1370                         xmulti_remove_handle(slot);
1371                         curl_easy_cleanup(slot->curl);
1372                 }
1373                 free(slot);
1374                 slot = next;
1375         }
1376         active_queue_head = NULL;
1377
1378         curl_easy_cleanup(curl_default);
1379
1380         curl_multi_cleanup(curlm);
1381         curl_global_cleanup();
1382
1383         string_list_clear(&extra_http_headers, 0);
1384
1385         curl_slist_free_all(pragma_header);
1386         pragma_header = NULL;
1387
1388         curl_slist_free_all(host_resolutions);
1389         host_resolutions = NULL;
1390
1391         if (curl_http_proxy) {
1392                 free((void *)curl_http_proxy);
1393                 curl_http_proxy = NULL;
1394         }
1395
1396         if (proxy_auth.password) {
1397                 memset(proxy_auth.password, 0, strlen(proxy_auth.password));
1398                 FREE_AND_NULL(proxy_auth.password);
1399         }
1400
1401         free((void *)curl_proxyuserpwd);
1402         curl_proxyuserpwd = NULL;
1403
1404         free((void *)http_proxy_authmethod);
1405         http_proxy_authmethod = NULL;
1406
1407         if (cert_auth.password) {
1408                 memset(cert_auth.password, 0, strlen(cert_auth.password));
1409                 FREE_AND_NULL(cert_auth.password);
1410         }
1411         ssl_cert_password_required = 0;
1412
1413         if (proxy_cert_auth.password) {
1414                 memset(proxy_cert_auth.password, 0, strlen(proxy_cert_auth.password));
1415                 FREE_AND_NULL(proxy_cert_auth.password);
1416         }
1417         proxy_ssl_cert_password_required = 0;
1418
1419         FREE_AND_NULL(cached_accept_language);
1420 }
1421
1422 struct active_request_slot *get_active_slot(void)
1423 {
1424         struct active_request_slot *slot = active_queue_head;
1425         struct active_request_slot *newslot;
1426
1427         int num_transfers;
1428
1429         /* Wait for a slot to open up if the queue is full */
1430         while (active_requests >= max_requests) {
1431                 curl_multi_perform(curlm, &num_transfers);
1432                 if (num_transfers < active_requests)
1433                         process_curl_messages();
1434         }
1435
1436         while (slot != NULL && slot->in_use)
1437                 slot = slot->next;
1438
1439         if (!slot) {
1440                 newslot = xmalloc(sizeof(*newslot));
1441                 newslot->curl = NULL;
1442                 newslot->in_use = 0;
1443                 newslot->next = NULL;
1444
1445                 slot = active_queue_head;
1446                 if (!slot) {
1447                         active_queue_head = newslot;
1448                 } else {
1449                         while (slot->next != NULL)
1450                                 slot = slot->next;
1451                         slot->next = newslot;
1452                 }
1453                 slot = newslot;
1454         }
1455
1456         if (!slot->curl) {
1457                 slot->curl = curl_easy_duphandle(curl_default);
1458                 curl_session_count++;
1459         }
1460
1461         active_requests++;
1462         slot->in_use = 1;
1463         slot->results = NULL;
1464         slot->finished = NULL;
1465         slot->callback_data = NULL;
1466         slot->callback_func = NULL;
1467         curl_easy_setopt(slot->curl, CURLOPT_COOKIEFILE, curl_cookie_file);
1468         if (curl_save_cookies)
1469                 curl_easy_setopt(slot->curl, CURLOPT_COOKIEJAR, curl_cookie_file);
1470         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
1471         curl_easy_setopt(slot->curl, CURLOPT_RESOLVE, host_resolutions);
1472         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
1473         curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, NULL);
1474         curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, NULL);
1475         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, NULL);
1476         curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, NULL);
1477         curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, -1L);
1478         curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 0);
1479         curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
1480         curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 1);
1481         curl_easy_setopt(slot->curl, CURLOPT_RANGE, NULL);
1482
1483         /*
1484          * Default following to off unless "ALWAYS" is configured; this gives
1485          * callers a sane starting point, and they can tweak for individual
1486          * HTTP_FOLLOW_* cases themselves.
1487          */
1488         if (http_follow_config == HTTP_FOLLOW_ALWAYS)
1489                 curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 1);
1490         else
1491                 curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 0);
1492
1493         curl_easy_setopt(slot->curl, CURLOPT_IPRESOLVE, git_curl_ipresolve);
1494         curl_easy_setopt(slot->curl, CURLOPT_HTTPAUTH, http_auth_methods);
1495         if (http_auth.password || http_auth.credential || curl_empty_auth_enabled())
1496                 init_curl_http_auth(slot->curl);
1497
1498         return slot;
1499 }
1500
1501 int start_active_slot(struct active_request_slot *slot)
1502 {
1503         CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
1504         int num_transfers;
1505
1506         if (curlm_result != CURLM_OK &&
1507             curlm_result != CURLM_CALL_MULTI_PERFORM) {
1508                 warning("curl_multi_add_handle failed: %s",
1509                         curl_multi_strerror(curlm_result));
1510                 active_requests--;
1511                 slot->in_use = 0;
1512                 return 0;
1513         }
1514
1515         /*
1516          * We know there must be something to do, since we just added
1517          * something.
1518          */
1519         curl_multi_perform(curlm, &num_transfers);
1520         return 1;
1521 }
1522
1523 struct fill_chain {
1524         void *data;
1525         int (*fill)(void *);
1526         struct fill_chain *next;
1527 };
1528
1529 static struct fill_chain *fill_cfg;
1530
1531 void add_fill_function(void *data, int (*fill)(void *))
1532 {
1533         struct fill_chain *new_fill = xmalloc(sizeof(*new_fill));
1534         struct fill_chain **linkp = &fill_cfg;
1535         new_fill->data = data;
1536         new_fill->fill = fill;
1537         new_fill->next = NULL;
1538         while (*linkp)
1539                 linkp = &(*linkp)->next;
1540         *linkp = new_fill;
1541 }
1542
1543 void fill_active_slots(void)
1544 {
1545         struct active_request_slot *slot = active_queue_head;
1546
1547         while (active_requests < max_requests) {
1548                 struct fill_chain *fill;
1549                 for (fill = fill_cfg; fill; fill = fill->next)
1550                         if (fill->fill(fill->data))
1551                                 break;
1552
1553                 if (!fill)
1554                         break;
1555         }
1556
1557         while (slot != NULL) {
1558                 if (!slot->in_use && slot->curl != NULL
1559                         && curl_session_count > min_curl_sessions) {
1560                         curl_easy_cleanup(slot->curl);
1561                         slot->curl = NULL;
1562                         curl_session_count--;
1563                 }
1564                 slot = slot->next;
1565         }
1566 }
1567
1568 void step_active_slots(void)
1569 {
1570         int num_transfers;
1571         CURLMcode curlm_result;
1572
1573         do {
1574                 curlm_result = curl_multi_perform(curlm, &num_transfers);
1575         } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
1576         if (num_transfers < active_requests) {
1577                 process_curl_messages();
1578                 fill_active_slots();
1579         }
1580 }
1581
1582 void run_active_slot(struct active_request_slot *slot)
1583 {
1584         fd_set readfds;
1585         fd_set writefds;
1586         fd_set excfds;
1587         int max_fd;
1588         struct timeval select_timeout;
1589         int finished = 0;
1590
1591         slot->finished = &finished;
1592         while (!finished) {
1593                 step_active_slots();
1594
1595                 if (slot->in_use) {
1596                         long curl_timeout;
1597                         curl_multi_timeout(curlm, &curl_timeout);
1598                         if (curl_timeout == 0) {
1599                                 continue;
1600                         } else if (curl_timeout == -1) {
1601                                 select_timeout.tv_sec  = 0;
1602                                 select_timeout.tv_usec = 50000;
1603                         } else {
1604                                 select_timeout.tv_sec  =  curl_timeout / 1000;
1605                                 select_timeout.tv_usec = (curl_timeout % 1000) * 1000;
1606                         }
1607
1608                         max_fd = -1;
1609                         FD_ZERO(&readfds);
1610                         FD_ZERO(&writefds);
1611                         FD_ZERO(&excfds);
1612                         curl_multi_fdset(curlm, &readfds, &writefds, &excfds, &max_fd);
1613
1614                         /*
1615                          * It can happen that curl_multi_timeout returns a pathologically
1616                          * long timeout when curl_multi_fdset returns no file descriptors
1617                          * to read.  See commit message for more details.
1618                          */
1619                         if (max_fd < 0 &&
1620                             (select_timeout.tv_sec > 0 ||
1621                              select_timeout.tv_usec > 50000)) {
1622                                 select_timeout.tv_sec  = 0;
1623                                 select_timeout.tv_usec = 50000;
1624                         }
1625
1626                         select(max_fd+1, &readfds, &writefds, &excfds, &select_timeout);
1627                 }
1628         }
1629
1630         /*
1631          * The value of slot->finished we set before the loop was used
1632          * to set our "finished" variable when our request completed.
1633          *
1634          * 1. The slot may not have been reused for another requst
1635          *    yet, in which case it still has &finished.
1636          *
1637          * 2. The slot may already be in-use to serve another request,
1638          *    which can further be divided into two cases:
1639          *
1640          * (a) If call run_active_slot() hasn't been called for that
1641          *     other request, slot->finished would have been cleared
1642          *     by get_active_slot() and has NULL.
1643          *
1644          * (b) If the request did call run_active_slot(), then the
1645          *     call would have updated slot->finished at the beginning
1646          *     of this function, and with the clearing of the member
1647          *     below, we would find that slot->finished is now NULL.
1648          *
1649          * In all cases, slot->finished has no useful information to
1650          * anybody at this point.  Some compilers warn us for
1651          * attempting to smuggle a pointer that is about to become
1652          * invalid, i.e. &finished.  We clear it here to assure them.
1653          */
1654         slot->finished = NULL;
1655 }
1656
1657 static void release_active_slot(struct active_request_slot *slot)
1658 {
1659         closedown_active_slot(slot);
1660         if (slot->curl) {
1661                 xmulti_remove_handle(slot);
1662                 if (curl_session_count > min_curl_sessions) {
1663                         curl_easy_cleanup(slot->curl);
1664                         slot->curl = NULL;
1665                         curl_session_count--;
1666                 }
1667         }
1668         fill_active_slots();
1669 }
1670
1671 void finish_all_active_slots(void)
1672 {
1673         struct active_request_slot *slot = active_queue_head;
1674
1675         while (slot != NULL)
1676                 if (slot->in_use) {
1677                         run_active_slot(slot);
1678                         slot = active_queue_head;
1679                 } else {
1680                         slot = slot->next;
1681                 }
1682 }
1683
1684 /* Helpers for modifying and creating URLs */
1685 static inline int needs_quote(int ch)
1686 {
1687         if (((ch >= 'A') && (ch <= 'Z'))
1688                         || ((ch >= 'a') && (ch <= 'z'))
1689                         || ((ch >= '0') && (ch <= '9'))
1690                         || (ch == '/')
1691                         || (ch == '-')
1692                         || (ch == '.'))
1693                 return 0;
1694         return 1;
1695 }
1696
1697 static char *quote_ref_url(const char *base, const char *ref)
1698 {
1699         struct strbuf buf = STRBUF_INIT;
1700         const char *cp;
1701         int ch;
1702
1703         end_url_with_slash(&buf, base);
1704
1705         for (cp = ref; (ch = *cp) != 0; cp++)
1706                 if (needs_quote(ch))
1707                         strbuf_addf(&buf, "%%%02x", ch);
1708                 else
1709                         strbuf_addch(&buf, *cp);
1710
1711         return strbuf_detach(&buf, NULL);
1712 }
1713
1714 void append_remote_object_url(struct strbuf *buf, const char *url,
1715                               const char *hex,
1716                               int only_two_digit_prefix)
1717 {
1718         end_url_with_slash(buf, url);
1719
1720         strbuf_addf(buf, "objects/%.*s/", 2, hex);
1721         if (!only_two_digit_prefix)
1722                 strbuf_addstr(buf, hex + 2);
1723 }
1724
1725 char *get_remote_object_url(const char *url, const char *hex,
1726                             int only_two_digit_prefix)
1727 {
1728         struct strbuf buf = STRBUF_INIT;
1729         append_remote_object_url(&buf, url, hex, only_two_digit_prefix);
1730         return strbuf_detach(&buf, NULL);
1731 }
1732
1733 void normalize_curl_result(CURLcode *result, long http_code,
1734                            char *errorstr, size_t errorlen)
1735 {
1736         /*
1737          * If we see a failing http code with CURLE_OK, we have turned off
1738          * FAILONERROR (to keep the server's custom error response), and should
1739          * translate the code into failure here.
1740          *
1741          * Likewise, if we see a redirect (30x code), that means we turned off
1742          * redirect-following, and we should treat the result as an error.
1743          */
1744         if (*result == CURLE_OK && http_code >= 300) {
1745                 *result = CURLE_HTTP_RETURNED_ERROR;
1746                 /*
1747                  * Normally curl will already have put the "reason phrase"
1748                  * from the server into curl_errorstr; unfortunately without
1749                  * FAILONERROR it is lost, so we can give only the numeric
1750                  * status code.
1751                  */
1752                 xsnprintf(errorstr, errorlen,
1753                           "The requested URL returned error: %ld",
1754                           http_code);
1755         }
1756 }
1757
1758 static int handle_curl_result(struct slot_results *results)
1759 {
1760         normalize_curl_result(&results->curl_result, results->http_code,
1761                               curl_errorstr, sizeof(curl_errorstr));
1762
1763         if (results->curl_result == CURLE_OK) {
1764                 credential_approve(&http_auth);
1765                 credential_approve(&proxy_auth);
1766                 credential_approve(&cert_auth);
1767                 return HTTP_OK;
1768         } else if (results->curl_result == CURLE_SSL_CERTPROBLEM) {
1769                 /*
1770                  * We can't tell from here whether it's a bad path, bad
1771                  * certificate, bad password, or something else wrong
1772                  * with the certificate.  So we reject the credential to
1773                  * avoid caching or saving a bad password.
1774                  */
1775                 credential_reject(&cert_auth);
1776                 return HTTP_NOAUTH;
1777 #ifdef GIT_CURL_HAVE_CURLE_SSL_PINNEDPUBKEYNOTMATCH
1778         } else if (results->curl_result == CURLE_SSL_PINNEDPUBKEYNOTMATCH) {
1779                 return HTTP_NOMATCHPUBLICKEY;
1780 #endif
1781         } else if (missing_target(results))
1782                 return HTTP_MISSING_TARGET;
1783         else if (results->http_code == 401) {
1784                 if ((http_auth.username && http_auth.password) ||\
1785                     (http_auth.authtype && http_auth.credential)) {
1786                         if (http_auth.multistage) {
1787                                 credential_clear_secrets(&http_auth);
1788                                 return HTTP_REAUTH;
1789                         }
1790                         credential_reject(&http_auth);
1791                         return HTTP_NOAUTH;
1792                 } else {
1793                         http_auth_methods &= ~CURLAUTH_GSSNEGOTIATE;
1794                         if (results->auth_avail) {
1795                                 http_auth_methods &= results->auth_avail;
1796                                 http_auth_methods_restricted = 1;
1797                         }
1798                         return HTTP_REAUTH;
1799                 }
1800         } else {
1801                 if (results->http_connectcode == 407)
1802                         credential_reject(&proxy_auth);
1803                 if (!curl_errorstr[0])
1804                         strlcpy(curl_errorstr,
1805                                 curl_easy_strerror(results->curl_result),
1806                                 sizeof(curl_errorstr));
1807                 return HTTP_ERROR;
1808         }
1809 }
1810
1811 int run_one_slot(struct active_request_slot *slot,
1812                  struct slot_results *results)
1813 {
1814         slot->results = results;
1815         if (!start_active_slot(slot)) {
1816                 xsnprintf(curl_errorstr, sizeof(curl_errorstr),
1817                           "failed to start HTTP request");
1818                 return HTTP_START_FAILED;
1819         }
1820
1821         run_active_slot(slot);
1822         return handle_curl_result(results);
1823 }
1824
1825 struct curl_slist *http_copy_default_headers(void)
1826 {
1827         struct curl_slist *headers = NULL;
1828         const struct string_list_item *item;
1829
1830         for_each_string_list_item(item, &extra_http_headers)
1831                 headers = curl_slist_append(headers, item->string);
1832
1833         return headers;
1834 }
1835
1836 static CURLcode curlinfo_strbuf(CURL *curl, CURLINFO info, struct strbuf *buf)
1837 {
1838         char *ptr;
1839         CURLcode ret;
1840
1841         strbuf_reset(buf);
1842         ret = curl_easy_getinfo(curl, info, &ptr);
1843         if (!ret && ptr)
1844                 strbuf_addstr(buf, ptr);
1845         return ret;
1846 }
1847
1848 /*
1849  * Check for and extract a content-type parameter. "raw"
1850  * should be positioned at the start of the potential
1851  * parameter, with any whitespace already removed.
1852  *
1853  * "name" is the name of the parameter. The value is appended
1854  * to "out".
1855  */
1856 static int extract_param(const char *raw, const char *name,
1857                          struct strbuf *out)
1858 {
1859         size_t len = strlen(name);
1860
1861         if (strncasecmp(raw, name, len))
1862                 return -1;
1863         raw += len;
1864
1865         if (*raw != '=')
1866                 return -1;
1867         raw++;
1868
1869         while (*raw && !isspace(*raw) && *raw != ';')
1870                 strbuf_addch(out, *raw++);
1871         return 0;
1872 }
1873
1874 /*
1875  * Extract a normalized version of the content type, with any
1876  * spaces suppressed, all letters lowercased, and no trailing ";"
1877  * or parameters.
1878  *
1879  * Note that we will silently remove even invalid whitespace. For
1880  * example, "text / plain" is specifically forbidden by RFC 2616,
1881  * but "text/plain" is the only reasonable output, and this keeps
1882  * our code simple.
1883  *
1884  * If the "charset" argument is not NULL, store the value of any
1885  * charset parameter there.
1886  *
1887  * Example:
1888  *   "TEXT/PLAIN; charset=utf-8" -> "text/plain", "utf-8"
1889  *   "text / plain" -> "text/plain"
1890  */
1891 static void extract_content_type(struct strbuf *raw, struct strbuf *type,
1892                                  struct strbuf *charset)
1893 {
1894         const char *p;
1895
1896         strbuf_reset(type);
1897         strbuf_grow(type, raw->len);
1898         for (p = raw->buf; *p; p++) {
1899                 if (isspace(*p))
1900                         continue;
1901                 if (*p == ';') {
1902                         p++;
1903                         break;
1904                 }
1905                 strbuf_addch(type, tolower(*p));
1906         }
1907
1908         if (!charset)
1909                 return;
1910
1911         strbuf_reset(charset);
1912         while (*p) {
1913                 while (isspace(*p) || *p == ';')
1914                         p++;
1915                 if (!extract_param(p, "charset", charset))
1916                         return;
1917                 while (*p && !isspace(*p))
1918                         p++;
1919         }
1920
1921         if (!charset->len && starts_with(type->buf, "text/"))
1922                 strbuf_addstr(charset, "ISO-8859-1");
1923 }
1924
1925 static void write_accept_language(struct strbuf *buf)
1926 {
1927         /*
1928          * MAX_DECIMAL_PLACES must not be larger than 3. If it is larger than
1929          * that, q-value will be smaller than 0.001, the minimum q-value the
1930          * HTTP specification allows. See
1931          * https://datatracker.ietf.org/doc/html/rfc7231#section-5.3.1 for q-value.
1932          */
1933         const int MAX_DECIMAL_PLACES = 3;
1934         const int MAX_LANGUAGE_TAGS = 1000;
1935         const int MAX_ACCEPT_LANGUAGE_HEADER_SIZE = 4000;
1936         char **language_tags = NULL;
1937         int num_langs = 0;
1938         const char *s = get_preferred_languages();
1939         int i;
1940         struct strbuf tag = STRBUF_INIT;
1941
1942         /* Don't add Accept-Language header if no language is preferred. */
1943         if (!s)
1944                 return;
1945
1946         /*
1947          * Split the colon-separated string of preferred languages into
1948          * language_tags array.
1949          */
1950         do {
1951                 /* collect language tag */
1952                 for (; *s && (isalnum(*s) || *s == '_'); s++)
1953                         strbuf_addch(&tag, *s == '_' ? '-' : *s);
1954
1955                 /* skip .codeset, @modifier and any other unnecessary parts */
1956                 while (*s && *s != ':')
1957                         s++;
1958
1959                 if (tag.len) {
1960                         num_langs++;
1961                         REALLOC_ARRAY(language_tags, num_langs);
1962                         language_tags[num_langs - 1] = strbuf_detach(&tag, NULL);
1963                         if (num_langs >= MAX_LANGUAGE_TAGS - 1) /* -1 for '*' */
1964                                 break;
1965                 }
1966         } while (*s++);
1967
1968         /* write Accept-Language header into buf */
1969         if (num_langs) {
1970                 int last_buf_len = 0;
1971                 int max_q;
1972                 int decimal_places;
1973                 char q_format[32];
1974
1975                 /* add '*' */
1976                 REALLOC_ARRAY(language_tags, num_langs + 1);
1977                 language_tags[num_langs++] = xstrdup("*");
1978
1979                 /* compute decimal_places */
1980                 for (max_q = 1, decimal_places = 0;
1981                      max_q < num_langs && decimal_places <= MAX_DECIMAL_PLACES;
1982                      decimal_places++, max_q *= 10)
1983                         ;
1984
1985                 xsnprintf(q_format, sizeof(q_format), ";q=0.%%0%dd", decimal_places);
1986
1987                 strbuf_addstr(buf, "Accept-Language: ");
1988
1989                 for (i = 0; i < num_langs; i++) {
1990                         if (i > 0)
1991                                 strbuf_addstr(buf, ", ");
1992
1993                         strbuf_addstr(buf, language_tags[i]);
1994
1995                         if (i > 0)
1996                                 strbuf_addf(buf, q_format, max_q - i);
1997
1998                         if (buf->len > MAX_ACCEPT_LANGUAGE_HEADER_SIZE) {
1999                                 strbuf_remove(buf, last_buf_len, buf->len - last_buf_len);
2000                                 break;
2001                         }
2002
2003                         last_buf_len = buf->len;
2004                 }
2005         }
2006
2007         for (i = 0; i < num_langs; i++)
2008                 free(language_tags[i]);
2009         free(language_tags);
2010 }
2011
2012 /*
2013  * Get an Accept-Language header which indicates user's preferred languages.
2014  *
2015  * Examples:
2016  *   LANGUAGE= -> ""
2017  *   LANGUAGE=ko:en -> "Accept-Language: ko, en; q=0.9, *; q=0.1"
2018  *   LANGUAGE=ko_KR.UTF-8:sr@latin -> "Accept-Language: ko-KR, sr; q=0.9, *; q=0.1"
2019  *   LANGUAGE=ko LANG=en_US.UTF-8 -> "Accept-Language: ko, *; q=0.1"
2020  *   LANGUAGE= LANG=en_US.UTF-8 -> "Accept-Language: en-US, *; q=0.1"
2021  *   LANGUAGE= LANG=C -> ""
2022  */
2023 const char *http_get_accept_language_header(void)
2024 {
2025         if (!cached_accept_language) {
2026                 struct strbuf buf = STRBUF_INIT;
2027                 write_accept_language(&buf);
2028                 if (buf.len > 0)
2029                         cached_accept_language = strbuf_detach(&buf, NULL);
2030         }
2031
2032         return cached_accept_language;
2033 }
2034
2035 static void http_opt_request_remainder(CURL *curl, off_t pos)
2036 {
2037         char buf[128];
2038         xsnprintf(buf, sizeof(buf), "%"PRIuMAX"-", (uintmax_t)pos);
2039         curl_easy_setopt(curl, CURLOPT_RANGE, buf);
2040 }
2041
2042 /* http_request() targets */
2043 #define HTTP_REQUEST_STRBUF     0
2044 #define HTTP_REQUEST_FILE       1
2045
2046 static int http_request(const char *url,
2047                         void *result, int target,
2048                         const struct http_get_options *options)
2049 {
2050         struct active_request_slot *slot;
2051         struct slot_results results;
2052         struct curl_slist *headers = http_copy_default_headers();
2053         struct strbuf buf = STRBUF_INIT;
2054         const char *accept_language;
2055         int ret;
2056
2057         slot = get_active_slot();
2058         curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
2059
2060         if (!result) {
2061                 curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1);
2062         } else {
2063                 curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
2064                 curl_easy_setopt(slot->curl, CURLOPT_WRITEDATA, result);
2065
2066                 if (target == HTTP_REQUEST_FILE) {
2067                         off_t posn = ftello(result);
2068                         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
2069                                          fwrite);
2070                         if (posn > 0)
2071                                 http_opt_request_remainder(slot->curl, posn);
2072                 } else
2073                         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
2074                                          fwrite_buffer);
2075         }
2076
2077         curl_easy_setopt(slot->curl, CURLOPT_HEADERFUNCTION, fwrite_wwwauth);
2078
2079         accept_language = http_get_accept_language_header();
2080
2081         if (accept_language)
2082                 headers = curl_slist_append(headers, accept_language);
2083
2084         strbuf_addstr(&buf, "Pragma:");
2085         if (options && options->no_cache)
2086                 strbuf_addstr(&buf, " no-cache");
2087         if (options && options->initial_request &&
2088             http_follow_config == HTTP_FOLLOW_INITIAL)
2089                 curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 1);
2090
2091         headers = curl_slist_append(headers, buf.buf);
2092
2093         /* Add additional headers here */
2094         if (options && options->extra_headers) {
2095                 const struct string_list_item *item;
2096                 if (options && options->extra_headers) {
2097                         for_each_string_list_item(item, options->extra_headers) {
2098                                 headers = curl_slist_append(headers, item->string);
2099                         }
2100                 }
2101         }
2102
2103         headers = http_append_auth_header(&http_auth, headers);
2104
2105         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
2106         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);
2107         curl_easy_setopt(slot->curl, CURLOPT_ENCODING, "");
2108         curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 0);
2109
2110         ret = run_one_slot(slot, &results);
2111
2112         if (options && options->content_type) {
2113                 struct strbuf raw = STRBUF_INIT;
2114                 curlinfo_strbuf(slot->curl, CURLINFO_CONTENT_TYPE, &raw);
2115                 extract_content_type(&raw, options->content_type,
2116                                      options->charset);
2117                 strbuf_release(&raw);
2118         }
2119
2120         if (options && options->effective_url)
2121                 curlinfo_strbuf(slot->curl, CURLINFO_EFFECTIVE_URL,
2122                                 options->effective_url);
2123
2124         curl_slist_free_all(headers);
2125         strbuf_release(&buf);
2126
2127         return ret;
2128 }
2129
2130 /*
2131  * Update the "base" url to a more appropriate value, as deduced by
2132  * redirects seen when requesting a URL starting with "url".
2133  *
2134  * The "asked" parameter is a URL that we asked curl to access, and must begin
2135  * with "base".
2136  *
2137  * The "got" parameter is the URL that curl reported to us as where we ended
2138  * up.
2139  *
2140  * Returns 1 if we updated the base url, 0 otherwise.
2141  *
2142  * Our basic strategy is to compare "base" and "asked" to find the bits
2143  * specific to our request. We then strip those bits off of "got" to yield the
2144  * new base. So for example, if our base is "http://example.com/foo.git",
2145  * and we ask for "http://example.com/foo.git/info/refs", we might end up
2146  * with "https://other.example.com/foo.git/info/refs". We would want the
2147  * new URL to become "https://other.example.com/foo.git".
2148  *
2149  * Note that this assumes a sane redirect scheme. It's entirely possible
2150  * in the example above to end up at a URL that does not even end in
2151  * "info/refs".  In such a case we die. There's not much we can do, such a
2152  * scheme is unlikely to represent a real git repository, and failing to
2153  * rewrite the base opens options for malicious redirects to do funny things.
2154  */
2155 static int update_url_from_redirect(struct strbuf *base,
2156                                     const char *asked,
2157                                     const struct strbuf *got)
2158 {
2159         const char *tail;
2160         size_t new_len;
2161
2162         if (!strcmp(asked, got->buf))
2163                 return 0;
2164
2165         if (!skip_prefix(asked, base->buf, &tail))
2166                 BUG("update_url_from_redirect: %s is not a superset of %s",
2167                     asked, base->buf);
2168
2169         new_len = got->len;
2170         if (!strip_suffix_mem(got->buf, &new_len, tail))
2171                 die(_("unable to update url base from redirection:\n"
2172                       "  asked for: %s\n"
2173                       "   redirect: %s"),
2174                     asked, got->buf);
2175
2176         strbuf_reset(base);
2177         strbuf_add(base, got->buf, new_len);
2178
2179         return 1;
2180 }
2181
2182 static int http_request_reauth(const char *url,
2183                                void *result, int target,
2184                                struct http_get_options *options)
2185 {
2186         int i = 3;
2187         int ret = http_request(url, result, target, options);
2188
2189         if (ret != HTTP_OK && ret != HTTP_REAUTH)
2190                 return ret;
2191
2192         if (options && options->effective_url && options->base_url) {
2193                 if (update_url_from_redirect(options->base_url,
2194                                              url, options->effective_url)) {
2195                         credential_from_url(&http_auth, options->base_url->buf);
2196                         url = options->effective_url->buf;
2197                 }
2198         }
2199
2200         while (ret == HTTP_REAUTH && --i) {
2201                 /*
2202                  * The previous request may have put cruft into our output stream; we
2203                  * should clear it out before making our next request.
2204                  */
2205                 switch (target) {
2206                 case HTTP_REQUEST_STRBUF:
2207                         strbuf_reset(result);
2208                         break;
2209                 case HTTP_REQUEST_FILE:
2210                         if (fflush(result)) {
2211                                 error_errno("unable to flush a file");
2212                                 return HTTP_START_FAILED;
2213                         }
2214                         rewind(result);
2215                         if (ftruncate(fileno(result), 0) < 0) {
2216                                 error_errno("unable to truncate a file");
2217                                 return HTTP_START_FAILED;
2218                         }
2219                         break;
2220                 default:
2221                         BUG("Unknown http_request target");
2222                 }
2223
2224                 credential_fill(&http_auth, 1);
2225
2226                 ret = http_request(url, result, target, options);
2227         }
2228         return ret;
2229 }
2230
2231 int http_get_strbuf(const char *url,
2232                     struct strbuf *result,
2233                     struct http_get_options *options)
2234 {
2235         return http_request_reauth(url, result, HTTP_REQUEST_STRBUF, options);
2236 }
2237
2238 /*
2239  * Downloads a URL and stores the result in the given file.
2240  *
2241  * If a previous interrupted download is detected (i.e. a previous temporary
2242  * file is still around) the download is resumed.
2243  */
2244 int http_get_file(const char *url, const char *filename,
2245                   struct http_get_options *options)
2246 {
2247         int ret;
2248         struct strbuf tmpfile = STRBUF_INIT;
2249         FILE *result;
2250
2251         strbuf_addf(&tmpfile, "%s.temp", filename);
2252         result = fopen(tmpfile.buf, "a");
2253         if (!result) {
2254                 error("Unable to open local file %s", tmpfile.buf);
2255                 ret = HTTP_ERROR;
2256                 goto cleanup;
2257         }
2258
2259         ret = http_request_reauth(url, result, HTTP_REQUEST_FILE, options);
2260         fclose(result);
2261
2262         if (ret == HTTP_OK && finalize_object_file(tmpfile.buf, filename))
2263                 ret = HTTP_ERROR;
2264 cleanup:
2265         strbuf_release(&tmpfile);
2266         return ret;
2267 }
2268
2269 int http_fetch_ref(const char *base, struct ref *ref)
2270 {
2271         struct http_get_options options = {0};
2272         char *url;
2273         struct strbuf buffer = STRBUF_INIT;
2274         int ret = -1;
2275
2276         options.no_cache = 1;
2277
2278         url = quote_ref_url(base, ref->name);
2279         if (http_get_strbuf(url, &buffer, &options) == HTTP_OK) {
2280                 strbuf_rtrim(&buffer);
2281                 if (buffer.len == the_hash_algo->hexsz)
2282                         ret = get_oid_hex(buffer.buf, &ref->old_oid);
2283                 else if (starts_with(buffer.buf, "ref: ")) {
2284                         ref->symref = xstrdup(buffer.buf + 5);
2285                         ret = 0;
2286                 }
2287         }
2288
2289         strbuf_release(&buffer);
2290         free(url);
2291         return ret;
2292 }
2293
2294 /* Helpers for fetching packs */
2295 static char *fetch_pack_index(unsigned char *hash, const char *base_url)
2296 {
2297         char *url, *tmp;
2298         struct strbuf buf = STRBUF_INIT;
2299
2300         if (http_is_verbose)
2301                 fprintf(stderr, "Getting index for pack %s\n", hash_to_hex(hash));
2302
2303         end_url_with_slash(&buf, base_url);
2304         strbuf_addf(&buf, "objects/pack/pack-%s.idx", hash_to_hex(hash));
2305         url = strbuf_detach(&buf, NULL);
2306
2307         strbuf_addf(&buf, "%s.temp", sha1_pack_index_name(hash));
2308         tmp = strbuf_detach(&buf, NULL);
2309
2310         if (http_get_file(url, tmp, NULL) != HTTP_OK) {
2311                 error("Unable to get pack index %s", url);
2312                 FREE_AND_NULL(tmp);
2313         }
2314
2315         free(url);
2316         return tmp;
2317 }
2318
2319 static int fetch_and_setup_pack_index(struct packed_git **packs_head,
2320         unsigned char *sha1, const char *base_url)
2321 {
2322         struct packed_git *new_pack;
2323         char *tmp_idx = NULL;
2324         int ret;
2325
2326         if (has_pack_index(sha1)) {
2327                 new_pack = parse_pack_index(sha1, sha1_pack_index_name(sha1));
2328                 if (!new_pack)
2329                         return -1; /* parse_pack_index() already issued error message */
2330                 goto add_pack;
2331         }
2332
2333         tmp_idx = fetch_pack_index(sha1, base_url);
2334         if (!tmp_idx)
2335                 return -1;
2336
2337         new_pack = parse_pack_index(sha1, tmp_idx);
2338         if (!new_pack) {
2339                 unlink(tmp_idx);
2340                 free(tmp_idx);
2341
2342                 return -1; /* parse_pack_index() already issued error message */
2343         }
2344
2345         ret = verify_pack_index(new_pack);
2346         if (!ret) {
2347                 close_pack_index(new_pack);
2348                 ret = finalize_object_file(tmp_idx, sha1_pack_index_name(sha1));
2349         }
2350         free(tmp_idx);
2351         if (ret)
2352                 return -1;
2353
2354 add_pack:
2355         new_pack->next = *packs_head;
2356         *packs_head = new_pack;
2357         return 0;
2358 }
2359
2360 int http_get_info_packs(const char *base_url, struct packed_git **packs_head)
2361 {
2362         struct http_get_options options = {0};
2363         int ret = 0;
2364         char *url;
2365         const char *data;
2366         struct strbuf buf = STRBUF_INIT;
2367         struct object_id oid;
2368
2369         end_url_with_slash(&buf, base_url);
2370         strbuf_addstr(&buf, "objects/info/packs");
2371         url = strbuf_detach(&buf, NULL);
2372
2373         options.no_cache = 1;
2374         ret = http_get_strbuf(url, &buf, &options);
2375         if (ret != HTTP_OK)
2376                 goto cleanup;
2377
2378         data = buf.buf;
2379         while (*data) {
2380                 if (skip_prefix(data, "P pack-", &data) &&
2381                     !parse_oid_hex(data, &oid, &data) &&
2382                     skip_prefix(data, ".pack", &data) &&
2383                     (*data == '\n' || *data == '\0')) {
2384                         fetch_and_setup_pack_index(packs_head, oid.hash, base_url);
2385                 } else {
2386                         data = strchrnul(data, '\n');
2387                 }
2388                 if (*data)
2389                         data++; /* skip past newline */
2390         }
2391
2392 cleanup:
2393         free(url);
2394         return ret;
2395 }
2396
2397 void release_http_pack_request(struct http_pack_request *preq)
2398 {
2399         if (preq->packfile) {
2400                 fclose(preq->packfile);
2401                 preq->packfile = NULL;
2402         }
2403         preq->slot = NULL;
2404         strbuf_release(&preq->tmpfile);
2405         curl_slist_free_all(preq->headers);
2406         free(preq->url);
2407         free(preq);
2408 }
2409
2410 static const char *default_index_pack_args[] =
2411         {"index-pack", "--stdin", NULL};
2412
2413 int finish_http_pack_request(struct http_pack_request *preq)
2414 {
2415         struct child_process ip = CHILD_PROCESS_INIT;
2416         int tmpfile_fd;
2417         int ret = 0;
2418
2419         fclose(preq->packfile);
2420         preq->packfile = NULL;
2421
2422         tmpfile_fd = xopen(preq->tmpfile.buf, O_RDONLY);
2423
2424         ip.git_cmd = 1;
2425         ip.in = tmpfile_fd;
2426         strvec_pushv(&ip.args, preq->index_pack_args ?
2427                      preq->index_pack_args :
2428                      default_index_pack_args);
2429
2430         if (preq->preserve_index_pack_stdout)
2431                 ip.out = 0;
2432         else
2433                 ip.no_stdout = 1;
2434
2435         if (run_command(&ip)) {
2436                 ret = -1;
2437                 goto cleanup;
2438         }
2439
2440 cleanup:
2441         close(tmpfile_fd);
2442         unlink(preq->tmpfile.buf);
2443         return ret;
2444 }
2445
2446 void http_install_packfile(struct packed_git *p,
2447                            struct packed_git **list_to_remove_from)
2448 {
2449         struct packed_git **lst = list_to_remove_from;
2450
2451         while (*lst != p)
2452                 lst = &((*lst)->next);
2453         *lst = (*lst)->next;
2454
2455         install_packed_git(the_repository, p);
2456 }
2457
2458 struct http_pack_request *new_http_pack_request(
2459         const unsigned char *packed_git_hash, const char *base_url) {
2460
2461         struct strbuf buf = STRBUF_INIT;
2462
2463         end_url_with_slash(&buf, base_url);
2464         strbuf_addf(&buf, "objects/pack/pack-%s.pack",
2465                 hash_to_hex(packed_git_hash));
2466         return new_direct_http_pack_request(packed_git_hash,
2467                                             strbuf_detach(&buf, NULL));
2468 }
2469
2470 struct http_pack_request *new_direct_http_pack_request(
2471         const unsigned char *packed_git_hash, char *url)
2472 {
2473         off_t prev_posn = 0;
2474         struct http_pack_request *preq;
2475
2476         CALLOC_ARRAY(preq, 1);
2477         strbuf_init(&preq->tmpfile, 0);
2478
2479         preq->url = url;
2480
2481         strbuf_addf(&preq->tmpfile, "%s.temp", sha1_pack_name(packed_git_hash));
2482         preq->packfile = fopen(preq->tmpfile.buf, "a");
2483         if (!preq->packfile) {
2484                 error("Unable to open local file %s for pack",
2485                       preq->tmpfile.buf);
2486                 goto abort;
2487         }
2488
2489         preq->slot = get_active_slot();
2490         preq->headers = object_request_headers();
2491         curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEDATA, preq->packfile);
2492         curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
2493         curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url);
2494         curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER, preq->headers);
2495
2496         /*
2497          * If there is data present from a previous transfer attempt,
2498          * resume where it left off
2499          */
2500         prev_posn = ftello(preq->packfile);
2501         if (prev_posn>0) {
2502                 if (http_is_verbose)
2503                         fprintf(stderr,
2504                                 "Resuming fetch of pack %s at byte %"PRIuMAX"\n",
2505                                 hash_to_hex(packed_git_hash),
2506                                 (uintmax_t)prev_posn);
2507                 http_opt_request_remainder(preq->slot->curl, prev_posn);
2508         }
2509
2510         return preq;
2511
2512 abort:
2513         strbuf_release(&preq->tmpfile);
2514         free(preq->url);
2515         free(preq);
2516         return NULL;
2517 }
2518
2519 /* Helpers for fetching objects (loose) */
2520 static size_t fwrite_sha1_file(char *ptr, size_t eltsize, size_t nmemb,
2521                                void *data)
2522 {
2523         unsigned char expn[4096];
2524         size_t size = eltsize * nmemb;
2525         int posn = 0;
2526         struct http_object_request *freq = data;
2527         struct active_request_slot *slot = freq->slot;
2528
2529         if (slot) {
2530                 CURLcode c = curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE,
2531                                                 &slot->http_code);
2532                 if (c != CURLE_OK)
2533                         BUG("curl_easy_getinfo for HTTP code failed: %s",
2534                                 curl_easy_strerror(c));
2535                 if (slot->http_code >= 300)
2536                         return nmemb;
2537         }
2538
2539         do {
2540                 ssize_t retval = xwrite(freq->localfile,
2541                                         (char *) ptr + posn, size - posn);
2542                 if (retval < 0)
2543                         return posn / eltsize;
2544                 posn += retval;
2545         } while (posn < size);
2546
2547         freq->stream.avail_in = size;
2548         freq->stream.next_in = (void *)ptr;
2549         do {
2550                 freq->stream.next_out = expn;
2551                 freq->stream.avail_out = sizeof(expn);
2552                 freq->zret = git_inflate(&freq->stream, Z_SYNC_FLUSH);
2553                 the_hash_algo->update_fn(&freq->c, expn,
2554                                          sizeof(expn) - freq->stream.avail_out);
2555         } while (freq->stream.avail_in && freq->zret == Z_OK);
2556         return nmemb;
2557 }
2558
2559 struct http_object_request *new_http_object_request(const char *base_url,
2560                                                     const struct object_id *oid)
2561 {
2562         char *hex = oid_to_hex(oid);
2563         struct strbuf filename = STRBUF_INIT;
2564         struct strbuf prevfile = STRBUF_INIT;
2565         int prevlocal;
2566         char prev_buf[PREV_BUF_SIZE];
2567         ssize_t prev_read = 0;
2568         off_t prev_posn = 0;
2569         struct http_object_request *freq;
2570
2571         CALLOC_ARRAY(freq, 1);
2572         strbuf_init(&freq->tmpfile, 0);
2573         oidcpy(&freq->oid, oid);
2574         freq->localfile = -1;
2575
2576         loose_object_path(the_repository, &filename, oid);
2577         strbuf_addf(&freq->tmpfile, "%s.temp", filename.buf);
2578
2579         strbuf_addf(&prevfile, "%s.prev", filename.buf);
2580         unlink_or_warn(prevfile.buf);
2581         rename(freq->tmpfile.buf, prevfile.buf);
2582         unlink_or_warn(freq->tmpfile.buf);
2583         strbuf_release(&filename);
2584
2585         if (freq->localfile != -1)
2586                 error("fd leakage in start: %d", freq->localfile);
2587         freq->localfile = open(freq->tmpfile.buf,
2588                                O_WRONLY | O_CREAT | O_EXCL, 0666);
2589         /*
2590          * This could have failed due to the "lazy directory creation";
2591          * try to mkdir the last path component.
2592          */
2593         if (freq->localfile < 0 && errno == ENOENT) {
2594                 char *dir = strrchr(freq->tmpfile.buf, '/');
2595                 if (dir) {
2596                         *dir = 0;
2597                         mkdir(freq->tmpfile.buf, 0777);
2598                         *dir = '/';
2599                 }
2600                 freq->localfile = open(freq->tmpfile.buf,
2601                                        O_WRONLY | O_CREAT | O_EXCL, 0666);
2602         }
2603
2604         if (freq->localfile < 0) {
2605                 error_errno("Couldn't create temporary file %s",
2606                             freq->tmpfile.buf);
2607                 goto abort;
2608         }
2609
2610         git_inflate_init(&freq->stream);
2611
2612         the_hash_algo->init_fn(&freq->c);
2613
2614         freq->url = get_remote_object_url(base_url, hex, 0);
2615
2616         /*
2617          * If a previous temp file is present, process what was already
2618          * fetched.
2619          */
2620         prevlocal = open(prevfile.buf, O_RDONLY);
2621         if (prevlocal != -1) {
2622                 do {
2623                         prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
2624                         if (prev_read>0) {
2625                                 if (fwrite_sha1_file(prev_buf,
2626                                                      1,
2627                                                      prev_read,
2628                                                      freq) == prev_read) {
2629                                         prev_posn += prev_read;
2630                                 } else {
2631                                         prev_read = -1;
2632                                 }
2633                         }
2634                 } while (prev_read > 0);
2635                 close(prevlocal);
2636         }
2637         unlink_or_warn(prevfile.buf);
2638         strbuf_release(&prevfile);
2639
2640         /*
2641          * Reset inflate/SHA1 if there was an error reading the previous temp
2642          * file; also rewind to the beginning of the local file.
2643          */
2644         if (prev_read == -1) {
2645                 memset(&freq->stream, 0, sizeof(freq->stream));
2646                 git_inflate_init(&freq->stream);
2647                 the_hash_algo->init_fn(&freq->c);
2648                 if (prev_posn>0) {
2649                         prev_posn = 0;
2650                         lseek(freq->localfile, 0, SEEK_SET);
2651                         if (ftruncate(freq->localfile, 0) < 0) {
2652                                 error_errno("Couldn't truncate temporary file %s",
2653                                             freq->tmpfile.buf);
2654                                 goto abort;
2655                         }
2656                 }
2657         }
2658
2659         freq->slot = get_active_slot();
2660         freq->headers = object_request_headers();
2661
2662         curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEDATA, freq);
2663         curl_easy_setopt(freq->slot->curl, CURLOPT_FAILONERROR, 0);
2664         curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
2665         curl_easy_setopt(freq->slot->curl, CURLOPT_ERRORBUFFER, freq->errorstr);
2666         curl_easy_setopt(freq->slot->curl, CURLOPT_URL, freq->url);
2667         curl_easy_setopt(freq->slot->curl, CURLOPT_HTTPHEADER, freq->headers);
2668
2669         /*
2670          * If we have successfully processed data from a previous fetch
2671          * attempt, only fetch the data we don't already have.
2672          */
2673         if (prev_posn>0) {
2674                 if (http_is_verbose)
2675                         fprintf(stderr,
2676                                 "Resuming fetch of object %s at byte %"PRIuMAX"\n",
2677                                 hex, (uintmax_t)prev_posn);
2678                 http_opt_request_remainder(freq->slot->curl, prev_posn);
2679         }
2680
2681         return freq;
2682
2683 abort:
2684         strbuf_release(&prevfile);
2685         free(freq->url);
2686         free(freq);
2687         return NULL;
2688 }
2689
2690 void process_http_object_request(struct http_object_request *freq)
2691 {
2692         if (!freq->slot)
2693                 return;
2694         freq->curl_result = freq->slot->curl_result;
2695         freq->http_code = freq->slot->http_code;
2696         freq->slot = NULL;
2697 }
2698
2699 int finish_http_object_request(struct http_object_request *freq)
2700 {
2701         struct stat st;
2702         struct strbuf filename = STRBUF_INIT;
2703
2704         close(freq->localfile);
2705         freq->localfile = -1;
2706
2707         process_http_object_request(freq);
2708
2709         if (freq->http_code == 416) {
2710                 warning("requested range invalid; we may already have all the data.");
2711         } else if (freq->curl_result != CURLE_OK) {
2712                 if (stat(freq->tmpfile.buf, &st) == 0)
2713                         if (st.st_size == 0)
2714                                 unlink_or_warn(freq->tmpfile.buf);
2715                 return -1;
2716         }
2717
2718         git_inflate_end(&freq->stream);
2719         the_hash_algo->final_oid_fn(&freq->real_oid, &freq->c);
2720         if (freq->zret != Z_STREAM_END) {
2721                 unlink_or_warn(freq->tmpfile.buf);
2722                 return -1;
2723         }
2724         if (!oideq(&freq->oid, &freq->real_oid)) {
2725                 unlink_or_warn(freq->tmpfile.buf);
2726                 return -1;
2727         }
2728         loose_object_path(the_repository, &filename, &freq->oid);
2729         freq->rename = finalize_object_file(freq->tmpfile.buf, filename.buf);
2730         strbuf_release(&filename);
2731
2732         return freq->rename;
2733 }
2734
2735 void abort_http_object_request(struct http_object_request *freq)
2736 {
2737         unlink_or_warn(freq->tmpfile.buf);
2738
2739         release_http_object_request(freq);
2740 }
2741
2742 void release_http_object_request(struct http_object_request *freq)
2743 {
2744         if (freq->localfile != -1) {
2745                 close(freq->localfile);
2746                 freq->localfile = -1;
2747         }
2748         FREE_AND_NULL(freq->url);
2749         if (freq->slot) {
2750                 freq->slot->callback_func = NULL;
2751                 freq->slot->callback_data = NULL;
2752                 release_active_slot(freq->slot);
2753                 freq->slot = NULL;
2754         }
2755         curl_slist_free_all(freq->headers);
2756         strbuf_release(&freq->tmpfile);
2757 }