libzypp 17.25.10
MediaCurl.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
13 #include <iostream>
14 #include <list>
15 
16 #include <zypp/base/Logger.h>
17 #include <zypp/ExternalProgram.h>
18 #include <zypp/base/String.h>
19 #include <zypp/base/Gettext.h>
20 #include <zypp/base/Sysconfig.h>
21 #include <zypp/base/Gettext.h>
22 
23 #include <zypp/media/MediaCurl.h>
24 #include <zypp/media/ProxyInfo.h>
27 #include <zypp/media/CurlConfig.h>
28 #include <zypp/media/CurlHelper.h>
29 #include <zypp/Target.h>
30 #include <zypp/ZYppFactory.h>
31 #include <zypp/ZConfig.h>
32 
33 #include <cstdlib>
34 #include <sys/types.h>
35 #include <sys/stat.h>
36 #include <sys/mount.h>
37 #include <errno.h>
38 #include <dirent.h>
39 #include <unistd.h>
40 
41 using std::endl;
42 
43 using namespace internal;
44 using namespace zypp::base;
45 
46 namespace zypp {
47 
48  namespace media {
49 
50  namespace {
51  struct ProgressData
52  {
53  ProgressData( CURL *_curl, time_t _timeout = 0, const Url & _url = Url(),
54  ByteCount expectedFileSize_r = 0,
55  callback::SendReport<DownloadProgressReport> *_report = nullptr )
56  : curl( _curl )
57  , url( _url )
58  , timeout( _timeout )
59  , reached( false )
60  , fileSizeExceeded ( false )
61  , report( _report )
62  , _expectedFileSize( expectedFileSize_r )
63  {}
64 
65  CURL *curl;
66  Url url;
67  time_t timeout;
68  bool reached;
70  callback::SendReport<DownloadProgressReport> *report;
71  ByteCount _expectedFileSize;
72 
73  time_t _timeStart = 0;
74  time_t _timeLast = 0;
75  time_t _timeRcv = 0;
76  time_t _timeNow = 0;
77 
78  double _dnlTotal = 0.0;
79  double _dnlLast = 0.0;
80  double _dnlNow = 0.0;
81 
82  int _dnlPercent= 0;
83 
84  double _drateTotal= 0.0;
85  double _drateLast = 0.0;
86 
87  void updateStats( double dltotal = 0.0, double dlnow = 0.0 )
88  {
89  time_t now = _timeNow = time(0);
90 
91  // If called without args (0.0), recompute based on the last values seen
92  if ( dltotal && dltotal != _dnlTotal )
93  _dnlTotal = dltotal;
94 
95  if ( dlnow && dlnow != _dnlNow )
96  {
97  _timeRcv = now;
98  _dnlNow = dlnow;
99  }
100  else if ( !_dnlNow && !_dnlTotal )
101  {
102  // Start time counting as soon as first data arrives.
103  // Skip the connection / redirection time at begin.
104  return;
105  }
106 
107  // init or reset if time jumps back
108  if ( !_timeStart || _timeStart > now )
109  _timeStart = _timeLast = _timeRcv = now;
110 
111  // timeout condition
112  if ( timeout )
113  reached = ( (now - _timeRcv) > timeout );
114 
115  // check if the downloaded data is already bigger than what we expected
116  fileSizeExceeded = _expectedFileSize > 0 && _expectedFileSize < static_cast<ByteCount::SizeType>(_dnlNow);
117 
118  // percentage:
119  if ( _dnlTotal )
120  _dnlPercent = int(_dnlNow * 100 / _dnlTotal);
121 
122  // download rates:
123  _drateTotal = _dnlNow / std::max( int(now - _timeStart), 1 );
124 
125  if ( _timeLast < now )
126  {
127  _drateLast = (_dnlNow - _dnlLast) / int(now - _timeLast);
128  // start new period
129  _timeLast = now;
130  _dnlLast = _dnlNow;
131  }
132  else if ( _timeStart == _timeLast )
134  }
135 
136  int reportProgress() const
137  {
138  if ( fileSizeExceeded )
139  return 1;
140  if ( reached )
141  return 1; // no-data timeout
142  if ( report && !(*report)->progress( _dnlPercent, url, _drateTotal, _drateLast ) )
143  return 1; // user requested abort
144  return 0;
145  }
146 
147 
148  // download rate of the last period (cca 1 sec)
149  double drate_period;
150  // bytes downloaded at the start of the last period
151  double dload_period;
152  // seconds from the start of the download
153  long secs;
154  // average download rate
155  double drate_avg;
156  // last time the progress was reported
157  time_t ltime;
158  // bytes downloaded at the moment the progress was last reported
159  double dload;
160  // bytes uploaded at the moment the progress was last reported
161  double uload;
162  };
163  }
164 
165 Pathname MediaCurl::_cookieFile = "/var/lib/YaST2/cookies";
166 
167 // we use this define to unbloat code as this C setting option
168 // and catching exception is done frequently.
170 #define SET_OPTION(opt,val) do { \
171  ret = curl_easy_setopt ( _curl, opt, val ); \
172  if ( ret != 0) { \
173  ZYPP_THROW(MediaCurlSetOptException(_url, _curlError)); \
174  } \
175  } while ( false )
176 
177 #define SET_OPTION_OFFT(opt,val) SET_OPTION(opt,(curl_off_t)val)
178 #define SET_OPTION_LONG(opt,val) SET_OPTION(opt,(long)val)
179 #define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val)
180 
181 MediaCurl::MediaCurl( const Url & url_r,
182  const Pathname & attach_point_hint_r )
183  : MediaHandler( url_r, attach_point_hint_r,
184  "/", // urlpath at attachpoint
185  true ), // does_download
186  _curl( NULL ),
187  _customHeaders(0L)
188 {
189  _curlError[0] = '\0';
190  _curlDebug = 0L;
191 
192  MIL << "MediaCurl::MediaCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl;
193 
195 
196  if( !attachPoint().empty())
197  {
198  PathInfo ainfo(attachPoint());
199  Pathname apath(attachPoint() + "XXXXXX");
200  char *atemp = ::strdup( apath.asString().c_str());
201  char *atest = NULL;
202  if( !ainfo.isDir() || !ainfo.userMayRWX() ||
203  atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
204  {
205  WAR << "attach point " << ainfo.path()
206  << " is not useable for " << url_r.getScheme() << endl;
207  setAttachPoint("", true);
208  }
209  else if( atest != NULL)
210  ::rmdir(atest);
211 
212  if( atemp != NULL)
213  ::free(atemp);
214  }
215 }
216 
218 {
220 }
221 
223 {
224  return _settings;
225 }
226 
227 
228 void MediaCurl::setCookieFile( const Pathname &fileName )
229 {
230  _cookieFile = fileName;
231 }
232 
234 
235 void MediaCurl::checkProtocol(const Url &url) const
236 {
237  curl_version_info_data *curl_info = NULL;
238  curl_info = curl_version_info(CURLVERSION_NOW);
239  // curl_info does not need any free (is static)
240  if (curl_info->protocols)
241  {
242  const char * const *proto;
243  std::string scheme( url.getScheme());
244  bool found = false;
245  for(proto=curl_info->protocols; !found && *proto; ++proto)
246  {
247  if( scheme == std::string((const char *)*proto))
248  found = true;
249  }
250  if( !found)
251  {
252  std::string msg("Unsupported protocol '");
253  msg += scheme;
254  msg += "'";
256  }
257  }
258 }
259 
261 {
262  {
263  char *ptr = getenv("ZYPP_MEDIA_CURL_DEBUG");
264  _curlDebug = (ptr && *ptr) ? str::strtonum<long>( ptr) : 0L;
265  if( _curlDebug > 0)
266  {
267  curl_easy_setopt( _curl, CURLOPT_VERBOSE, 1L);
268  curl_easy_setopt( _curl, CURLOPT_DEBUGFUNCTION, log_curl);
269  curl_easy_setopt( _curl, CURLOPT_DEBUGDATA, &_curlDebug);
270  }
271  }
272 
273  curl_easy_setopt(_curl, CURLOPT_HEADERFUNCTION, log_redirects_curl);
274  curl_easy_setopt(_curl, CURLOPT_HEADERDATA, &_lastRedirect);
275  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_ERRORBUFFER, _curlError );
276  if ( ret != 0 ) {
277  ZYPP_THROW(MediaCurlSetOptException(_url, "Error setting error buffer"));
278  }
279 
280  SET_OPTION(CURLOPT_FAILONERROR, 1L);
281  SET_OPTION(CURLOPT_NOSIGNAL, 1L);
282 
283  // create non persistant settings
284  // so that we don't add headers twice
285  TransferSettings vol_settings(_settings);
286 
287  // add custom headers for download.opensuse.org (bsc#955801)
288  if ( _url.getHost() == "download.opensuse.org" )
289  {
290  vol_settings.addHeader(anonymousIdHeader());
291  vol_settings.addHeader(distributionFlavorHeader());
292  }
293  vol_settings.addHeader("Pragma:");
294 
295  _settings.setTimeout(ZConfig::instance().download_transfer_timeout());
297 
299 
300  // fill some settings from url query parameters
301  try
302  {
304  }
305  catch ( const MediaException &e )
306  {
307  disconnectFrom();
308  ZYPP_RETHROW(e);
309  }
310  // if the proxy was not set (or explicitly unset) by url, then look...
311  if ( _settings.proxy().empty() )
312  {
313  // ...at the system proxy settings
315  }
316 
319  {
320  switch ( env::ZYPP_MEDIA_CURL_IPRESOLVE() )
321  {
322  case 4: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); break;
323  case 6: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); break;
324  }
325  }
326 
330  SET_OPTION(CURLOPT_CONNECTTIMEOUT, _settings.connectTimeout());
331  // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
332  // just in case curl does not trigger its progress callback frequently
333  // enough.
334  if ( _settings.timeout() )
335  {
336  SET_OPTION(CURLOPT_TIMEOUT, 3600L);
337  }
338 
339  // follow any Location: header that the server sends as part of
340  // an HTTP header (#113275)
341  SET_OPTION(CURLOPT_FOLLOWLOCATION, 1L);
342  // 3 redirects seem to be too few in some cases (bnc #465532)
343  SET_OPTION(CURLOPT_MAXREDIRS, 6L);
344 
345  if ( _url.getScheme() == "https" )
346  {
347 #if CURLVERSION_AT_LEAST(7,19,4)
348  // restrict following of redirections from https to https only
349  SET_OPTION( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
350 #endif
351 
354  {
356  }
357 
359  {
360  SET_OPTION(CURLOPT_SSLCERT, _settings.clientCertificatePath().c_str());
361  }
362  if( ! _settings.clientKeyPath().empty() )
363  {
364  SET_OPTION(CURLOPT_SSLKEY, _settings.clientKeyPath().c_str());
365  }
366 
367 #ifdef CURLSSLOPT_ALLOW_BEAST
368  // see bnc#779177
369  ret = curl_easy_setopt( _curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
370  if ( ret != 0 ) {
371  disconnectFrom();
373  }
374 #endif
375  SET_OPTION(CURLOPT_SSL_VERIFYPEER, _settings.verifyPeerEnabled() ? 1L : 0L);
376  SET_OPTION(CURLOPT_SSL_VERIFYHOST, _settings.verifyHostEnabled() ? 2L : 0L);
377  // bnc#903405 - POODLE: libzypp should only talk TLS
378  SET_OPTION(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
379  }
380 
381  SET_OPTION(CURLOPT_USERAGENT, _settings.userAgentString().c_str() );
382 
383  /* Fixes bsc#1174011 "auth=basic ignored in some cases"
384  * We should proactively add the password to the request if basic auth is configured
385  * and a password is available in the credentials but not in the URL.
386  *
387  * We will be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
388  * and ask the server first about the auth method
389  */
390  if ( _settings.authType() == "basic"
391  && _settings.username().size()
392  && !_settings.password().size() ) {
393 
394  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
395  const auto cred = cm.getCred( _url );
396  if ( cred && cred->valid() ) {
397  if ( !_settings.username().size() )
398  _settings.setUsername(cred->username());
399  _settings.setPassword(cred->password());
400  }
401  }
402 
403  /*---------------------------------------------------------------*
404  CURLOPT_USERPWD: [user name]:[password]
405 
406  Url::username/password -> CURLOPT_USERPWD
407  If not provided, anonymous FTP identification
408  *---------------------------------------------------------------*/
409 
410  if ( _settings.userPassword().size() )
411  {
412  SET_OPTION(CURLOPT_USERPWD, _settings.userPassword().c_str());
413  std::string use_auth = _settings.authType();
414  if (use_auth.empty())
415  use_auth = "digest,basic"; // our default
416  long auth = CurlAuthData::auth_type_str2long(use_auth);
417  if( auth != CURLAUTH_NONE)
418  {
419  DBG << "Enabling HTTP authentication methods: " << use_auth
420  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
421  SET_OPTION(CURLOPT_HTTPAUTH, auth);
422  }
423  }
424 
425  if ( _settings.proxyEnabled() && ! _settings.proxy().empty() )
426  {
427  DBG << "Proxy: '" << _settings.proxy() << "'" << endl;
428  SET_OPTION(CURLOPT_PROXY, _settings.proxy().c_str());
429  SET_OPTION(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
430  /*---------------------------------------------------------------*
431  * CURLOPT_PROXYUSERPWD: [user name]:[password]
432  *
433  * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
434  * If not provided, $HOME/.curlrc is evaluated
435  *---------------------------------------------------------------*/
436 
437  std::string proxyuserpwd = _settings.proxyUserPassword();
438 
439  if ( proxyuserpwd.empty() )
440  {
441  CurlConfig curlconf;
442  CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
443  if ( curlconf.proxyuserpwd.empty() )
444  DBG << "Proxy: ~/.curlrc does not contain the proxy-user option" << endl;
445  else
446  {
447  proxyuserpwd = curlconf.proxyuserpwd;
448  DBG << "Proxy: using proxy-user from ~/.curlrc" << endl;
449  }
450  }
451  else
452  {
453  DBG << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << endl;
454  }
455 
456  if ( ! proxyuserpwd.empty() )
457  {
458  SET_OPTION(CURLOPT_PROXYUSERPWD, curlUnEscape( proxyuserpwd ).c_str());
459  }
460  }
461 #if CURLVERSION_AT_LEAST(7,19,4)
462  else if ( _settings.proxy() == EXPLICITLY_NO_PROXY )
463  {
464  // Explicitly disabled in URL (see fillSettingsFromUrl()).
465  // This should also prevent libcurl from looking into the environment.
466  DBG << "Proxy: explicitly NOPROXY" << endl;
467  SET_OPTION(CURLOPT_NOPROXY, "*");
468  }
469 #endif
470  else
471  {
472  DBG << "Proxy: not explicitly set" << endl;
473  DBG << "Proxy: libcurl may look into the environment" << endl;
474  }
475 
477  if ( _settings.minDownloadSpeed() != 0 )
478  {
479  SET_OPTION(CURLOPT_LOW_SPEED_LIMIT, _settings.minDownloadSpeed());
480  // default to 10 seconds at low speed
481  SET_OPTION(CURLOPT_LOW_SPEED_TIME, 60L);
482  }
483 
484 #if CURLVERSION_AT_LEAST(7,15,5)
485  if ( _settings.maxDownloadSpeed() != 0 )
486  SET_OPTION_OFFT(CURLOPT_MAX_RECV_SPEED_LARGE, _settings.maxDownloadSpeed());
487 #endif
488 
489  /*---------------------------------------------------------------*
490  *---------------------------------------------------------------*/
491 
494  if ( str::strToBool( _url.getQueryParam( "cookies" ), true ) )
495  SET_OPTION(CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
496  else
497  MIL << "No cookies requested" << endl;
498  SET_OPTION(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
499  SET_OPTION(CURLOPT_PROGRESSFUNCTION, &progressCallback );
500  SET_OPTION(CURLOPT_NOPROGRESS, 0L);
501 
502 #if CURLVERSION_AT_LEAST(7,18,0)
503  // bnc #306272
504  SET_OPTION(CURLOPT_PROXY_TRANSFER_MODE, 1L );
505 #endif
506  // append settings custom headers to curl
507  for ( TransferSettings::Headers::const_iterator it = vol_settings.headersBegin();
508  it != vol_settings.headersEnd();
509  ++it )
510  {
511  // MIL << "HEADER " << *it << std::endl;
512 
513  _customHeaders = curl_slist_append(_customHeaders, it->c_str());
514  if ( !_customHeaders )
516  }
517 
518  SET_OPTION(CURLOPT_HTTPHEADER, _customHeaders);
519 }
520 
522 
523 
524 void MediaCurl::attachTo (bool next)
525 {
526  if ( next )
528 
529  if ( !_url.isValid() )
531 
534  {
536  }
537 
538  disconnectFrom(); // clean _curl if needed
539  _curl = curl_easy_init();
540  if ( !_curl ) {
542  }
543  try
544  {
545  setupEasy();
546  }
547  catch (Exception & ex)
548  {
549  disconnectFrom();
550  ZYPP_RETHROW(ex);
551  }
552 
553  // FIXME: need a derived class to propelly compare url's
555  setMediaSource(media);
556 }
557 
558 bool
560 {
561  return MediaHandler::checkAttachPoint( apoint, true, true);
562 }
563 
565 
567 {
568  if ( _customHeaders )
569  {
570  curl_slist_free_all(_customHeaders);
571  _customHeaders = 0L;
572  }
573 
574  if ( _curl )
575  {
576  curl_easy_cleanup( _curl );
577  _curl = NULL;
578  }
579 }
580 
582 
583 void MediaCurl::releaseFrom( const std::string & ejectDev )
584 {
585  disconnect();
586 }
587 
588 Url MediaCurl::getFileUrl( const Pathname & filename_r ) const
589 {
590  // Simply extend the URLs pathname. An 'absolute' URL path
591  // is achieved by encoding the leading '/' in an URL path:
592  // URL: ftp://user@server -> ~user
593  // URL: ftp://user@server/ -> ~user
594  // URL: ftp://user@server// -> ~user
595  // URL: ftp://user@server/%2F -> /
596  // ^- this '/' is just a separator
597  Url newurl( _url );
598  newurl.setPathName( ( Pathname("./"+_url.getPathName()) / filename_r ).asString().substr(1) );
599  return newurl;
600 }
601 
603 
604 void MediaCurl::getFile(const Pathname & filename , const ByteCount &expectedFileSize_r) const
605 {
606  // Use absolute file name to prevent access of files outside of the
607  // hierarchy below the attach point.
608  getFileCopy(filename, localPath(filename).absolutename(), expectedFileSize_r);
609 }
610 
612 
613 void MediaCurl::getFileCopy( const Pathname & filename , const Pathname & target, const ByteCount &expectedFileSize_r ) const
614 {
616 
617  Url fileurl(getFileUrl(filename));
618 
619  bool retry = false;
620 
621  do
622  {
623  try
624  {
625  doGetFileCopy(filename, target, report, expectedFileSize_r);
626  retry = false;
627  }
628  // retry with proper authentication data
629  catch (MediaUnauthorizedException & ex_r)
630  {
631  if(authenticate(ex_r.hint(), !retry))
632  retry = true;
633  else
634  {
636  ZYPP_RETHROW(ex_r);
637  }
638  }
639  // unexpected exception
640  catch (MediaException & excpt_r)
641  {
643  if( typeid(excpt_r) == typeid( media::MediaFileNotFoundException ) ||
644  typeid(excpt_r) == typeid( media::MediaNotAFileException ) )
645  {
647  }
648  report->finish(fileurl, reason, excpt_r.asUserHistory());
649  ZYPP_RETHROW(excpt_r);
650  }
651  }
652  while (retry);
653 
655 }
656 
658 
659 bool MediaCurl::getDoesFileExist( const Pathname & filename ) const
660 {
661  bool retry = false;
662 
663  do
664  {
665  try
666  {
667  return doGetDoesFileExist( filename );
668  }
669  // authentication problem, retry with proper authentication data
670  catch (MediaUnauthorizedException & ex_r)
671  {
672  if(authenticate(ex_r.hint(), !retry))
673  retry = true;
674  else
675  ZYPP_RETHROW(ex_r);
676  }
677  // unexpected exception
678  catch (MediaException & excpt_r)
679  {
680  ZYPP_RETHROW(excpt_r);
681  }
682  }
683  while (retry);
684 
685  return false;
686 }
687 
689 
691  CURLcode code,
692  bool timeout_reached) const
693 {
694  if ( code != 0 )
695  {
696  Url url;
697  if (filename.empty())
698  url = _url;
699  else
700  url = getFileUrl(filename);
701 
702  std::string err;
703  {
704  switch ( code )
705  {
706  case CURLE_UNSUPPORTED_PROTOCOL:
707  err = " Unsupported protocol";
708  if ( !_lastRedirect.empty() )
709  {
710  err += " or redirect (";
711  err += _lastRedirect;
712  err += ")";
713  }
714  break;
715  case CURLE_URL_MALFORMAT:
716  case CURLE_URL_MALFORMAT_USER:
717  err = " Bad URL";
718  break;
719  case CURLE_LOGIN_DENIED:
720  ZYPP_THROW(
721  MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
722  break;
723  case CURLE_HTTP_RETURNED_ERROR:
724  {
725  long httpReturnCode = 0;
726  CURLcode infoRet = curl_easy_getinfo( _curl,
727  CURLINFO_RESPONSE_CODE,
728  &httpReturnCode );
729  if ( infoRet == CURLE_OK )
730  {
731  std::string msg = "HTTP response: " + str::numstring( httpReturnCode );
732  switch ( httpReturnCode )
733  {
734  case 401:
735  {
736  std::string auth_hint = getAuthHint();
737 
738  DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
739  DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
740 
742  url, "Login failed.", _curlError, auth_hint
743  ));
744  }
745 
746  case 502: // bad gateway (bnc #1070851)
747  case 503: // service temporarily unavailable (bnc #462545)
749  case 504: // gateway timeout
751  case 403:
752  {
753  std::string msg403;
754  if ( url.getHost().find(".suse.com") != std::string::npos )
755  msg403 = _("Visit the SUSE Customer Center to check whether your registration is valid and has not expired.");
756  else if (url.asString().find("novell.com") != std::string::npos)
757  msg403 = _("Visit the Novell Customer Center to check whether your registration is valid and has not expired.");
759  }
760  case 404:
761  case 410:
763  }
764 
765  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
767  }
768  else
769  {
770  std::string msg = "Unable to retrieve HTTP response:";
771  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
773  }
774  }
775  break;
776  case CURLE_FTP_COULDNT_RETR_FILE:
777 #if CURLVERSION_AT_LEAST(7,16,0)
778  case CURLE_REMOTE_FILE_NOT_FOUND:
779 #endif
780  case CURLE_FTP_ACCESS_DENIED:
781  case CURLE_TFTP_NOTFOUND:
782  err = "File not found";
784  break;
785  case CURLE_BAD_PASSWORD_ENTERED:
786  case CURLE_FTP_USER_PASSWORD_INCORRECT:
787  err = "Login failed";
788  break;
789  case CURLE_COULDNT_RESOLVE_PROXY:
790  case CURLE_COULDNT_RESOLVE_HOST:
791  case CURLE_COULDNT_CONNECT:
792  case CURLE_FTP_CANT_GET_HOST:
793  err = "Connection failed";
794  break;
795  case CURLE_WRITE_ERROR:
796  err = "Write error";
797  break;
798  case CURLE_PARTIAL_FILE:
799  case CURLE_OPERATION_TIMEDOUT:
800  timeout_reached = true; // fall though to TimeoutException
801  // fall though...
802  case CURLE_ABORTED_BY_CALLBACK:
803  if( timeout_reached )
804  {
805  err = "Timeout reached";
807  }
808  else
809  {
810  err = "User abort";
811  }
812  break;
813  case CURLE_SSL_PEER_CERTIFICATE:
814  default:
815  err = "Curl error " + str::numstring( code );
816  break;
817  }
818 
819  // uhm, no 0 code but unknown curl exception
821  }
822  }
823  else
824  {
825  // actually the code is 0, nothing happened
826  }
827 }
828 
830 
831 bool MediaCurl::doGetDoesFileExist( const Pathname & filename ) const
832 {
833  DBG << filename.asString() << endl;
834 
835  if(!_url.isValid())
837 
838  if(_url.getHost().empty())
840 
841  Url url(getFileUrl(filename));
842 
843  DBG << "URL: " << url.asString() << endl;
844  // Use URL without options and without username and passwd
845  // (some proxies dislike them in the URL).
846  // Curl seems to need the just scheme, hostname and a path;
847  // the rest was already passed as curl options (in attachTo).
848  Url curlUrl( clearQueryString(url) );
849 
850  //
851  // See also Bug #154197 and ftp url definition in RFC 1738:
852  // The url "ftp://user@host/foo/bar/file" contains a path,
853  // that is relative to the user's home.
854  // The url "ftp://user@host//foo/bar/file" (or also with
855  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
856  // contains an absolute path.
857  //
858  _lastRedirect.clear();
859  std::string urlBuffer( curlUrl.asString());
860  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
861  urlBuffer.c_str() );
862  if ( ret != 0 ) {
864  }
865 
866  // instead of returning no data with NOBODY, we return
867  // little data, that works with broken servers, and
868  // works for ftp as well, because retrieving only headers
869  // ftp will return always OK code ?
870  // See http://curl.haxx.se/docs/knownbugs.html #58
871  if ( (_url.getScheme() == "http" || _url.getScheme() == "https") &&
873  ret = curl_easy_setopt( _curl, CURLOPT_NOBODY, 1L );
874  else
875  ret = curl_easy_setopt( _curl, CURLOPT_RANGE, "0-1" );
876 
877  if ( ret != 0 ) {
878  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
879  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
880  /* yes, this is why we never got to get NOBODY working before,
881  because setting it changes this option too, and we also
882  need to reset it
883  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
884  */
885  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
887  }
888 
889  AutoFILE file { ::fopen( "/dev/null", "w" ) };
890  if ( !file ) {
891  ERR << "fopen failed for /dev/null" << endl;
892  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
893  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
894  /* yes, this is why we never got to get NOBODY working before,
895  because setting it changes this option too, and we also
896  need to reset it
897  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
898  */
899  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
900  if ( ret != 0 ) {
902  }
903  ZYPP_THROW(MediaWriteException("/dev/null"));
904  }
905 
906  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, (*file) );
907  if ( ret != 0 ) {
908  std::string err( _curlError);
909  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
910  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
911  /* yes, this is why we never got to get NOBODY working before,
912  because setting it changes this option too, and we also
913  need to reset it
914  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
915  */
916  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
917  if ( ret != 0 ) {
919  }
921  }
922 
923  CURLcode ok = curl_easy_perform( _curl );
924  MIL << "perform code: " << ok << " [ " << curl_easy_strerror(ok) << " ]" << endl;
925 
926  // reset curl settings
927  if ( _url.getScheme() == "http" || _url.getScheme() == "https" )
928  {
929  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
930  if ( ret != 0 ) {
932  }
933 
934  /* yes, this is why we never got to get NOBODY working before,
935  because setting it changes this option too, and we also
936  need to reset it
937  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
938  */
939  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L);
940  if ( ret != 0 ) {
942  }
943 
944  }
945  else
946  {
947  // for FTP we set different options
948  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL);
949  if ( ret != 0 ) {
951  }
952  }
953 
954  // as we are not having user interaction, the user can't cancel
955  // the file existence checking, a callback or timeout return code
956  // will be always a timeout.
957  try {
958  evaluateCurlCode( filename, ok, true /* timeout */);
959  }
960  catch ( const MediaFileNotFoundException &e ) {
961  // if the file did not exist then we can return false
962  return false;
963  }
964  catch ( const MediaException &e ) {
965  // some error, we are not sure about file existence, rethrw
966  ZYPP_RETHROW(e);
967  }
968  // exists
969  return ( ok == CURLE_OK );
970 }
971 
973 
974 
975 #if DETECT_DIR_INDEX
976 bool MediaCurl::detectDirIndex() const
977 {
978  if(_url.getScheme() != "http" && _url.getScheme() != "https")
979  return false;
980  //
981  // try to check the effective url and set the not_a_file flag
982  // if the url path ends with a "/", what usually means, that
983  // we've received a directory index (index.html content).
984  //
985  // Note: This may be dangerous and break file retrieving in
986  // case of some server redirections ... ?
987  //
988  bool not_a_file = false;
989  char *ptr = NULL;
990  CURLcode ret = curl_easy_getinfo( _curl,
991  CURLINFO_EFFECTIVE_URL,
992  &ptr);
993  if ( ret == CURLE_OK && ptr != NULL)
994  {
995  try
996  {
997  Url eurl( ptr);
998  std::string path( eurl.getPathName());
999  if( !path.empty() && path != "/" && *path.rbegin() == '/')
1000  {
1001  DBG << "Effective url ("
1002  << eurl
1003  << ") seems to provide the index of a directory"
1004  << endl;
1005  not_a_file = true;
1006  }
1007  }
1008  catch( ... )
1009  {}
1010  }
1011  return not_a_file;
1012 }
1013 #endif
1014 
1016 
1017 void MediaCurl::doGetFileCopy(const Pathname & filename , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, const ByteCount &expectedFileSize_r, RequestOptions options ) const
1018 {
1019  Pathname dest = target.absolutename();
1020  if( assert_dir( dest.dirname() ) )
1021  {
1022  DBG << "assert_dir " << dest.dirname() << " failed" << endl;
1023  ZYPP_THROW( MediaSystemException(getFileUrl(filename), "System error on " + dest.dirname().asString()) );
1024  }
1025 
1026  ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) };
1027  AutoFILE file;
1028  {
1029  AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
1030  if( ! buf )
1031  {
1032  ERR << "out of memory for temp file name" << endl;
1033  ZYPP_THROW(MediaSystemException(getFileUrl(filename), "out of memory for temp file name"));
1034  }
1035 
1036  AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1037  if( tmp_fd == -1 )
1038  {
1039  ERR << "mkstemp failed for file '" << destNew << "'" << endl;
1040  ZYPP_THROW(MediaWriteException(destNew));
1041  }
1042  destNew = ManagedFile( (*buf), filesystem::unlink );
1043 
1044  file = ::fdopen( tmp_fd, "we" );
1045  if ( ! file )
1046  {
1047  ERR << "fopen failed for file '" << destNew << "'" << endl;
1048  ZYPP_THROW(MediaWriteException(destNew));
1049  }
1050  tmp_fd.resetDispose(); // don't close it here! ::fdopen moved ownership to file
1051  }
1052 
1053  DBG << "dest: " << dest << endl;
1054  DBG << "temp: " << destNew << endl;
1055 
1056  // set IFMODSINCE time condition (no download if not modified)
1057  if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
1058  {
1059  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1060  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
1061  }
1062  else
1063  {
1064  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1065  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1066  }
1067  try
1068  {
1069  doGetFileCopyFile(filename, dest, file, report, expectedFileSize_r, options);
1070  }
1071  catch (Exception &e)
1072  {
1073  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1074  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1075  ZYPP_RETHROW(e);
1076  }
1077 
1078  long httpReturnCode = 0;
1079  CURLcode infoRet = curl_easy_getinfo(_curl,
1080  CURLINFO_RESPONSE_CODE,
1081  &httpReturnCode);
1082  bool modified = true;
1083  if (infoRet == CURLE_OK)
1084  {
1085  DBG << "HTTP response: " + str::numstring(httpReturnCode);
1086  if ( httpReturnCode == 304
1087  || ( httpReturnCode == 213 && (_url.getScheme() == "ftp" || _url.getScheme() == "tftp") ) ) // not modified
1088  {
1089  DBG << " Not modified.";
1090  modified = false;
1091  }
1092  DBG << endl;
1093  }
1094  else
1095  {
1096  WAR << "Could not get the reponse code." << endl;
1097  }
1098 
1099  if (modified || infoRet != CURLE_OK)
1100  {
1101  // apply umask
1102  if ( ::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 ) ) )
1103  {
1104  ERR << "Failed to chmod file " << destNew << endl;
1105  }
1106 
1107  file.resetDispose(); // we're going to close it manually here
1108  if ( ::fclose( file ) )
1109  {
1110  ERR << "Fclose failed for file '" << destNew << "'" << endl;
1111  ZYPP_THROW(MediaWriteException(destNew));
1112  }
1113 
1114  // move the temp file into dest
1115  if ( rename( destNew, dest ) != 0 ) {
1116  ERR << "Rename failed" << endl;
1118  }
1119  destNew.resetDispose(); // no more need to unlink it
1120  }
1121 
1122  DBG << "done: " << PathInfo(dest) << endl;
1123 }
1124 
1126 
1127 void MediaCurl::doGetFileCopyFile(const Pathname & filename , const Pathname & dest, FILE *file, callback::SendReport<DownloadProgressReport> & report, const ByteCount &expectedFileSize_r, RequestOptions options ) const
1128 {
1129  DBG << filename.asString() << endl;
1130 
1131  if(!_url.isValid())
1133 
1134  if(_url.getHost().empty())
1136 
1137  Url url(getFileUrl(filename));
1138 
1139  DBG << "URL: " << url.asString() << endl;
1140  // Use URL without options and without username and passwd
1141  // (some proxies dislike them in the URL).
1142  // Curl seems to need the just scheme, hostname and a path;
1143  // the rest was already passed as curl options (in attachTo).
1144  Url curlUrl( clearQueryString(url) );
1145 
1146  //
1147  // See also Bug #154197 and ftp url definition in RFC 1738:
1148  // The url "ftp://user@host/foo/bar/file" contains a path,
1149  // that is relative to the user's home.
1150  // The url "ftp://user@host//foo/bar/file" (or also with
1151  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1152  // contains an absolute path.
1153  //
1154  _lastRedirect.clear();
1155  std::string urlBuffer( curlUrl.asString());
1156  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1157  urlBuffer.c_str() );
1158  if ( ret != 0 ) {
1160  }
1161 
1162  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
1163  if ( ret != 0 ) {
1165  }
1166 
1167  // Set callback and perform.
1168  ProgressData progressData(_curl, _settings.timeout(), url, expectedFileSize_r, &report);
1169  if (!(options & OPTION_NO_REPORT_START))
1170  report->start(url, dest);
1171  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
1172  WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1173  }
1174 
1175  ret = curl_easy_perform( _curl );
1176 #if CURLVERSION_AT_LEAST(7,19,4)
1177  // bnc#692260: If the client sends a request with an If-Modified-Since header
1178  // with a future date for the server, the server may respond 200 sending a
1179  // zero size file.
1180  // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
1181  if ( ftell(file) == 0 && ret == 0 )
1182  {
1183  long httpReturnCode = 33;
1184  if ( curl_easy_getinfo( _curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
1185  {
1186  long conditionUnmet = 33;
1187  if ( curl_easy_getinfo( _curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
1188  {
1189  WAR << "TIMECONDITION unmet - retry without." << endl;
1190  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1191  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1192  ret = curl_easy_perform( _curl );
1193  }
1194  }
1195  }
1196 #endif
1197 
1198  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
1199  WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1200  }
1201 
1202  if ( ret != 0 )
1203  {
1204  ERR << "curl error: " << ret << ": " << _curlError
1205  << ", temp file size " << ftell(file)
1206  << " bytes." << endl;
1207 
1208  // the timeout is determined by the progress data object
1209  // which holds whether the timeout was reached or not,
1210  // otherwise it would be a user cancel
1211  try {
1212 
1213  if ( progressData.fileSizeExceeded )
1214  ZYPP_THROW(MediaFileSizeExceededException(url, progressData._expectedFileSize));
1215 
1216  evaluateCurlCode( filename, ret, progressData.reached );
1217  }
1218  catch ( const MediaException &e ) {
1219  // some error, we are not sure about file existence, rethrw
1220  ZYPP_RETHROW(e);
1221  }
1222  }
1223 
1224 #if DETECT_DIR_INDEX
1225  if (!ret && detectDirIndex())
1226  {
1228  }
1229 #endif // DETECT_DIR_INDEX
1230 }
1231 
1233 
1234 void MediaCurl::getDir( const Pathname & dirname, bool recurse_r ) const
1235 {
1236  filesystem::DirContent content;
1237  getDirInfo( content, dirname, /*dots*/false );
1238 
1239  for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
1240  Pathname filename = dirname + it->name;
1241  int res = 0;
1242 
1243  switch ( it->type ) {
1244  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
1245  case filesystem::FT_FILE:
1246  getFile( filename, 0 );
1247  break;
1248  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
1249  if ( recurse_r ) {
1250  getDir( filename, recurse_r );
1251  } else {
1252  res = assert_dir( localPath( filename ) );
1253  if ( res ) {
1254  WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
1255  }
1256  }
1257  break;
1258  default:
1259  // don't provide devices, sockets, etc.
1260  break;
1261  }
1262  }
1263 }
1264 
1266 
1267 void MediaCurl::getDirInfo( std::list<std::string> & retlist,
1268  const Pathname & dirname, bool dots ) const
1269 {
1270  getDirectoryYast( retlist, dirname, dots );
1271 }
1272 
1274 
1276  const Pathname & dirname, bool dots ) const
1277 {
1278  getDirectoryYast( retlist, dirname, dots );
1279 }
1280 
1282 //
1283 int MediaCurl::aliveCallback( void *clientp, double /*dltotal*/, double dlnow, double /*ultotal*/, double /*ulnow*/ )
1284 {
1285  ProgressData *pdata = reinterpret_cast<ProgressData *>( clientp );
1286  if( pdata )
1287  {
1288  // Do not propagate dltotal in alive callbacks. MultiCurl uses this to
1289  // prevent a percentage raise while downloading a metalink file. Download
1290  // activity however is indicated by propagating the download rate (via dlnow).
1291  pdata->updateStats( 0.0, dlnow );
1292  return pdata->reportProgress();
1293  }
1294  return 0;
1295 }
1296 
1297 int MediaCurl::progressCallback( void *clientp, double dltotal, double dlnow, double ultotal, double ulnow )
1298 {
1299  ProgressData *pdata = reinterpret_cast<ProgressData *>( clientp );
1300  if( pdata )
1301  {
1302  // work around curl bug that gives us old data
1303  long httpReturnCode = 0;
1304  if ( curl_easy_getinfo( pdata->curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0 )
1305  return aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
1306 
1307  pdata->updateStats( dltotal, dlnow );
1308  return pdata->reportProgress();
1309  }
1310  return 0;
1311 }
1312 
1314 {
1315  ProgressData *pdata = reinterpret_cast<ProgressData *>(clientp);
1316  return pdata ? pdata->curl : 0;
1317 }
1318 
1320 
1321 std::string MediaCurl::getAuthHint() const
1322 {
1323  long auth_info = CURLAUTH_NONE;
1324 
1325  CURLcode infoRet =
1326  curl_easy_getinfo(_curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1327 
1328  if(infoRet == CURLE_OK)
1329  {
1330  return CurlAuthData::auth_type_long2str(auth_info);
1331  }
1332 
1333  return "";
1334 }
1335 
1340 void MediaCurl::resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
1341 {
1342  ProgressData *data = reinterpret_cast<ProgressData *>(clientp);
1343  if ( data ) {
1344  data->_expectedFileSize = expectedFileSize;
1345  }
1346 }
1347 
1349 
1350 bool MediaCurl::authenticate(const std::string & availAuthTypes, bool firstTry) const
1351 {
1353  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
1354  CurlAuthData_Ptr credentials;
1355 
1356  // get stored credentials
1357  AuthData_Ptr cmcred = cm.getCred(_url);
1358 
1359  if (cmcred && firstTry)
1360  {
1361  credentials.reset(new CurlAuthData(*cmcred));
1362  DBG << "got stored credentials:" << endl << *credentials << endl;
1363  }
1364  // if not found, ask user
1365  else
1366  {
1367 
1368  CurlAuthData_Ptr curlcred;
1369  curlcred.reset(new CurlAuthData());
1371 
1372  // preset the username if present in current url
1373  if (!_url.getUsername().empty() && firstTry)
1374  curlcred->setUsername(_url.getUsername());
1375  // if CM has found some credentials, preset the username from there
1376  else if (cmcred)
1377  curlcred->setUsername(cmcred->username());
1378 
1379  // indicate we have no good credentials from CM
1380  cmcred.reset();
1381 
1382  std::string prompt_msg = str::Format(_("Authentication required for '%s'")) % _url.asString();
1383 
1384  // set available authentication types from the exception
1385  // might be needed in prompt
1386  curlcred->setAuthType(availAuthTypes);
1387 
1388  // ask user
1389  if (auth_report->prompt(_url, prompt_msg, *curlcred))
1390  {
1391  DBG << "callback answer: retry" << endl
1392  << "CurlAuthData: " << *curlcred << endl;
1393 
1394  if (curlcred->valid())
1395  {
1396  credentials = curlcred;
1397  // if (credentials->username() != _url.getUsername())
1398  // _url.setUsername(credentials->username());
1406  }
1407  }
1408  else
1409  {
1410  DBG << "callback answer: cancel" << endl;
1411  }
1412  }
1413 
1414  // set username and password
1415  if (credentials)
1416  {
1417  // HACK, why is this const?
1418  const_cast<MediaCurl*>(this)->_settings.setUsername(credentials->username());
1419  const_cast<MediaCurl*>(this)->_settings.setPassword(credentials->password());
1420 
1421  // set username and password
1422  CURLcode ret = curl_easy_setopt(_curl, CURLOPT_USERPWD, _settings.userPassword().c_str());
1424 
1425  // set available authentication types from the exception
1426  if (credentials->authType() == CURLAUTH_NONE)
1427  credentials->setAuthType(availAuthTypes);
1428 
1429  // set auth type (seems this must be set _after_ setting the userpwd)
1430  if (credentials->authType() != CURLAUTH_NONE)
1431  {
1432  // FIXME: only overwrite if not empty?
1433  const_cast<MediaCurl*>(this)->_settings.setAuthType(credentials->authTypeAsString());
1434  ret = curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, credentials->authType());
1436  }
1437 
1438  if (!cmcred)
1439  {
1440  credentials->setUrl(_url);
1441  cm.addCred(*credentials);
1442  cm.save();
1443  }
1444 
1445  return true;
1446  }
1447 
1448  return false;
1449 }
1450 
1451 //need a out of line definiton, otherwise vtable is emitted for every translation unit
1453 
1454 
1455  } // namespace media
1456 } // namespace zypp
1457 //
#define CONNECT_TIMEOUT
Definition: CurlHelper.h:21
#define EXPLICITLY_NO_PROXY
Definition: CurlHelper.h:25
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:400
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:392
Interface to gettext.
#define _(MSG)
Definition: Gettext.h:37
#define DBG
Definition: Logger.h:90
#define MIL
Definition: Logger.h:91
#define ERR
Definition: Logger.h:93
#define WAR
Definition: Logger.h:92
double _dnlTotal
Bytes to download or 0 if unknown.
Definition: MediaCurl.cc:78
double uload
Definition: MediaCurl.cc:161
time_t _timeLast
Start last period(~1sec)
Definition: MediaCurl.cc:74
bool reached
Definition: MediaCurl.cc:68
bool fileSizeExceeded
Definition: MediaCurl.cc:69
ByteCount _expectedFileSize
Definition: MediaCurl.cc:71
double _drateTotal
Download rate so far.
Definition: MediaCurl.cc:84
double dload
Definition: MediaCurl.cc:159
#define SET_OPTION_OFFT(opt, val)
Definition: MediaCurl.cc:177
double dload_period
Definition: MediaCurl.cc:151
callback::SendReport< DownloadProgressReport > * report
Definition: MediaCurl.cc:70
long secs
Definition: MediaCurl.cc:153
int _dnlPercent
Percent completed or 0 if _dnlTotal is unknown.
Definition: MediaCurl.cc:82
CURL * curl
Definition: MediaCurl.cc:65
Url url
Definition: MediaCurl.cc:66
double _drateLast
Download rate in last period.
Definition: MediaCurl.cc:85
time_t ltime
Definition: MediaCurl.cc:157
time_t _timeNow
Now.
Definition: MediaCurl.cc:76
#define SET_OPTION(opt, val)
Definition: MediaCurl.cc:170
double drate_period
Definition: MediaCurl.cc:149
double drate_avg
Definition: MediaCurl.cc:155
time_t _timeStart
Start total stats.
Definition: MediaCurl.cc:73
time_t _timeRcv
Start of no-data timeout.
Definition: MediaCurl.cc:75
double _dnlNow
Bytes downloaded now.
Definition: MediaCurl.cc:80
double _dnlLast
Bytes downloaded at period start.
Definition: MediaCurl.cc:79
time_t timeout
Definition: MediaCurl.cc:67
Convenience interface for handling authentication data of media user.
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
Definition: AutoDispose.h:93
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:162
Store and operate with byte count.
Definition: ByteCount.h:31
Base class for Exception.
Definition: Exception.h:146
std::string asUserHistory() const
A single (multiline) string composed of asUserString and historyAsString.
Definition: Exception.cc:91
Url manipulation class.
Definition: Url.h:92
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:528
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:492
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:567
std::string getPathName(EEncoding eflag=zypp::url::E_DECODED) const
Returns the path name from the URL.
Definition: Url.cc:599
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:655
void setPathName(const std::string &path, EEncoding eflag=zypp::url::E_DECODED)
Set the path name.
Definition: Url.cc:759
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:583
bool isValid() const
Verifies the Url.
Definition: Url.cc:484
static ZConfig & instance()
Singleton ctor.
Definition: Resolver.cc:126
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:221
bool userMayRWX() const
Definition: PathInfo.h:353
const Pathname & path() const
Return current Pathname.
Definition: PathInfo.h:246
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:170
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:124
const std::string & asString() const
String representation.
Definition: Pathname.h:91
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
Pathname absolutename() const
Return this path, adding a leading '/' if relative.
Definition: Pathname.h:139
const char * c_str() const
String representation.
Definition: Pathname.h:110
void save()
Saves any unsaved credentials added via addUserCred() or addGlobalCred() methods.
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
void addCred(const AuthData &cred)
Add new credentials with user callbacks.
Curl HTTP authentication data.
Definition: MediaUserAuth.h:74
static std::string auth_type_long2str(long auth_type)
Converts a long of ORed CURLAUTH_* identifiers into a string of comma separated list of authenticatio...
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl.h:33
virtual void setupEasy()
initializes the curl easy handle with the data from the url
Definition: MediaCurl.cc:260
Url getFileUrl(const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
Definition: MediaCurl.cc:588
static void setCookieFile(const Pathname &)
Definition: MediaCurl.cc:228
virtual bool getDoesFileExist(const Pathname &filename) const override
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly,...
Definition: MediaCurl.cc:659
@ OPTION_NO_IFMODSINCE
to not add a IFMODSINCE header if target exists
Definition: MediaCurl.h:44
@ OPTION_NO_REPORT_START
do not send a start ProgressReport
Definition: MediaCurl.h:46
static void resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
MediaMultiCurl needs to reset the expected filesize in case a metalink file is downloaded otherwise t...
Definition: MediaCurl.cc:1340
std::string _currentCookieFile
Definition: MediaCurl.h:170
static int aliveCallback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
Definition: MediaCurl.cc:1283
static Pathname _cookieFile
Definition: MediaCurl.h:171
static int progressCallback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
Callback reporting download progress.
Definition: MediaCurl.cc:1297
std::string _lastRedirect
to log/report redirections
Definition: MediaCurl.h:173
Url clearQueryString(const Url &url) const
Definition: MediaCurl.cc:217
virtual void attachTo(bool next=false) override
Call concrete handler to attach the media.
Definition: MediaCurl.cc:524
virtual void getDirInfo(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const override
Call concrete handler to provide a content list of directory on media via retlist.
Definition: MediaCurl.cc:1267
char _curlError[CURL_ERROR_SIZE]
Definition: MediaCurl.h:177
void checkProtocol(const Url &url) const
check the url is supported by the curl library
Definition: MediaCurl.cc:235
MediaCurl(const Url &url_r, const Pathname &attach_point_hint_r)
Definition: MediaCurl.cc:181
virtual void getFileCopy(const Pathname &srcFilename, const Pathname &targetFilename, const ByteCount &expectedFileSize_r) const override
Definition: MediaCurl.cc:613
bool detectDirIndex() const
void evaluateCurlCode(const zypp::Pathname &filename, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition: MediaCurl.cc:690
TransferSettings _settings
Definition: MediaCurl.h:179
virtual void doGetFileCopy(const Pathname &srcFilename, const Pathname &targetFilename, callback::SendReport< DownloadProgressReport > &_report, const ByteCount &expectedFileSize_r, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1017
virtual bool checkAttachPoint(const Pathname &apoint) const override
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
Definition: MediaCurl.cc:559
virtual void getDir(const Pathname &dirname, bool recurse_r) const override
Call concrete handler to provide directory content (not recursive!) below attach point.
Definition: MediaCurl.cc:1234
TransferSettings & settings()
Definition: MediaCurl.cc:222
bool authenticate(const std::string &availAuthTypes, bool firstTry) const
Definition: MediaCurl.cc:1350
virtual void getFile(const Pathname &filename, const ByteCount &expectedFileSize_r) const override
Call concrete handler to provide file below attach point.
Definition: MediaCurl.cc:604
static CURL * progressCallback_getcurl(void *clientp)
Definition: MediaCurl.cc:1313
virtual void releaseFrom(const std::string &ejectDev) override
Call concrete handler to release the media.
Definition: MediaCurl.cc:583
void doGetFileCopyFile(const Pathname &srcFilename, const Pathname &dest, FILE *file, callback::SendReport< DownloadProgressReport > &_report, const ByteCount &expectedFileSize_r, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1127
virtual void disconnectFrom() override
Definition: MediaCurl.cc:566
virtual bool doGetDoesFileExist(const Pathname &filename) const
Definition: MediaCurl.cc:831
curl_slist * _customHeaders
Definition: MediaCurl.h:178
std::string getAuthHint() const
Return a comma separated list of available authentication methods supported by server.
Definition: MediaCurl.cc:1321
Just inherits Exception to separate media exceptions.
Abstract base class for 'physical' MediaHandler like MediaCD, etc.
Definition: MediaHandler.h:45
bool isUseableAttachPoint(const Pathname &path, bool mtab=true) const
Ask media manager, if the specified path is already used as attach point or if there are another atta...
Url url() const
Url used.
Definition: MediaHandler.h:507
virtual bool checkAttachPoint(const Pathname &apoint) const
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
void disconnect()
Use concrete handler to isconnect media.
Pathname createAttachPoint() const
Try to create a default / temporary attach point.
void setMediaSource(const MediaSourceRef &ref)
Set new media source reference.
Pathname localPath(const Pathname &pathname) const
Files provided will be available at 'localPath(filename)'.
const Url _url
Url to handle.
Definition: MediaHandler.h:110
void getDirectoryYast(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const
Retrieve and if available scan dirname/directory.yast.
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
Pathname attachPoint() const
Return the currently used attach point.
Media source internally used by MediaManager and MediaHandler.
Definition: MediaSource.h:37
const std::string & hint() const
comma separated list of available authentication types
Holds transfer setting.
std::string proxy() const
proxy host
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
long connectTimeout() const
connection timeout
std::string password() const
auth password
long timeout() const
transfer timeout
Headers::const_iterator headersEnd() const
end iterators to additional headers
std::string userPassword() const
returns the user and password as a user:pass string
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
void setAuthType(std::string &&val_r)
set the allowed authentication types
void setUsername(std::string &&val_r)
sets the auth username
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3"
void setConnectTimeout(long t)
set the connect timeout
std::string userAgentString() const
user agent string
void setPassword(std::string &&val_r)
sets the auth password
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar"
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
bool verifyHostEnabled() const
Whether to verify host for ssl.
Pathname clientCertificatePath() const
SSL client certificate file.
Pathname certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
bool headRequestsAllowed() const
whether HEAD requests are allowed
std::string proxyUsername() const
proxy auth username
std::string authType() const
get the allowed authentication types
bool proxyEnabled() const
proxy is enabled
std::string username() const
auth username
Pathname clientKeyPath() const
SSL client key file.
void setTimeout(long t)
set the transfer timeout
Headers::const_iterator headersBegin() const
begin iterators to additional headers
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
int ZYPP_MEDIA_CURL_IPRESOLVE()
Definition: CurlHelper.h:36
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: CurlHelper.cc:110
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition: CurlHelper.cc:62
const char * anonymousIdHeader()
initialized only once, this gets the anonymous id from the target, which we pass in the http header
Definition: CurlHelper.cc:294
void globalInitCurlOnce()
Definition: CurlHelper.cc:19
const char * distributionFlavorHeader()
initialized only once, this gets the distribution flavor from the target, which we pass in the http h...
Definition: CurlHelper.cc:308
std::string curlUnEscape(std::string text_r)
Definition: CurlHelper.cc:351
const char * agentString()
initialized only once, this gets the agent string which also includes the curl version
Definition: CurlHelper.cc:322
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: CurlHelper.cc:258
Url clearQueryString(const Url &url)
Definition: CurlHelper.cc:358
int log_curl(CURL *curl, curl_infotype info, char *ptr, size_t len, void *max_lvl)
Definition: CurlHelper.cc:28
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:813
int unlink(const Pathname &path)
Like 'unlink'.
Definition: PathInfo.cc:662
int rename(const Pathname &oldpath, const Pathname &newpath)
Like 'rename'.
Definition: PathInfo.cc:704
int rmdir(const Pathname &path)
Like 'rmdir'.
Definition: PathInfo.cc:367
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
Definition: PathInfo.cc:320
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
Definition: PathInfo.cc:1164
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:547
shared_ptr< CurlAuthData > CurlAuthData_Ptr
shared_ptr< AuthData > AuthData_Ptr
Definition: MediaUserAuth.h:69
std::string numstring(char n, int w=0)
Definition: String.h:286
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:426
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:2
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition: ManagedFile.h:27
AutoDispose<int> calling ::close
Definition: AutoDispose.h:281
AutoDispose<FILE*> calling ::fclose
Definition: AutoDispose.h:292
Structure holding values of curlrc options.
Definition: CurlConfig.h:17
std::string proxyuserpwd
Definition: CurlConfig.h:39
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: CurlConfig.cc:24
Convenient building of std::string with boost::format.
Definition: String.h:250