23 inline bool preloadEnabled()
25 const char *val = ::getenv(
"ZYPP_PCK_PRELOAD");
31 return ( std::string_view( val ) ==
"1" );
34 zypp::Pathname pckCachedLocation (
const PoolItem &pck ) {
76 if (
_parent._requiredDls.empty() ) {
84 MIL <<
"No more jobs pending, exiting worker" << std::endl;
92 _parent._requiredDls.pop_front();
94 auto loc =
_job.lookupLocation();
100 userData.
set(
"description",
_(
"No mirror found") );
106 ERR <<
"Failed to create target dir for file: " <<
_targetPath << std::endl;
108 userData.
set(
"description",
_(
"Could not create target File") );
123 if ( !pathInfo.
isFile() ) {
124 if ( pathInfo.
isDir () )
129 }
else if ( is_checksum(
_targetPath, loc.checksum() ) ) {
133 userData.
set(
"Url",
url );
134 userData.
set(
"description",
_(
"Already in Cache") );
156 _req->resetRequestRanges();
164 _req->transferSettings() = settings;
177 const auto &pi =
_job;
224 auto &repoDlInfo =
_parent._dlRepoInfo.at(
_job.repository().id() );
226 std::vector<RepoUrl>::iterator curr = repoDlInfo._baseUrls.end();
227 int currentSmallestRef = INT_MAX;
229 for (
auto i = repoDlInfo._baseUrls.begin(); i != repoDlInfo._baseUrls.end(); i++ ) {
230 auto mirrorPtr = &(*i);
232 if ( skip == mirrorPtr )
240 if ( ( i->refs + i->miss ) < currentSmallestRef ) {
241 currentSmallestRef = ( i->refs + i->miss );
246 if ( curr == repoDlInfo._baseUrls.end() )
252 MIL <<
"Request for " << req.
url() <<
" started" << std::endl;
260 userData.
set(
"Url",
_req->url() );
271 _parent.reportBytesDownloaded( downloaded );
275 MIL <<
"Request for " << req.
url() <<
" finished. (" << err.
toString() <<
")" << std::endl;
282 userData.
set(
"Url", req.
url() );
283 userData.
set(
"description",
_(
"Finished") );
288 const auto &error = req.
error();
289 switch ( error.type() ) {
313 std::string authHint = error.extraInfoValue(
"authHint", std::string());
328 MIL <<
"Download from mirror failed for file " << req.
url () <<
" trying to taint mirror and move on" << std::endl;
337 MIL <<
"Found new mirror: " <<
url <<
" recovering, retry count: " <<
_notFoundRetry << std::endl;
340 _req->transferSettings () = settings;
351 DBG <<
"BUG: Download error flag is set , but Error code is NoError" << std::endl;
362 userData.
set(
"Url",
url );
364 userData.
set(
"description",
_(
"Already in Cache") );
366 _parent._missedDownloads =
true;
367 _parent._report->fileDone( localPath, e, userData );
380 if ( settings.
proxy().empty() )
386 const auto &loc =
_job.lookupLocation();
392 url.appendPathName( loc.filename() );
395 for (
const auto & el :
_myMirror->headers ) {
396 std::string header { el.first };
399 MIL <<
"Added custom header -> " << header << std::endl;
404 resultSet = settings;
410 zyppng::NetworkRequestRef
_req;
434 if ( !preloadEnabled() ) {
435 MIL <<
"CommitPackagePreloader disabled" << std::endl;
441 MIL <<
"No receiver for the CommitPreloadReport, skipping preload phase" << std::endl;
446 _dispatcher = std::make_shared<zyppng::NetworkRequestDispatcher>();
453 _pTracker = std::make_shared<internal::ProgressTracker>();
463 for (
const auto &step : steps ) {
464 switch ( step.stepType() )
483 if ( pi->lookupLocation().checksum().empty() )
487 if( !pckCachedLocation(pi).empty() )
495 ERR <<
"Failed to create predownload cache for repo " << pi.
repoInfo().
alias() << std::endl;
500 std::vector<RepoUrl> repoUrls;
502 std::for_each( bu.begin(), bu.end(), [&](
const zypp::Url &u ) {
503 media::UrlResolverPlugin::HeaderList custom_headers;
504 Url url = media::UrlResolverPlugin::resolveUrl(u, custom_headers);
505 MIL <<
"Trying scheme '" << url.getScheme() <<
"'" << std::endl;
507 if ( media::MediaHandlerFactory::handlerType(url) != media::MediaHandlerFactory::MediaCURLType )
510 repoUrls.push_back( RepoUrl {
511 .baseUrl = std::move(url),
512 .headers = std::move(custom_headers)
517 if( repoUrls.empty() ) {
518 MIL <<
"Skipping predownload for " << step.satSolvable() <<
" no downloading URL" << std::endl;
524 if ( repoUrls.begin()->baseUrl.getHost() ==
"download.opensuse.org" ){
531 ._baseUrls = std::move(repoUrls)
541 if ( _requiredDls.empty() )
545 std::sort( _requiredDls.begin(), _requiredDls.end(), [](
const PoolItem &
a ,
const PoolItem &
b ) { return a.repository() < b.repository(); });
547 const auto &workerDone = [&,
this](){
548 if ( std::all_of( _workers.begin(), _workers.end(), [](
const auto &w ) { return w->finished();} ) )
562 if (_requiredDls.empty())
564 auto worker = std::make_shared<PreloadWorker>(*
this);
565 worker->sigWorkerFinished().connect(workerDone);
567 _workers.push_back( std::move(worker) );
570 if( std::any_of( _workers.begin(), _workers.end(), [](
const auto &w ) { return !w->finished(); } ) ) {
571 MIL <<
"Running preload event loop!" << std::endl;
575 MIL <<
"Preloading done, returning" << std::endl;
580 if ( !preloadEnabled() ) {
581 MIL <<
"CommitPackagePreloader disabled" << std::endl;
585 filesystem::clean_dir ( Repository(elem.first).info().predownloadPath() );
600 userData.
set(
"dbps_avg" ,
static_cast<double>(
_pTracker->_drateTotal ) );
601 userData.
set(
"dbps_current",
static_cast<double>(
_pTracker->_drateLast ) );
602 userData.
set(
"bytesReceived",
static_cast<double>(
_pTracker->_dnlNow ) );
603 userData.
set(
"bytesRequired",
static_cast<double>(
_pTracker->_dnlTotal ) );
Store and operate with byte count.
void onRequestProgress(zyppng::NetworkRequest &req, zypp::ByteCount count)
RepoUrl * findUsableMirror(RepoUrl *skip=nullptr, bool allowTainted=true)
Tries to find a usable mirror.
zyppng::NetworkRequestRef _req
Repository::IdType _currentRepoId
void makeJobUrl(zypp::Url &resultUrl, media::TransferSettings &resultSet)
void onRequestStarted(zyppng::NetworkRequest &req)
void failCurrentJob(const zypp::Pathname &localPath, const std::optional< zypp::Url > &url, media::CommitPreloadReport::Error e, const std::optional< std::string > &errorMessage)
zyppng::SignalProxy< void()> sigWorkerFinished()
PreloadWorker(CommitPackagePreloader &parent)
zyppng::Signal< void()> _sigFinished
zypp::Pathname _targetPath
bool taintCurrentMirror()
Taints the current mirror, returns true if a alternative was found.
std::set< RepoUrl * > _taintedMirrors
CommitPackagePreloader & _parent
void onRequestFinished(zyppng::NetworkRequest &req, const zyppng::NetworkRequestError &err)
callback::SendReport< media::CommitPreloadReport > _report
ByteCount _downloadedBytes
zyppng::Ref< internal::ProgressTracker > _pTracker
std::map< Repository::IdType, RepoDownloadData > _dlRepoInfo
void reportBytesDownloaded(ByteCount newBytes)
void preloadTransaction(const std::vector< sat::Transaction::Step > &steps)
zyppng::NetworkRequestDispatcherRef _dispatcher
std::deque< PoolItem > _requiredDls
Combining sat::Solvable and ResStatus.
Pathname predownloadPath() const
Path where this repo packages are predownloaded.
url_set baseUrls() const
The complete set of repository urls.
IdType id() const
Expert backdoor.
sat::detail::RepoIdType IdType
static ZConfig & instance()
Singleton ctor.
Typesafe passing of user data via callbacks.
bool set(const std::string &key_r, AnyType val_r)
Set the value for key (nonconst version always returns true).
Wrapper class for stat/lstat.
bool isExist() const
Return whether valid stat info exists.
static ManagedFile asManagedFile()
Create a temporary file and convert it to a automatically cleaned up ManagedFile.
std::string alias() const
unique identifier for this source.
@ TRANSACTION_MULTIINSTALL
[M] Install(multiversion) item (
@ TRANSACTION_INSTALL
[+] Install(update) item
The NetworkRequestError class Represents a error that occured in.
std::string toString() const
toString Returns a string representation of the error
bool hasError() const
Checks if there was a error with the request.
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count)> sigBytesDownloaded()
Signals that new data has been downloaded, this is only the payload and does not include control data...
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
NetworkRequestError error() const
Returns the last set Error.
SignalProxy< void(NetworkRequest &req)> sigStarted()
Signals that the dispatcher dequeued the request and actually starts downloading data.
TransferSettings & transferSettings()
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Url clearQueryString(const Url &url)
int rmdir(const Pathname &path)
Like 'rmdir'.
int unlink(const Pathname &path)
Like 'unlink'.
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
int rename(const Pathname &oldpath, const Pathname &newpath)
Like 'rename'.
static const RepoIdType noRepoId(0)
Id to denote Repo::noRepository.
const std::string & asString(const std::string &t)
Global asString() that works with std::string too.
Easy-to use interface to the ZYPP dependency resolver.
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Pathname cachedLocation(const OnMediaLocation &loc_r, const RepoInfo &repo_r)
media::UrlResolverPlugin::HeaderList headers
RepoInfo repoInfo() const
Repository repository() const