21#include <zypp-core/base/DefaultIntegral>
23#include <zypp-media/MediaException>
27#include <zypp-core/base/UserRequestException>
31#undef ZYPP_BASE_LOGGER_LOGGROUP
32#define ZYPP_BASE_LOGGER_LOGGROUP "zypp::fetcher"
68 if ( lhs->location.medianr() == rhs->location.medianr() )
69 return lhs->location.filename() < rhs->location.filename();
71 return lhs->location.medianr() < rhs->location.medianr();
118 return str <<
obj->location;
141 Fetcher::Options
options()
const;
228 {
return new Impl( *
this ); }
231 std::set<FetcherIndex_Ptr,SameFetcherIndex>
_indexes;
278 _resources.push_back(
job);
294 _resources.push_back(
job);
300 if ( _mediaSetAccess )
301 _mediaSetAccess->precacheFiles( {
resource} );
307 _resources.push_back(
job);
327 _mediaSetAccess = &media;
337 DBG <<
"Adding fetcher cache: '" <<
cache_dir <<
"'." << endl;
343 ERR <<
"Not adding cache: '" <<
cache_dir <<
"'. Not a directory." << endl;
348 ERR <<
"Not adding cache '" <<
cache_dir <<
"'. Path does not exists." << endl;
368 MIL <<
"start fetcher with " << _caches.size() <<
" cache directories." << endl;
426 downloadAndReadIndexList( media,
dest_dir );
444 if ( _dircontent.find(
resource.filename().asString())
445 != _dircontent.end() )
449 std::copy(
filled.begin(),
filled.end(), std::back_inserter(content));
458 std::copy(
tofill.begin(),
tofill.end(), std::back_inserter(content));
469 MIL <<
"Adding directory " <<
resource.filename() << endl;
472 getDirectoryContent(media,
resource, content);
477 WAR <<
"Skipping subtree hidden at " <<
resource.filename() << endl;
485 for ( filesystem::DirContent::const_iterator
it = content.begin();
501 if ( _checksums.find(filename.
asString()) != _checksums.end() )
508 WAR <<
"Resource " << filename <<
" has no checksum in the index either." << endl;
518 addDirJobs(media, filename,
dest_dir, flags);
539 MIL <<
"Not found in cache, retrieving..." << endl;
571 WAR <<
"optional resource " <<
resource <<
" could not be transferred." << endl;
576 excpt.remember(
"Can't provide " +
resource.filename().asString() );
587 setRepoIndexConsumer( bind( &ContentReaderHelper::consumeIndex,
this, _1 ) );
599 if (
index.basename() ==
"CHECKSUMS" ||
index.basename() ==
"SHA1SUMS" )
601 else if (
index.basename() ==
"content" )
604 WAR <<
index <<
": index file format not known" << endl;
612 MIL <<
index <<
" contains " <<
reader._repoindex->mediaFileChecksums.size() <<
" checksums." << endl;
613 for_(
it,
reader._repoindex->mediaFileChecksums.begin(),
reader._repoindex->mediaFileChecksums.end() )
623 std::ifstream
in(
index.c_str() );
634 if ( checksum.empty() )
639 WAR <<
"Missing filename in CHECKSUMS file: " <<
index.asString() <<
" (" << checksum <<
")" << endl;
671 sigloc.changeFilename(
sigloc.filename().extend(
".asc") );
672 keyloc.changeFilename(
keyloc.filename().extend(
".key") );
694 WAR <<
"No public key specified by user for index '" <<
keyloc.filename() <<
"'"<< endl;
709 if ( _indexes.empty() )
711 MIL <<
"No indexes to read." << endl;
715 for_(
it_idx, _indexes.begin(), _indexes.end() )
717 if ( (*it_idx)->read )
719 DBG <<
"Already read index " <<
PathInfo(
dest_dir + (*it_idx)->location.filename()) << endl;
724 downloadIndex( media, (*it_idx)->location,
dest_dir );
726 readIndex(
dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().dirname() );
728 MIL <<
"Remember read index " <<
PathInfo(
dest_dir + (*it_idx)->location.filename()) << endl;
729 (*it_idx)->read =
true;
732 MIL <<
"done reading indexes" << endl;
738 if ( !_mediaSetAccess )
749 _mediaSetAccess =
nullptr;
754 downloadAndReadIndexList(media,
dest_dir);
775 getDirectoryContent(media,
jobp->location.filename().dirname(), content);
778 MIL <<
"Autodiscovering signed indexes on '"
779 <<
jobp->location.filename().dirname() <<
"' for '"
780 <<
jobp->location.filename() <<
"'" << endl;
782 autoaddIndexes(content, media,
jobp->location.filename().dirname(),
dest_dir);
786 getDirectoryContent(media,
Pathname(
"/"), content);
789 MIL <<
"Autodiscovering signed indexes on '"
791 <<
jobp->location.filename() <<
"'" << endl;
798 if (
jobp->location.checksum().empty() )
800 if ( _checksums.find(
jobp->location.filename().asString())
801 != _checksums.end() )
830 if ( ! progress.
incr() )
838 for ( std::list<FetcherJob_Ptr>::const_iterator
it_res =
obj._resources.begin();
it_res !=
obj._resources.end(); ++
it_res )
911 _pimpl->setMediaSetAccess( media );
Reference counted access to a Tp object calling a custom Dispose function when the last AutoDispose h...
void resetDispose()
Set no dispose function.
void reset()
Reset to default Ctor values.
shared_ptr< Impl > _pimpl
void setDispose(const Dispose &dispose_r)
Set a new dispose function.
Base class for Exception.
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
void autoaddIndexes(const filesystem::DirContent &content, MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
auto discovery and reading of indexes
void downloadIndex(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
download the indexes and reads them
Impl & operator=(const Impl &)=delete
void readContentFileIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for content file
void readIndex(const Pathname &index, const Pathname &basedir)
reads a downloaded index file and updates internal attributes table
std::set< FetcherIndex_Ptr, SameFetcherIndex > _indexes
friend std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
std::map< std::string, CheckSum > _checksums
void downloadAndReadIndexList(MediaSetAccess &media, const Pathname &dest_dir)
download the indexes and reads them
void addDirJobs(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags)
scan the directory and adds the individual jobs
Impl * clone() const
clone for RWCOW_pointer
void addCachePath(const Pathname &cache_dir)
std::list< FetcherJob_Ptr > _resources
void readChecksumsIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for CHECKSUMS file
void enqueueDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Impl & operator=(Impl &&)=delete
void addIndex(const OnMediaLocation &resource)
std::map< std::string, filesystem::DirContent > _dircontent
void provideToDest(MediaSetAccess &media_r, const Pathname &destDir_r, const FetcherJob_Ptr &jobp_r)
Provide the resource to dest_dir.
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
MediaSetAccess * _mediaSetAccess
Pathname locateInCache(const OnMediaLocation &resource_r, const Pathname &destDir_r)
Tries to locate the file represented by job by looking at the cache (matching checksum is mandatory).
std::set< Pathname > _caches
Impl(const Impl &)=default
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
void validate(const Pathname &localfile_r, const std::list< FileChecker > &checkers_r)
Validates the provided file against its checkers.
Fetcher::Options options() const
void getDirectoryContent(MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content)
reads the content of a directory but keeps a cache
static shared_ptr< Impl > nullimpl()
Offer default Impl.
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress)
void setOptions(Fetcher::Options options)
Fetcher::Options _options
void setMediaSetAccess(MediaSetAccess &media)
This class allows to retrieve a group of files in a confortable way, providing some smartness that do...
void start(const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir The media has to be provides with setMediaSetA...
RWCOW_pointer< Impl > _pimpl
Pointer to implementation.
void addIndex(const OnMediaLocation &resource)
Adds an index containing metadata (for example checksums ) that will be retrieved and read before the...
Options options() const
Get current options.
void reset()
Reset the transfer (jobs) list.
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
void enqueueDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory.
void setOptions(Options options)
Set the Fetcher options.
@ AutoAddChecksumsIndexes
If a CHECKSUMS file is found, it is downloaded and read.
@ AutoAddContentFileIndexes
If a content file is found, it is downloaded and read.
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory and always check for checksums.
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
void addCachePath(const Pathname &cache_dir)
adds a directory to the list of directories where to look for cached files
void setMediaSetAccess(MediaSetAccess &media)
Sets the media set access that will be used to precache and to download the files when start is calle...
Maintain [min,max] and counter (value) for progress counting.
void sendTo(const ReceiverFnc &fnc_r)
Set ReceiverFnc.
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
bool incr(value_type val_r=1)
Increment counter value (default by 1).
Class representing one GPG Public Key (PublicKeyData + ASCII armored in a tempfile).
Checks for the validity of a signature.
Wrapper class for stat/lstat.
bool isExist() const
Return whether valid stat info exists.
const std::string & asString() const
String representation.
Parse repoindex part from a content file.
String related utilities and Regular expression matching.
int unlink(const Pathname &path)
Like 'unlink'.
std::list< DirEntry > DirContent
Returned by readdir.
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
std::string stripFirstWord(std::string &line, const bool ltrim_first)
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Easy-to use interface to the ZYPP dependency resolver.
function< void(const Pathname &file) FileChecker)
Functor signature used to check files.
std::ostream & operator<<(std::ostream &str, const SerialNumber &obj)
shared_ptr< FetcherIndex > FetcherIndex_Ptr
std::string asString(const Patch::Category &obj)
shared_ptr< FetcherJob > FetcherJob_Ptr
parser::susetags::RepoIndex_Ptr _repoindex
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
class that represents indexes which add metadata to fetcher jobs and therefore need to be retrieved i...
OnMediaLocation location
Index localtion.
FetcherIndex(const OnMediaLocation &loc)
DefaultIntegral< bool, false > read
Whether we read this index.
Class to encapsulate the OnMediaLocation object and the FileChecker together.
std::list< FileChecker > checkers
FetcherJob(const FetcherJob &)=default
FetcherJob(FetcherJob &&)=default
FetcherJob & operator=(const FetcherJob &)=default
ZYPP_DECLARE_FLAGS(Flags, Flag)
FetcherJob & operator=(FetcherJob &&)=default
FetcherJob(const OnMediaLocation &loc)
std::set ordering (less semantic)
bool operator()(const FetcherIndex_Ptr &lhs, const FetcherIndex_Ptr &rhs) const
Listentry returned by readdir.
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
#define ZYPP_DECLARE_OPERATORS_FOR_FLAGS(Name)