libzypp: zypp/repo/yum/Downloader.cc Source File

libzypp 17.25.10

Downloader.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
9
10 #include <fstream>
11 #include <solv/solvversion.h>
12 #include <zypp/base/String.h>
13 #include <zypp/base/LogTools.h>
14 #include <zypp/base/Function.h>
15 #include <zypp/ZConfig.h>
16
17 #include "Downloader.h"
18 #include <zypp/repo/MediaInfoDownloader.h>
19 #include <zypp/base/UserRequestException.h>
20 #include <zypp/parser/xml/Reader.h>
21 #include <zypp/parser/yum/RepomdFileReader.h>
22
23 using namespace zypp::xml;
24 using namespace zypp::parser::yum;
25
26 namespace zypp
27 {
28 namespace repo
29 {
30 namespace yum
31 {
33 namespace
34 {
35 inline OnMediaLocation loc_with_path_prefix( OnMediaLocation loc_r, const Pathname & prefix_r )
36 {
37 if ( ! prefix_r.empty() && prefix_r != "/" )
38 loc_r.changeFilename( prefix_r / loc_r.filename() );
39 return loc_r;
40 }
41
42 // search old repository file to run the delta algorithm on
43 Pathname search_deltafile( const Pathname & dir, const Pathname & file )
44 {
45 Pathname deltafile;
46 if ( ! PathInfo(dir).isDir() )
47 return deltafile;
48
49 // Strip the checksum preceding the file stem so we can look for an
50 // old *-primary.xml which may contain some reusable blocks.
51 std::string base { file.basename() };
52 size_t hypoff = base.find( "-" );
53 if ( hypoff != std::string::npos )
54 base.replace( 0, hypoff + 1, "" );
55
56 std::list<std::string> retlist;
57 if ( ! filesystem::readdir( retlist, dir, false ) )
58 {
59 for ( const auto & fn : retlist )
60 {
61 if ( str::endsWith( fn, base ) )
62 deltafile = fn;
63 }
64 }
65 return deltafile;
66 }
67 } // namespace
69
83 struct Downloader::Impl
84 {
85 NON_COPYABLE( Impl );
86 NON_MOVABLE( Impl );
87
88 Impl( Downloader & downloader_r, MediaSetAccess & media_r, const Pathname & destDir_r )
89 : _downloader { downloader_r }
90 , _media { media_r }
91 , _destDir { destDir_r }
92 {
93 addWantedLocale( ZConfig::instance().textLocale() );
94 for ( const Locale & it : ZConfig::instance().repoRefreshLocales() )
95 addWantedLocale( it );
96 }
97
102 bool operator()( const OnMediaLocation & loc_r, const std::string & typestr_r )
103 {
104 if ( str::endsWith( typestr_r, "_db" ) )
105 return true; // skip sqlitedb
106
107 bool zchk { str::endsWith( typestr_r, "_zck" ) };
108 #if defined(LIBSOLVEXT_FEATURE_ZCHUNK_COMPRESSION)
109 const std::string & basetype { zchk ? typestr_r.substr( 0, typestr_r.size()-4 ) : typestr_r };
110 #else
111 if ( zchk )
112 return true; // skip zchunk if not supported by libsolv
113 const std::string & basetype { typestr_r };
114 #endif
115
116 // filter well known resource types
117 if ( basetype == "other" || basetype == "filelists" )
118 return true; // skip it
119
120 // filter localized susedata
121 if ( str::startsWith( basetype, "susedata." ) )
122 {
123 // susedata.LANG
124 if ( ! wantLocale( Locale(basetype.c_str()+9) ) )
125 return true; // skip it
126 }
127
128 // may take it... (prefer zchnk)
129 if ( zchk || !_wantedFiles.count( basetype ) )
130 _wantedFiles[basetype] = loc_r;
131
132 return true;
133 }
134
135 void finalize()
136 {
137 // schedule fileS for download
138 for ( const auto & el : _wantedFiles )
139 {
140 const OnMediaLocation & loc { el.second };
141 const OnMediaLocation & loc_with_path { loc_with_path_prefix( loc, _downloader.repoInfo().path() ) };
142 _downloader.enqueueDigested( loc_with_path, FileChecker(), search_deltafile( deltaDir()/"repodata", loc.filename() ) );
143 }
144 }
145
146 private:
147 const Pathname & deltaDir() const
148 { return _downloader._deltaDir; }
149
150 bool wantLocale( const Locale & locale_r ) const
151 { return _wantedLocales.count( locale_r ); }
152
153 void addWantedLocale( Locale locale_r )
154 {
155 while ( locale_r )
156 {
157 _wantedLocales.insert( locale_r );
158 locale_r = locale_r.fallback();
159 }
160 }
161
162 private:
163 Downloader & _downloader;
164 MediaSetAccess & _media;
165 const Pathname & _destDir;
166
167 LocaleSet _wantedLocales;
168 std::map<std::string,OnMediaLocation> _wantedFiles;
169 };
170
172 //
173 // class Downloader
174 //
176
177 Downloader::Downloader( const RepoInfo & info_r, const Pathname & deltaDir_r )
178 : repo::Downloader { info_r}
179 , _deltaDir { deltaDir_r }
180 {}
181
182 void Downloader::download( MediaSetAccess & media_r, const Pathname & destDir_r, const ProgressData::ReceiverFnc & progress_r )
183 {
184 downloadMediaInfo( destDir_r, media_r );
185
186 Pathname masterIndex { repoInfo().path() / "/repodata/repomd.xml" };
187 defaultDownloadMasterIndex( media_r, destDir_r, masterIndex );
188
189 // setup parser
190 Impl pimpl( *this, media_r, destDir_r );
191 RepomdFileReader( destDir_r / masterIndex, std::ref(pimpl) );
192 pimpl.finalize();
193
194 // ready, go!
195 start( destDir_r, media_r );
196 }
197
198 RepoStatus Downloader::status( MediaSetAccess & media_r )
199 {
200 RepoStatus ret { media_r.provideOptionalFile( repoInfo().path() / "/repodata/repomd.xml" ) };
201 if ( !ret.empty() ) // else: mandatory master index is missing
202 ret = ret && RepoStatus( media_r.provideOptionalFile( "/media.1/media" ) );
203 // else: mandatory master index is missing -> stay empty
204 return ret;
205 }
206 } // namespace yum
207 } // namespace repo
208 } // namespace zypp
209
210
211
Function.h
LogTools.h
MediaInfoDownloader.h
Reader.h
RepomdFileReader.h
Interface of repomd.xml file reader.
String.h
UserRequestException.h
ZConfig.h
zypp::Fetcher::start
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir You have to provde a media set access media to...
Definition: Fetcher.cc:872
zypp::Fetcher::enqueueDigested
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:831
zypp::Locale
'Language[_Country]' codes.
Definition: Locale.h:50
zypp::Locale::fallback
Locale fallback() const
Return the fallback locale for this locale, if no fallback exists the empty Locale::noCode.
Definition: Locale.cc:208
zypp::MediaSetAccess
Media access layer responsible for handling files distributed on a set of media with media change and...
Definition: MediaSetAccess.h:81
zypp::MediaSetAccess::provideOptionalFile
Pathname provideOptionalFile(const Pathname &file, unsigned media_nr=1)
Provides an optional file from media media_nr.
Definition: MediaSetAccess.cc:178
zypp::OnMediaLocation
Describes a resource file located on a medium.
Definition: OnMediaLocation.h:39
zypp::OnMediaLocation::filename
const Pathname & filename() const
The path to the resource on the medium.
Definition: OnMediaLocation.cc:85
zypp::OnMediaLocation::changeFilename
OnMediaLocation & changeFilename(Pathname filename_r)
Individual manipulation of filename (prefer setLocation).
Definition: OnMediaLocation.cc:97
zypp::ProgressData::ReceiverFnc
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
Definition: ProgressData.h:139
zypp::RepoInfo
What is known about a repository.
Definition: RepoInfo.h:72
zypp::RepoInfo::path
Pathname path() const
Repository path.
Definition: RepoInfo.cc:722
zypp::RepoStatus
Track changing files or directories.
Definition: RepoStatus.h:41
zypp::ZConfig::instance
static ZConfig & instance()
Singleton ctor.
Definition: Resolver.cc:126
zypp::filesystem::PathInfo
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:221
zypp::filesystem::Pathname
Pathname.
Definition: Pathname.h:45
zypp::filesystem::Pathname::basename
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:128
zypp::filesystem::Pathname::empty
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
zypp::parser::yum::RepomdFileReader
Reads through a repomd.xml file and collects type, location, checksum and other data about metadata f...
Definition: RepomdFileReader.h:37
zypp::repo::Downloader::defaultDownloadMasterIndex
void defaultDownloadMasterIndex(MediaSetAccess &media_r, const Pathname &destdir_r, const Pathname &masterIndex_r)
Common workflow downloading a (signed) master index file.
Definition: Downloader.cc:49
zypp::repo::Downloader::Downloader
Downloader()
Constructor.
Definition: Downloader.cc:26
zypp::repo::Downloader::repoInfo
const RepoInfo & repoInfo() const
Definition: Downloader.h:58
zypp::repo::yum::Downloader
Downloader for YUM (rpm-nmd) repositories Encapsulates all the knowledge of which files have to be do...
Definition: Downloader.h:41
zypp::repo::yum::Downloader::_deltaDir
Pathname _deltaDir
Definition: Downloader.h:73
zypp::repo::yum::Downloader::download
void download(MediaSetAccess &media_r, const Pathname &destDir_r, const ProgressData::ReceiverFnc &progress_r=ProgressData::ReceiverFnc()) override
Download metadata to a local directory.
Definition: Downloader.cc:182
zypp::repo::yum::Downloader::status
RepoStatus status(MediaSetAccess &media_r) override
Status of the remote repository.
Definition: Downloader.cc:198
zypp::filesystem::readdir
int readdir(std::list< std::string > &retlist_r, const Pathname &path_r, bool dots_r)
Return content of directory via retlist.
Definition: PathInfo.cc:598
zypp::parser::yum
Definition: RepomdFileReader.cc:35
zypp::repo::susetags::search_deltafile
static Pathname search_deltafile(const Pathname &dir, const Pathname &file)
Definition: Downloader.cc:45
zypp::repo::downloadMediaInfo
void downloadMediaInfo(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progressrcv)
Downloads the media info (/media.1) to a local directory.
Definition: MediaInfoDownloader.cc:24
zypp::str::startsWith
bool startsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasPrefix
Definition: String.h:1081
zypp::str::endsWith
bool endsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasSuffix
Definition: String.h:1088
zypp::xml
Definition: libxmlfwd.cc:22
zypp
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:2
zypp::LocaleSet
std::unordered_set< Locale > LocaleSet
Definition: Locale.h:27
zypp::FileChecker
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
Definition: FileChecker.h:28
zypp::repo::yum::Downloader::Impl
Helper filtering the files offered by a RepomdFileReader.
Definition: Downloader.cc:84
zypp::repo::yum::Downloader::Impl::_wantedLocales
LocaleSet _wantedLocales
Locales do download.
Definition: Downloader.cc:167
zypp::repo::yum::Downloader::Impl::deltaDir
const Pathname & deltaDir() const
Definition: Downloader.cc:147
zypp::repo::yum::Downloader::Impl::_destDir
const Pathname & _destDir
Definition: Downloader.cc:165
zypp::repo::yum::Downloader::Impl::_media
MediaSetAccess & _media
Definition: Downloader.cc:164
zypp::repo::yum::Downloader::Impl::wantLocale
bool wantLocale(const Locale &locale_r) const
Definition: Downloader.cc:150
zypp::repo::yum::Downloader::Impl::_downloader
Downloader & _downloader
Definition: Downloader.cc:163
zypp::repo::yum::Downloader::Impl::addWantedLocale
void addWantedLocale(Locale locale_r)
Definition: Downloader.cc:153
zypp::repo::yum::Downloader::Impl::_wantedFiles
std::map< std::string, OnMediaLocation > _wantedFiles
Definition: Downloader.cc:168
zypp::repo::yum::Downloader::Impl::operator()
bool operator()(const OnMediaLocation &loc_r, const std::string &typestr_r)
The callback invoked by the RepomdFileReader.
Definition: Downloader.cc:102
zypp::repo::yum::Downloader::Impl::NON_COPYABLE
NON_COPYABLE(Impl)
zypp::repo::yum::Downloader::Impl::finalize
void finalize()
Definition: Downloader.cc:135
zypp::repo::yum::Downloader::Impl::Impl
Impl(Downloader &downloader_r, MediaSetAccess &media_r, const Pathname &destDir_r)
Definition: Downloader.cc:88
zypp::repo::yum::Downloader::Impl::NON_MOVABLE
NON_MOVABLE(Impl)
Downloader.h