libzypp: zypp/repo/susetags/Downloader.cc Source File

libzypp 17.25.10

Downloader.cc
Go to the documentation of this file.
1
2 #include <iostream>
3 #include <fstream>
4
5 #include <zypp/base/LogTools.h>
6 #include <zypp/base/Gettext.h>
7 #include <zypp/base/String.h>
8 #include <zypp/base/Regex.h>
9 #include <zypp/OnMediaLocation.h>
10 #include <zypp/MediaSetAccess.h>
11 #include <zypp/Fetcher.h>
12 #include <zypp/Locale.h>
13 #include <zypp/ZConfig.h>
14 #include <zypp/repo/MediaInfoDownloader.h>
15 #include <zypp/repo/susetags/Downloader.h>
16 #include <zypp/parser/ParseException.h>
17 #include <zypp/parser/susetags/RepoIndex.h>
18 #include <zypp/base/UserRequestException.h>
19
20 using namespace zypp::parser;
21 using namespace zypp::parser::susetags;
22
23 namespace zypp
24 {
25 namespace repo
26 {
27 namespace susetags
28 {
29
30 Downloader::Downloader( const RepoInfo &repoinfo, const Pathname &delta_dir )
31 : repo::Downloader(repoinfo), _delta_dir(delta_dir)
32 {
33 }
34
35 RepoStatus Downloader::status( MediaSetAccess &media )
36 {
37 RepoStatus ret( media.provideOptionalFile( repoInfo().path() + "/content") );
38 if ( !ret.empty() ) // else: mandatory master index is missing
39 ret = ret && RepoStatus( media.provideOptionalFile( "/media.1/media" ) );
40 // else: mandatory master index is missing -> stay empty
41 return ret;
42 }
43
44 // search old repository file file to run the delta algorithm on
45 static Pathname search_deltafile( const Pathname &dir, const Pathname &file )
46 {
47 Pathname deltafile(dir + file.basename());
48 if (PathInfo(deltafile).isExist())
49 return deltafile;
50 return Pathname();
51 }
52
53
55 void Downloader::download( MediaSetAccess &media,
56 const Pathname &dest_dir,
57 const ProgressData::ReceiverFnc & progress )
58 {
59 downloadMediaInfo( dest_dir, media );
60
61 Pathname masterIndex( repoInfo().path() / "/content" );
62 defaultDownloadMasterIndex( media, dest_dir, masterIndex );
63
64 // Content file first to get the repoindex
65 {
66 Pathname inputfile( dest_dir / masterIndex );
67 ContentFileReader content;
68 content.setRepoIndexConsumer( bind( &Downloader::consumeIndex, this, _1 ) );
69 content.parse( inputfile );
70 }
71 if ( ! _repoindex )
72 {
73 ZYPP_THROW( ParseException( (dest_dir+repoInfo().path()).asString() + ": " + "No repository index in content file." ) );
74 }
75 MIL << "RepoIndex: " << _repoindex << endl;
76 if ( _repoindex->metaFileChecksums.empty() )
77 {
78 ZYPP_THROW( ParseException( (dest_dir+repoInfo().path()).asString() + ": " + "No metadata checksums in content file." ) );
79 }
80 if ( _repoindex->signingKeys.empty() )
81 {
82 WAR << "No signing keys defined." << endl;
83 }
84
85 // Prepare parsing
86 Pathname descr_dir = _repoindex->descrdir; // path below reporoot
87 //_datadir = _repoIndex->datadir; // path below reporoot
88
89 std::map<std::string,RepoIndex::FileChecksumMap::const_iterator> availablePackageTranslations;
90
91 for_( it, _repoindex->metaFileChecksums.begin(), _repoindex->metaFileChecksums.end() )
92 {
93 // omit unwanted translations
94 if ( str::hasPrefix( it->first, "packages" ) )
95 {
96 static const str::regex rx_packages( "^packages((.gz)?|(.([^.]*))(.gz)?)$" );
97 str::smatch what;
98 if ( str::regex_match( it->first, what, rx_packages ) )
99 {
100 if ( what[4].empty() // packages(.gz)?
101 || what[4] == "DU"
102 || what[4] == "en" )
103 { ; /* always downloaded */ }
104 else if ( what[4] == "FL" )
105 { continue; /* never downloaded */ }
106 else
107 {
108 // remember and decide later
109 availablePackageTranslations[what[4]] = it;
110 continue;
111 }
112 }
113 else
114 continue; // discard
115 }
116 else if ( it->first == "patterns.pat"
117 || it->first == "patterns.pat.gz" )
118 {
119 // take all patterns in one go
120 }
121 else if ( str::endsWith( it->first, ".pat" )
122 || str::endsWith( it->first, ".pat.gz" ) )
123 {
124
125 // *** see also zypp/parser/susetags/RepoParser.cc ***
126
127 // omit unwanted patterns, see https://bugzilla.novell.com/show_bug.cgi?id=298716
128 // expect "<name>.<arch>.pat[.gz]", <name> might contain additional dots
129 // split at dots, take .pat or .pat.gz into account
130
131 std::vector<std::string> patparts;
132 unsigned archpos = 2;
133 // expect "<name>.<arch>.pat[.gz]", <name> might contain additional dots
134 unsigned count = str::split( it->first, std::back_inserter(patparts), "." );
135 if ( patparts[count-1] == "gz" )
136 archpos++;
137
138 if ( count > archpos )
139 {
140 try // might by an invalid architecture
141 {
142 Arch patarch( patparts[count-archpos] );
143 if ( !patarch.compatibleWith( ZConfig::instance().systemArchitecture() ) )
144 {
145 // discard, if not compatible
146 MIL << "Discarding pattern " << it->first << endl;
147 continue;
148 }
149 }
150 catch ( const Exception & excpt )
151 {
152 WAR << "Pattern file name does not contain recognizable architecture: " << it->first << endl;
153 // keep .pat file if it doesn't contain an recognizable arch
154 }
155 }
156 }
157 MIL << "adding job " << it->first << endl;
158 OnMediaLocation location( repoInfo().path() + descr_dir + it->first, 1 );
159 location.setChecksum( it->second );
160 enqueueDigested(location, FileChecker(), search_deltafile(_delta_dir + descr_dir, it->first));
161 }
162
163 // check whether to download more package translations:
164 {
165 auto fnc_checkTransaltions( [&]( const Locale & locale_r ) {
166 for ( Locale toGet( locale_r ); toGet; toGet = toGet.fallback() )
167 {
168 auto it( availablePackageTranslations.find( toGet.code() ) );
169 if ( it != availablePackageTranslations.end() )
170 {
171 auto mit( it->second );
172 MIL << "adding job " << mit->first << endl;
173 OnMediaLocation location( repoInfo().path() + descr_dir + mit->first, 1 );
174 location.setChecksum( mit->second );
175 enqueueDigested(location, FileChecker(), search_deltafile(_delta_dir + descr_dir, mit->first));
176 break;
177 }
178 }
179 });
180 for ( const Locale & it : ZConfig::instance().repoRefreshLocales() )
181 {
182 fnc_checkTransaltions( it );
183 }
184 fnc_checkTransaltions( ZConfig::instance().textLocale() );
185 }
186
187 for_( it, _repoindex->mediaFileChecksums.begin(), _repoindex->mediaFileChecksums.end() )
188 {
189 // Repo adopts license files listed in HASH
190 if ( it->first != "license.tar.gz" )
191 continue;
192
193 MIL << "adding job " << it->first << endl;
194 OnMediaLocation location( repoInfo().path() + it->first, 1 );
195 location.setChecksum( it->second );
196 enqueueDigested(location, FileChecker(), search_deltafile(_delta_dir, it->first));
197 }
198
199 for_( it, _repoindex->signingKeys.begin(),_repoindex->signingKeys.end() )
200 {
201 MIL << "adding job " << it->first << endl;
202 OnMediaLocation location( repoInfo().path() + it->first, 1 );
203 location.setChecksum( it->second );
204 enqueueDigested(location);
205 }
206
207 start( dest_dir, media );
208 }
209
210 void Downloader::consumeIndex( const RepoIndex_Ptr & data_r )
211 {
212 MIL << "Consuming repo index" << endl;
213 _repoindex = data_r;
214 }
215
216 }// ns susetags
217 }// ns source
218 } // ns zypp
for_
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Definition: Easy.h:28
ZYPP_THROW
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:392
Fetcher.h
Gettext.h
Interface to gettext.
Locale.h
LogTools.h
MIL
#define MIL
Definition: Logger.h:91
WAR
#define WAR
Definition: Logger.h:92
MediaInfoDownloader.h
MediaSetAccess.h
OnMediaLocation.h
ParseException.h
Regex.h
RepoIndex.h
String.h
UserRequestException.h
ZConfig.h
zypp::Arch
Architecture.
Definition: Arch.h:37
zypp::Arch::compatibleWith
bool compatibleWith(const Arch &targetArch_r) const
Compatibility relation.
Definition: Arch.cc:501
zypp::Exception
Base class for Exception.
Definition: Exception.h:146
zypp::Fetcher::start
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir You have to provde a media set access media to...
Definition: Fetcher.cc:872
zypp::Fetcher::enqueueDigested
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
Enqueue a object for transferal, they will not be transferred until start() is called.
Definition: Fetcher.cc:831
zypp::Locale
'Language[_Country]' codes.
Definition: Locale.h:50
zypp::Locale::fallback
Locale fallback() const
Return the fallback locale for this locale, if no fallback exists the empty Locale::noCode.
Definition: Locale.cc:208
zypp::MediaSetAccess
Media access layer responsible for handling files distributed on a set of media with media change and...
Definition: MediaSetAccess.h:81
zypp::MediaSetAccess::provideOptionalFile
Pathname provideOptionalFile(const Pathname &file, unsigned media_nr=1)
Provides an optional file from media media_nr.
Definition: MediaSetAccess.cc:178
zypp::OnMediaLocation
Describes a resource file located on a medium.
Definition: OnMediaLocation.h:39
zypp::OnMediaLocation::setChecksum
OnMediaLocation & setChecksum(CheckSum val_r)
Set the checksum.
Definition: OnMediaLocation.cc:121
zypp::ProgressData::ReceiverFnc
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
Definition: ProgressData.h:139
zypp::RepoInfo
What is known about a repository.
Definition: RepoInfo.h:72
zypp::RepoStatus
Track changing files or directories.
Definition: RepoStatus.h:41
zypp::RepoStatus::empty
bool empty() const
Whether the status is empty (empty checksum)
Definition: RepoStatus.cc:222
zypp::ZConfig::systemArchitecture
Arch systemArchitecture() const
The system architecture zypp uses.
Definition: ZConfig.cc:856
zypp::ZConfig::instance
static ZConfig & instance()
Singleton ctor.
Definition: Resolver.cc:126
zypp::ZConfig::repoRefreshLocales
LocaleSet repoRefreshLocales() const
List of locales for which translated package descriptions should be downloaded.
Definition: ZConfig.cc:1034
zypp::filesystem::PathInfo
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:221
zypp::filesystem::Pathname
Pathname.
Definition: Pathname.h:45
zypp::filesystem::Pathname::basename
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:128
zypp::parser::ParseException
Definition: ParseException.h:34
zypp::parser::susetags::ContentFileReader
Parse repoindex part from a content file.
Definition: ContentFileReader.h:46
zypp::parser::susetags::ContentFileReader::parse
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
Definition: ContentFileReader.cc:175
zypp::parser::susetags::ContentFileReader::setRepoIndexConsumer
void setRepoIndexConsumer(const RepoIndexConsumer &fnc_r)
Consumer to call when repo index was parsed.
Definition: ContentFileReader.h:66
zypp::repo::Downloader::defaultDownloadMasterIndex
void defaultDownloadMasterIndex(MediaSetAccess &media_r, const Pathname &destdir_r, const Pathname &masterIndex_r)
Common workflow downloading a (signed) master index file.
Definition: Downloader.cc:49
zypp::repo::Downloader::Downloader
Downloader()
Constructor.
Definition: Downloader.cc:26
zypp::repo::Downloader::repoInfo
const RepoInfo & repoInfo() const
Definition: Downloader.h:58
zypp::repo::susetags::Downloader
Downloader for SUSETags (YaST2) repositories Encapsulates all the knowledge of which files have to be...
Definition: Downloader.h:35
zypp::repo::susetags::Downloader::download
void download(MediaSetAccess &media, const Pathname &dest_dir, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc()) override
Download metadata to a local directory.
Definition: Downloader.cc:55
zypp::repo::susetags::Downloader::_delta_dir
Pathname _delta_dir
Definition: Downloader.h:69
zypp::repo::susetags::Downloader::consumeIndex
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
Content file parser consumer.
Definition: Downloader.cc:210
zypp::repo::susetags::Downloader::_repoindex
parser::susetags::RepoIndex_Ptr _repoindex
Definition: Downloader.h:68
zypp::repo::susetags::Downloader::status
RepoStatus status(MediaSetAccess &media) override
Status of the remote repository.
Definition: Downloader.cc:35
zypp::str::regex
Regular expression.
Definition: Regex.h:95
zypp::str::smatch
Regular expression match result.
Definition: Regex.h:163
zypp::parser::susetags
Definition: ContentFileReader.cc:36
zypp::parser
Definition: HistoryLogReader.cc:29
zypp::repo::susetags::search_deltafile
static Pathname search_deltafile(const Pathname &dir, const Pathname &file)
Definition: Downloader.cc:45
zypp::repo::downloadMediaInfo
void downloadMediaInfo(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progressrcv)
Downloads the media info (/media.1) to a local directory.
Definition: MediaInfoDownloader.cc:24
zypp::str::hasPrefix
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition: String.h:1023
zypp::str::endsWith
bool endsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasSuffix
Definition: String.h:1088
zypp::str::regex_match
bool regex_match(const std::string &s, smatch &matches, const regex &regex)
\relates regex \ingroup ZYPP_STR_REGEX \relates regex \ingroup ZYPP_STR_REGEX
Definition: Regex.h:70
zypp::str::split
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \t", const Trim trim_r=NO_TRIM)
Split line_r into words.
Definition: String.h:527
zypp
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:2
zypp::asString
std::string asString(const DefaultIntegral< Tp, TInitial > &obj)
Definition: DefaultIntegral.h:98
zypp::FileChecker
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
Definition: FileChecker.h:28
Downloader.h