Branch data Line data Source code
1 : : /* Debuginfo-over-http server.
2 : : Copyright (C) 2019-2021 Red Hat, Inc.
3 : : This file is part of elfutils.
4 : :
5 : : This file is free software; you can redistribute it and/or modify
6 : : it under the terms of the GNU General Public License as published by
7 : : the Free Software Foundation; either version 3 of the License, or
8 : : (at your option) any later version.
9 : :
10 : : elfutils is distributed in the hope that it will be useful, but
11 : : WITHOUT ANY WARRANTY; without even the implied warranty of
12 : : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 : : GNU General Public License for more details.
14 : :
15 : : You should have received a copy of the GNU General Public License
16 : : along with this program. If not, see <http://www.gnu.org/licenses/>. */
17 : :
18 : :
19 : : /* cargo-cult from libdwfl linux-kernel-modules.c */
20 : : /* In case we have a bad fts we include this before config.h because it
21 : : can't handle _FILE_OFFSET_BITS.
22 : : Everything we need here is fine if its declarations just come first.
23 : : Also, include sys/types.h before fts. On some systems fts.h is not self
24 : : contained. */
25 : : #ifdef BAD_FTS
26 : : #include <sys/types.h>
27 : : #include <fts.h>
28 : : #endif
29 : :
30 : : #ifdef HAVE_CONFIG_H
31 : : #include "config.h"
32 : : #endif
33 : :
34 : : extern "C" {
35 : : #include "printversion.h"
36 : : }
37 : :
38 : : #include "debuginfod.h"
39 : : #include <dwarf.h>
40 : : #include <system.h>
41 : :
42 : : #include <argp.h>
43 : : #ifdef __GNUC__
44 : : #undef __attribute__ /* glibc bug - rhbz 1763325 */
45 : : #endif
46 : :
47 : : #include <unistd.h>
48 : : #include <stdlib.h>
49 : : #include <error.h>
50 : : #include <libintl.h>
51 : : #include <locale.h>
52 : : #include <pthread.h>
53 : : #include <signal.h>
54 : : #include <sys/stat.h>
55 : : #include <sys/time.h>
56 : : #include <sys/vfs.h>
57 : : #include <unistd.h>
58 : : #include <fcntl.h>
59 : : #include <netdb.h>
60 : :
61 : :
62 : : /* If fts.h is included before config.h, its indirect inclusions may not
63 : : give us the right LFS aliases of these functions, so map them manually. */
64 : : #ifdef BAD_FTS
65 : : #ifdef _FILE_OFFSET_BITS
66 : : #define open open64
67 : : #define fopen fopen64
68 : : #endif
69 : : #else
70 : : #include <sys/types.h>
71 : : #include <fts.h>
72 : : #endif
73 : :
74 : : #include <cstring>
75 : : #include <vector>
76 : : #include <set>
77 : : #include <map>
78 : : #include <string>
79 : : #include <iostream>
80 : : #include <iomanip>
81 : : #include <ostream>
82 : : #include <sstream>
83 : : #include <mutex>
84 : : #include <deque>
85 : : #include <condition_variable>
86 : : #include <thread>
87 : : // #include <regex> // on rhel7 gcc 4.8, not competent
88 : : #include <regex.h>
89 : : // #include <algorithm>
90 : : using namespace std;
91 : :
92 : : #include <gelf.h>
93 : : #include <libdwelf.h>
94 : :
95 : : #include <microhttpd.h>
96 : :
97 : : #if MHD_VERSION >= 0x00097002
98 : : // libmicrohttpd 0.9.71 broke API
99 : : #define MHD_RESULT enum MHD_Result
100 : : #else
101 : : #define MHD_RESULT int
102 : : #endif
103 : :
104 : : #include <curl/curl.h>
105 : : #include <archive.h>
106 : : #include <archive_entry.h>
107 : : #include <sqlite3.h>
108 : :
109 : : #ifdef __linux__
110 : : #include <sys/syscall.h>
111 : : #endif
112 : :
113 : : #ifdef __linux__
114 : : #define tid() syscall(SYS_gettid)
115 : : #else
116 : : #define tid() pthread_self()
117 : : #endif
118 : :
119 : :
120 : : inline bool
121 : 729 : string_endswith(const string& haystack, const string& needle)
122 : : {
123 [ + - ]: 1458 : return (haystack.size() >= needle.size() &&
124 [ + - + + ]: 729 : equal(haystack.end()-needle.size(), haystack.end(),
125 : 729 : needle.begin()));
126 : : }
127 : :
128 : :
129 : : // Roll this identifier for every sqlite schema incompatibility.
130 : : #define BUILDIDS "buildids9"
131 : :
132 : : #if SQLITE_VERSION_NUMBER >= 3008000
133 : : #define WITHOUT_ROWID "without rowid"
134 : : #else
135 : : #define WITHOUT_ROWID ""
136 : : #endif
137 : :
138 : : static const char DEBUGINFOD_SQLITE_DDL[] =
139 : : "pragma foreign_keys = on;\n"
140 : : "pragma synchronous = 0;\n" // disable fsync()s - this cache is disposable across a machine crash
141 : : "pragma journal_mode = wal;\n" // https://sqlite.org/wal.html
142 : : "pragma wal_checkpoint = truncate;\n" // clean out any preexisting wal file
143 : : "pragma journal_size_limit = 0;\n" // limit steady state file (between grooming, which also =truncate's)
144 : : "pragma auto_vacuum = incremental;\n" // https://sqlite.org/pragma.html
145 : : "pragma busy_timeout = 1000;\n" // https://sqlite.org/pragma.html
146 : : // NB: all these are overridable with -D option
147 : :
148 : : // Normalization table for interning file names
149 : : "create table if not exists " BUILDIDS "_files (\n"
150 : : " id integer primary key not null,\n"
151 : : " name text unique not null\n"
152 : : " );\n"
153 : : // Normalization table for interning buildids
154 : : "create table if not exists " BUILDIDS "_buildids (\n"
155 : : " id integer primary key not null,\n"
156 : : " hex text unique not null);\n"
157 : : // Track the completion of scanning of a given file & sourcetype at given time
158 : : "create table if not exists " BUILDIDS "_file_mtime_scanned (\n"
159 : : " mtime integer not null,\n"
160 : : " file integer not null,\n"
161 : : " size integer not null,\n" // in bytes
162 : : " sourcetype text(1) not null\n"
163 : : " check (sourcetype IN ('F', 'R')),\n"
164 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
165 : : " primary key (file, mtime, sourcetype)\n"
166 : : " ) " WITHOUT_ROWID ";\n"
167 : : "create table if not exists " BUILDIDS "_f_de (\n"
168 : : " buildid integer not null,\n"
169 : : " debuginfo_p integer not null,\n"
170 : : " executable_p integer not null,\n"
171 : : " file integer not null,\n"
172 : : " mtime integer not null,\n"
173 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
174 : : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
175 : : " primary key (buildid, file, mtime)\n"
176 : : " ) " WITHOUT_ROWID ";\n"
177 : : "create table if not exists " BUILDIDS "_f_s (\n"
178 : : " buildid integer not null,\n"
179 : : " artifactsrc integer not null,\n"
180 : : " file integer not null,\n" // NB: not necessarily entered into _mtime_scanned
181 : : " mtime integer not null,\n"
182 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
183 : : " foreign key (artifactsrc) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
184 : : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
185 : : " primary key (buildid, artifactsrc, file, mtime)\n"
186 : : " ) " WITHOUT_ROWID ";\n"
187 : : "create table if not exists " BUILDIDS "_r_de (\n"
188 : : " buildid integer not null,\n"
189 : : " debuginfo_p integer not null,\n"
190 : : " executable_p integer not null,\n"
191 : : " file integer not null,\n"
192 : : " mtime integer not null,\n"
193 : : " content integer not null,\n"
194 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
195 : : " foreign key (content) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
196 : : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
197 : : " primary key (buildid, debuginfo_p, executable_p, file, content, mtime)\n"
198 : : " ) " WITHOUT_ROWID ";\n"
199 : : "create table if not exists " BUILDIDS "_r_sref (\n" // outgoing dwarf sourcefile references from rpm
200 : : " buildid integer not null,\n"
201 : : " artifactsrc integer not null,\n"
202 : : " foreign key (artifactsrc) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
203 : : " foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"
204 : : " primary key (buildid, artifactsrc)\n"
205 : : " ) " WITHOUT_ROWID ";\n"
206 : : "create table if not exists " BUILDIDS "_r_sdef (\n" // rpm contents that may satisfy sref
207 : : " file integer not null,\n"
208 : : " mtime integer not null,\n"
209 : : " content integer not null,\n"
210 : : " foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
211 : : " foreign key (content) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"
212 : : " primary key (content, file, mtime)\n"
213 : : " ) " WITHOUT_ROWID ";\n"
214 : : // create views to glue together some of the above tables, for webapi D queries
215 : : "create view if not exists " BUILDIDS "_query_d as \n"
216 : : "select\n"
217 : : " b.hex as buildid, n.mtime, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1\n"
218 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_f_de n\n"
219 : : " where b.id = n.buildid and f0.id = n.file and n.debuginfo_p = 1\n"
220 : : "union all select\n"
221 : : " b.hex as buildid, n.mtime, 'R' as sourcetype, f0.name as source0, n.mtime as mtime, f1.name as source1\n"
222 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files f1, " BUILDIDS "_r_de n\n"
223 : : " where b.id = n.buildid and f0.id = n.file and f1.id = n.content and n.debuginfo_p = 1\n"
224 : : ";"
225 : : // ... and for E queries
226 : : "create view if not exists " BUILDIDS "_query_e as \n"
227 : : "select\n"
228 : : " b.hex as buildid, n.mtime, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1\n"
229 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_f_de n\n"
230 : : " where b.id = n.buildid and f0.id = n.file and n.executable_p = 1\n"
231 : : "union all select\n"
232 : : " b.hex as buildid, n.mtime, 'R' as sourcetype, f0.name as source0, n.mtime as mtime, f1.name as source1\n"
233 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files f1, " BUILDIDS "_r_de n\n"
234 : : " where b.id = n.buildid and f0.id = n.file and f1.id = n.content and n.executable_p = 1\n"
235 : : ";"
236 : : // ... and for S queries
237 : : "create view if not exists " BUILDIDS "_query_s as \n"
238 : : "select\n"
239 : : " b.hex as buildid, fs.name as artifactsrc, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1, null as source0ref\n"
240 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files fs, " BUILDIDS "_f_s n\n"
241 : : " where b.id = n.buildid and f0.id = n.file and fs.id = n.artifactsrc\n"
242 : : "union all select\n"
243 : : " b.hex as buildid, f1.name as artifactsrc, 'R' as sourcetype, f0.name as source0, sd.mtime as mtime, f1.name as source1, fsref.name as source0ref\n"
244 : : " from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files f1, " BUILDIDS "_files fsref, "
245 : : " " BUILDIDS "_r_sdef sd, " BUILDIDS "_r_sref sr, " BUILDIDS "_r_de sde\n"
246 : : " where b.id = sr.buildid and f0.id = sd.file and fsref.id = sde.file and f1.id = sd.content\n"
247 : : " and sr.artifactsrc = sd.content and sde.buildid = sr.buildid\n"
248 : : ";"
249 : : // and for startup overview counts
250 : : "drop view if exists " BUILDIDS "_stats;\n"
251 : : "create view if not exists " BUILDIDS "_stats as\n"
252 : : " select 'file d/e' as label,count(*) as quantity from " BUILDIDS "_f_de\n"
253 : : "union all select 'file s',count(*) from " BUILDIDS "_f_s\n"
254 : : "union all select 'archive d/e',count(*) from " BUILDIDS "_r_de\n"
255 : : "union all select 'archive sref',count(*) from " BUILDIDS "_r_sref\n"
256 : : "union all select 'archive sdef',count(*) from " BUILDIDS "_r_sdef\n"
257 : : "union all select 'buildids',count(*) from " BUILDIDS "_buildids\n"
258 : : "union all select 'filenames',count(*) from " BUILDIDS "_files\n"
259 : : "union all select 'files scanned (#)',count(*) from " BUILDIDS "_file_mtime_scanned\n"
260 : : "union all select 'files scanned (mb)',coalesce(sum(size)/1024/1024,0) from " BUILDIDS "_file_mtime_scanned\n"
261 : : #if SQLITE_VERSION_NUMBER >= 3016000
262 : : "union all select 'index db size (mb)',page_count*page_size/1024/1024 as size FROM pragma_page_count(), pragma_page_size()\n"
263 : : #endif
264 : : ";\n"
265 : :
266 : : // schema change history & garbage collection
267 : : //
268 : : // XXX: we could have migration queries here to bring prior-schema
269 : : // data over instead of just dropping it.
270 : : //
271 : : // buildids9: widen the mtime_scanned table
272 : : "" // <<< we are here
273 : : // buildids8: slim the sref table
274 : : "drop table if exists buildids8_f_de;\n"
275 : : "drop table if exists buildids8_f_s;\n"
276 : : "drop table if exists buildids8_r_de;\n"
277 : : "drop table if exists buildids8_r_sref;\n"
278 : : "drop table if exists buildids8_r_sdef;\n"
279 : : "drop table if exists buildids8_file_mtime_scanned;\n"
280 : : "drop table if exists buildids8_files;\n"
281 : : "drop table if exists buildids8_buildids;\n"
282 : : // buildids7: separate _norm table into dense subtype tables
283 : : "drop table if exists buildids7_f_de;\n"
284 : : "drop table if exists buildids7_f_s;\n"
285 : : "drop table if exists buildids7_r_de;\n"
286 : : "drop table if exists buildids7_r_sref;\n"
287 : : "drop table if exists buildids7_r_sdef;\n"
288 : : "drop table if exists buildids7_file_mtime_scanned;\n"
289 : : "drop table if exists buildids7_files;\n"
290 : : "drop table if exists buildids7_buildids;\n"
291 : : // buildids6: drop bolo/rfolo again, represent sources / rpmcontents in main table
292 : : "drop table if exists buildids6_norm;\n"
293 : : "drop table if exists buildids6_files;\n"
294 : : "drop table if exists buildids6_buildids;\n"
295 : : "drop view if exists buildids6;\n"
296 : : // buildids5: redefine srcfile1 column to be '.'-less (for rpms)
297 : : "drop table if exists buildids5_norm;\n"
298 : : "drop table if exists buildids5_files;\n"
299 : : "drop table if exists buildids5_buildids;\n"
300 : : "drop table if exists buildids5_bolo;\n"
301 : : "drop table if exists buildids5_rfolo;\n"
302 : : "drop view if exists buildids5;\n"
303 : : // buildids4: introduce rpmfile RFOLO
304 : : "drop table if exists buildids4_norm;\n"
305 : : "drop table if exists buildids4_files;\n"
306 : : "drop table if exists buildids4_buildids;\n"
307 : : "drop table if exists buildids4_bolo;\n"
308 : : "drop table if exists buildids4_rfolo;\n"
309 : : "drop view if exists buildids4;\n"
310 : : // buildids3*: split out srcfile BOLO
311 : : "drop table if exists buildids3_norm;\n"
312 : : "drop table if exists buildids3_files;\n"
313 : : "drop table if exists buildids3_buildids;\n"
314 : : "drop table if exists buildids3_bolo;\n"
315 : : "drop view if exists buildids3;\n"
316 : : // buildids2: normalized buildid and filenames into interning tables;
317 : : "drop table if exists buildids2_norm;\n"
318 : : "drop table if exists buildids2_files;\n"
319 : : "drop table if exists buildids2_buildids;\n"
320 : : "drop view if exists buildids2;\n"
321 : : // buildids1: made buildid and artifacttype NULLable, to represent cached-negative
322 : : // lookups from sources, e.g. files or rpms that contain no buildid-indexable content
323 : : "drop table if exists buildids1;\n"
324 : : // buildids: original
325 : : "drop table if exists buildids;\n"
326 : : ;
327 : :
328 : : static const char DEBUGINFOD_SQLITE_CLEANUP_DDL[] =
329 : : "pragma wal_checkpoint = truncate;\n" // clean out any preexisting wal file
330 : : ;
331 : :
332 : :
333 : :
334 : :
335 : : /* Name and version of program. */
336 : : /* ARGP_PROGRAM_VERSION_HOOK_DEF = print_version; */ // not this simple for C++
337 : :
338 : : /* Bug report address. */
339 : : ARGP_PROGRAM_BUG_ADDRESS_DEF = PACKAGE_BUGREPORT;
340 : :
341 : : /* Definitions of arguments for argp functions. */
342 : : static const struct argp_option options[] =
343 : : {
344 : : { NULL, 0, NULL, 0, "Scanners:", 1 },
345 : : { "scan-file-dir", 'F', NULL, 0, "Enable ELF/DWARF file scanning.", 0 },
346 : : { "scan-rpm-dir", 'R', NULL, 0, "Enable RPM scanning.", 0 },
347 : : { "scan-deb-dir", 'U', NULL, 0, "Enable DEB scanning.", 0 },
348 : : { "scan-archive", 'Z', "EXT=CMD", 0, "Enable arbitrary archive scanning.", 0 },
349 : : // "source-oci-imageregistry" ...
350 : :
351 : : { NULL, 0, NULL, 0, "Options:", 2 },
352 : : { "logical", 'L', NULL, 0, "Follow symlinks, default=ignore.", 0 },
353 : : { "rescan-time", 't', "SECONDS", 0, "Number of seconds to wait between rescans, 0=disable.", 0 },
354 : : { "groom-time", 'g', "SECONDS", 0, "Number of seconds to wait between database grooming, 0=disable.", 0 },
355 : : { "maxigroom", 'G', NULL, 0, "Run a complete database groom/shrink pass at startup.", 0 },
356 : : { "concurrency", 'c', "NUM", 0, "Limit scanning thread concurrency to NUM.", 0 },
357 : : { "include", 'I', "REGEX", 0, "Include files matching REGEX, default=all.", 0 },
358 : : { "exclude", 'X', "REGEX", 0, "Exclude files matching REGEX, default=none.", 0 },
359 : : { "port", 'p', "NUM", 0, "HTTP port to listen on, default 8002.", 0 },
360 : : { "database", 'd', "FILE", 0, "Path to sqlite database.", 0 },
361 : : { "ddl", 'D', "SQL", 0, "Apply extra sqlite ddl/pragma to connection.", 0 },
362 : : { "verbose", 'v', NULL, 0, "Increase verbosity.", 0 },
363 : : #define ARGP_KEY_FDCACHE_FDS 0x1001
364 : : { "fdcache-fds", ARGP_KEY_FDCACHE_FDS, "NUM", 0, "Maximum number of archive files to keep in fdcache.", 0 },
365 : : #define ARGP_KEY_FDCACHE_MBS 0x1002
366 : : { "fdcache-mbs", ARGP_KEY_FDCACHE_MBS, "MB", 0, "Maximum total size of archive file fdcache.", 0 },
367 : : #define ARGP_KEY_FDCACHE_PREFETCH 0x1003
368 : : { "fdcache-prefetch", ARGP_KEY_FDCACHE_PREFETCH, "NUM", 0, "Number of archive files to prefetch into fdcache.", 0 },
369 : : #define ARGP_KEY_FDCACHE_MINTMP 0x1004
370 : : { "fdcache-mintmp", ARGP_KEY_FDCACHE_MINTMP, "NUM", 0, "Minimum free space% on tmpdir.", 0 },
371 : : { NULL, 0, NULL, 0, NULL, 0 }
372 : : };
373 : :
374 : : /* Short description of program. */
375 : : static const char doc[] = "Serve debuginfo-related content across HTTP from files under PATHs.";
376 : :
377 : : /* Strings for arguments in help texts. */
378 : : static const char args_doc[] = "[PATH ...]";
379 : :
380 : : /* Prototype for option handler. */
381 : : static error_t parse_opt (int key, char *arg, struct argp_state *state);
382 : :
383 : : /* Data structure to communicate with argp functions. */
384 : : static struct argp argp =
385 : : {
386 : : options, parse_opt, args_doc, doc, NULL, NULL, NULL
387 : : };
388 : :
389 : :
390 : : static string db_path;
391 : : static sqlite3 *db; // single connection, serialized across all our threads!
392 : : static sqlite3 *dbq; // webapi query-servicing readonly connection, serialized ditto!
393 : : static unsigned verbose;
394 : : static volatile sig_atomic_t interrupted = 0;
395 : : static volatile sig_atomic_t forced_rescan_count = 0;
396 : : static volatile sig_atomic_t sigusr1 = 0;
397 : : static volatile sig_atomic_t forced_groom_count = 0;
398 : : static volatile sig_atomic_t sigusr2 = 0;
399 : : static unsigned http_port = 8002;
400 : : static unsigned rescan_s = 300;
401 : : static unsigned groom_s = 86400;
402 : : static bool maxigroom = false;
403 : : static unsigned concurrency = std::thread::hardware_concurrency() ?: 1;
404 : : static set<string> source_paths;
405 : : static bool scan_files = false;
406 : : static map<string,string> scan_archives;
407 : : static vector<string> extra_ddl;
408 : : static regex_t file_include_regex;
409 : : static regex_t file_exclude_regex;
410 : : static bool traverse_logical;
411 : : static long fdcache_fds;
412 : : static long fdcache_mbs;
413 : : static long fdcache_prefetch;
414 : : static long fdcache_mintmp;
415 : : static string tmpdir;
416 : :
417 : : static void set_metric(const string& key, double value);
418 : : // static void inc_metric(const string& key);
419 : : static void set_metric(const string& metric,
420 : : const string& lname, const string& lvalue,
421 : : double value);
422 : : static void inc_metric(const string& metric,
423 : : const string& lname, const string& lvalue);
424 : : static void add_metric(const string& metric,
425 : : const string& lname, const string& lvalue,
426 : : double value);
427 : : // static void add_metric(const string& metric, double value);
428 : :
429 : : class tmp_inc_metric { // a RAII style wrapper for exception-safe scoped increment & decrement
430 : : string m, n, v;
431 : : public:
432 : 153 : tmp_inc_metric(const string& mname, const string& lname, const string& lvalue):
433 [ + - + - ]: 153 : m(mname), n(lname), v(lvalue)
434 : : {
435 [ + - ]: 153 : add_metric (m, n, v, 1);
436 : 153 : }
437 : 153 : ~tmp_inc_metric()
438 : 153 : {
439 : 153 : add_metric (m, n, v, -1);
440 : 153 : }
441 : : };
442 : :
443 : : class tmp_ms_metric { // a RAII style wrapper for exception-safe scoped timing
444 : : string m, n, v;
445 : : struct timespec ts_start;
446 : : public:
447 : 5182 : tmp_ms_metric(const string& mname, const string& lname, const string& lvalue):
448 [ + - + - ]: 5182 : m(mname), n(lname), v(lvalue)
449 : : {
450 : 5181 : clock_gettime (CLOCK_MONOTONIC, & ts_start);
451 : 5181 : }
452 : 5182 : ~tmp_ms_metric()
453 : 5181 : {
454 : : struct timespec ts_end;
455 : 5182 : clock_gettime (CLOCK_MONOTONIC, & ts_end);
456 : 5182 : double deltas = (ts_end.tv_sec - ts_start.tv_sec)
457 : 5182 : + (ts_end.tv_nsec - ts_start.tv_nsec)/1.e9;
458 : :
459 : 5182 : add_metric (m + "_milliseconds_sum", n, v, (deltas*1000.0));
460 : 5182 : inc_metric (m + "_milliseconds_count", n, v);
461 : 5182 : }
462 : : };
463 : :
464 : :
465 : : /* Handle program arguments. */
466 : : static error_t
467 : 51 : parse_opt (int key, char *arg,
468 : : struct argp_state *state __attribute__ ((unused)))
469 : : {
470 : : int rc;
471 [ + + + + : 51 : switch (key)
+ + + + -
+ + - - -
- + + - +
+ + ]
472 : : {
473 : 10 : case 'v': verbose ++; break;
474 [ + - ]: 3 : case 'd': db_path = string(arg); break;
475 : 3 : case 'p': http_port = (unsigned) atoi(arg);
476 [ + - - + ]: 3 : if (http_port == 0 || http_port > 65535)
477 : 0 : argp_failure(state, 1, EINVAL, "port number");
478 : 3 : break;
479 : 4 : case 'F': scan_files = true; break;
480 : 1 : case 'R':
481 [ + - + - : 1 : scan_archives[".rpm"]="cat"; // libarchive groks rpm natively
+ - ]
482 : 1 : break;
483 : 2 : case 'U':
484 [ + - + - : 2 : scan_archives[".deb"]="(bsdtar -O -x -f - data.tar\\*)<";
+ - ]
485 [ + - + - : 2 : scan_archives[".ddeb"]="(bsdtar -O -x -f - data.tar\\*)<";
+ - ]
486 [ + - + - : 2 : scan_archives[".ipk"]="(bsdtar -O -x -f - data.tar\\*)<";
+ - ]
487 : : // .udeb too?
488 : 2 : break;
489 : 2 : case 'Z':
490 : : {
491 : 2 : char* extension = strchr(arg, '=');
492 [ - + ]: 2 : if (arg[0] == '\0')
493 : 0 : argp_failure(state, 1, EINVAL, "missing EXT");
494 [ + + ]: 2 : else if (extension)
495 [ + - + - : 1 : scan_archives[string(arg, (extension-arg))]=string(extension+1);
+ - ]
496 : : else
497 [ + - + - : 1 : scan_archives[string(arg)]=string("cat");
+ - ]
498 : : }
499 : 2 : break;
500 : 2 : case 'L':
501 : 2 : traverse_logical = true;
502 : 2 : break;
503 [ # # # # ]: 0 : case 'D': extra_ddl.push_back(string(arg)); break;
504 : 1 : case 't':
505 : 1 : rescan_s = (unsigned) atoi(arg);
506 : 1 : break;
507 : 1 : case 'g':
508 : 1 : groom_s = (unsigned) atoi(arg);
509 : 1 : break;
510 : 0 : case 'G':
511 : 0 : maxigroom = true;
512 : 0 : break;
513 : 0 : case 'c':
514 : 0 : concurrency = (unsigned) atoi(arg);
515 [ # # ]: 0 : if (concurrency < 1) concurrency = 1;
516 : 0 : break;
517 : 0 : case 'I':
518 : : // NB: no problem with unconditional free here - an earlier failed regcomp would exit program
519 : 0 : regfree (&file_include_regex);
520 : 0 : rc = regcomp (&file_include_regex, arg, REG_EXTENDED|REG_NOSUB);
521 [ # # ]: 0 : if (rc != 0)
522 : 0 : argp_failure(state, 1, EINVAL, "regular expression");
523 : 0 : break;
524 : 0 : case 'X':
525 : 0 : regfree (&file_exclude_regex);
526 : 0 : rc = regcomp (&file_exclude_regex, arg, REG_EXTENDED|REG_NOSUB);
527 [ # # ]: 0 : if (rc != 0)
528 : 0 : argp_failure(state, 1, EINVAL, "regular expression");
529 : 0 : break;
530 : 1 : case ARGP_KEY_FDCACHE_FDS:
531 : 1 : fdcache_fds = atol (arg);
532 : 1 : break;
533 : 1 : case ARGP_KEY_FDCACHE_MBS:
534 : 1 : fdcache_mbs = atol (arg);
535 : 1 : break;
536 : 0 : case ARGP_KEY_FDCACHE_PREFETCH:
537 : 0 : fdcache_prefetch = atol (arg);
538 : 0 : break;
539 : 1 : case ARGP_KEY_FDCACHE_MINTMP:
540 : 1 : fdcache_mintmp = atol (arg);
541 : 1 : break;
542 : 6 : case ARGP_KEY_ARG:
543 [ + - + - ]: 6 : source_paths.insert(string(arg));
544 : 6 : break;
545 : : // case 'h': argp_state_help (state, stderr, ARGP_HELP_LONG|ARGP_HELP_EXIT_OK);
546 : 13 : default: return ARGP_ERR_UNKNOWN;
547 : : }
548 : :
549 : 38 : return 0;
550 : : }
551 : :
552 : :
553 : : ////////////////////////////////////////////////////////////////////////
554 : :
555 : :
556 : : // represent errors that may get reported to an ostream and/or a libmicrohttpd connection
557 : :
558 : : struct reportable_exception
559 : : {
560 : : int code;
561 : : string message;
562 : :
563 : 15 : reportable_exception(int c, const string& m): code(c), message(m) {}
564 : 35 : reportable_exception(const string& m): code(503), message(m) {}
565 : : reportable_exception(): code(503), message() {}
566 : :
567 : : void report(ostream& o) const; // defined under obatched() class below
568 : :
569 : 19 : MHD_RESULT mhd_send_response(MHD_Connection* c) const {
570 : 38 : MHD_Response* r = MHD_create_response_from_buffer (message.size(),
571 : 19 : (void*) message.c_str(),
572 : : MHD_RESPMEM_MUST_COPY);
573 : 19 : MHD_add_response_header (r, "Content-Type", "text/plain");
574 : 19 : MHD_RESULT rc = MHD_queue_response (c, code, r);
575 : 19 : MHD_destroy_response (r);
576 : 19 : return rc;
577 : : }
578 : : };
579 : :
580 : :
581 : : struct sqlite_exception: public reportable_exception
582 : : {
583 : 2 : sqlite_exception(int rc, const string& msg):
584 [ + - + - : 2 : reportable_exception(string("sqlite3 error: ") + msg + ": " + string(sqlite3_errstr(rc) ?: "?")) {
+ - + - +
- + - + -
+ - ]
585 [ + - + - : 2 : inc_metric("error_count","sqlite3",sqlite3_errstr(rc));
+ - + - +
- ]
586 : 2 : }
587 : : };
588 : :
589 : : struct libc_exception: public reportable_exception
590 : : {
591 : 30 : libc_exception(int rc, const string& msg):
592 [ + - + - : 30 : reportable_exception(string("libc error: ") + msg + ": " + string(strerror(rc) ?: "?")) {
+ - + - +
- + - +
- ]
593 [ + - + - : 30 : inc_metric("error_count","libc",strerror(rc));
+ - + - ]
594 : 30 : }
595 : : };
596 : :
597 : :
598 : : struct archive_exception: public reportable_exception
599 : : {
600 : 0 : archive_exception(const string& msg):
601 [ # # # # : 0 : reportable_exception(string("libarchive error: ") + msg) {
# # ]
602 [ # # # # : 0 : inc_metric("error_count","libarchive",msg);
# # ]
603 : 0 : }
604 : 0 : archive_exception(struct archive* a, const string& msg):
605 [ # # # # : 0 : reportable_exception(string("libarchive error: ") + msg + ": " + string(archive_error_string(a) ?: "?")) {
# # # # #
# # # # #
# # ]
606 [ # # # # : 0 : inc_metric("error_count","libarchive",msg + ": " + string(archive_error_string(a) ?: "?"));
# # # # #
# # # # #
# # ]
607 : 0 : }
608 : : };
609 : :
610 : :
611 : : struct elfutils_exception: public reportable_exception
612 : : {
613 : 0 : elfutils_exception(int rc, const string& msg):
614 [ # # # # : 0 : reportable_exception(string("elfutils error: ") + msg + ": " + string(elf_errmsg(rc) ?: "?")) {
# # # # #
# # # # #
# # ]
615 [ # # # # : 0 : inc_metric("error_count","elfutils",elf_errmsg(rc));
# # # # #
# ]
616 : 0 : }
617 : : };
618 : :
619 : :
620 : : ////////////////////////////////////////////////////////////////////////
621 : :
622 : : template <typename Payload>
623 : : class workq
624 : : {
625 : : set<Payload> q; // eliminate duplicates
626 : : mutex mtx;
627 : : condition_variable cv;
628 : : bool dead;
629 : : unsigned idlers;
630 : :
631 : : public:
632 : 3 : workq() { dead = false; idlers = 0; }
633 : 3 : ~workq() {}
634 : :
635 : 173 : void push_back(const Payload& p)
636 : : {
637 [ + - ]: 346 : unique_lock<mutex> lock(mtx);
638 [ + - ]: 173 : q.insert (p);
639 [ + - + - : 173 : set_metric("thread_work_pending","role","scan", q.size());
+ - + - ]
640 : 173 : cv.notify_all();
641 : 173 : }
642 : :
643 : : // kill this workqueue, wake up all idlers / scanners
644 : 3 : void nuke() {
645 [ + - ]: 6 : unique_lock<mutex> lock(mtx);
646 : : // optional: q.clear();
647 : 3 : dead = true;
648 : 3 : cv.notify_all();
649 : 3 : }
650 : :
651 : : // clear the workqueue, when scanning is interrupted with USR2
652 : 0 : void clear() {
653 [ # # ]: 0 : unique_lock<mutex> lock(mtx);
654 : 0 : q.clear();
655 [ # # # # : 0 : set_metric("thread_work_pending","role","scan", q.size());
# # # # ]
656 : 0 : cv.notify_all(); // maybe wake up waiting idlers
657 : 0 : }
658 : :
659 : : // block this scanner thread until there is work to do and no active
660 : 185 : bool wait_front (Payload& p)
661 : : {
662 [ + - ]: 370 : unique_lock<mutex> lock(mtx);
663 [ + + + + : 682 : while (!dead && (q.size() == 0 || idlers > 0))
+ + + + ]
664 : 497 : cv.wait(lock);
665 [ + + ]: 185 : if (dead)
666 : 12 : return false;
667 : : else
668 : : {
669 [ + - ]: 173 : p = * q.begin();
670 [ + - ]: 173 : q.erase (q.begin());
671 [ + - + - : 173 : set_metric("thread_work_pending","role","scan", q.size());
+ - + - ]
672 [ + + ]: 173 : if (q.size() == 0)
673 : 93 : cv.notify_all(); // maybe wake up waiting idlers
674 : 173 : return true;
675 : : }
676 : : }
677 : :
678 : : // block this idler thread until there is no work to do
679 : 70 : void wait_idle ()
680 : : {
681 [ + - ]: 70 : unique_lock<mutex> lock(mtx);
682 : 70 : cv.notify_all(); // maybe wake up waiting scanners
683 [ + + + + : 75 : while (!dead && (q.size() != 0))
+ + ]
684 : 5 : cv.wait(lock);
685 : 70 : idlers ++;
686 : 70 : }
687 : :
688 : 67 : void done_idle ()
689 : : {
690 [ + - ]: 134 : unique_lock<mutex> lock(mtx);
691 : 67 : idlers --;
692 : 67 : cv.notify_all(); // maybe wake up waiting scanners, but probably not (shutting down)
693 : 67 : }
694 : : };
695 : :
696 : : typedef struct stat stat_t;
697 : : typedef pair<string,stat_t> scan_payload;
698 : 361 : inline bool operator< (const scan_payload& a, const scan_payload& b)
699 : : {
700 : 361 : return a.first < b.first; // don't bother compare the stat fields
701 : : }
702 : : static workq<scan_payload> scanq; // just a single one
703 : : // producer & idler: thread_main_fts_source_paths()
704 : : // consumer: thread_main_scanner()
705 : : // idler: thread_main_groom()
706 : :
707 : :
708 : :
709 : : ////////////////////////////////////////////////////////////////////////
710 : :
711 : :
712 : : // Print a standard timestamp.
713 : : static ostream&
714 : 1767 : timestamp (ostream &o)
715 : : {
716 : : char datebuf[80];
717 : 1767 : char *now2 = NULL;
718 : 1767 : time_t now_t = time(NULL);
719 : 1766 : struct tm *now = gmtime (&now_t);
720 [ + - ]: 1767 : if (now)
721 : : {
722 : 1767 : (void) strftime (datebuf, sizeof (datebuf), "%c", now);
723 : 1767 : now2 = datebuf;
724 : : }
725 : :
726 : : return o << "[" << (now2 ? now2 : "") << "] "
727 [ + - + - : 1767 : << "(" << getpid () << "/" << tid() << "): ";
+ - + - +
- + - + -
+ - + - ]
728 : : }
729 : :
730 : :
731 : : // A little class that impersonates an ostream to the extent that it can
732 : : // take << streaming operations. It batches up the bits into an internal
733 : : // stringstream until it is destroyed; then flushes to the original ostream.
734 : : // It adds a timestamp
735 : : class obatched
736 : : {
737 : : private:
738 : : ostream& o;
739 : : stringstream stro;
740 : : static mutex lock;
741 : : public:
742 : 1767 : obatched(ostream& oo, bool timestamp_p = true): o(oo)
743 : : {
744 [ + - ]: 1767 : if (timestamp_p)
745 [ + - ]: 1767 : timestamp(stro);
746 : 1767 : }
747 : 1767 : ~obatched()
748 : 1767 : {
749 : 3534 : unique_lock<mutex> do_not_cross_the_streams(obatched::lock);
750 : 1767 : o << stro.str();
751 : 1767 : o.flush();
752 : 1767 : }
753 : : operator ostream& () { return stro; }
754 : 1767 : template <typename T> ostream& operator << (const T& t) { stro << t; return stro; }
755 : : };
756 : : mutex obatched::lock; // just the one, since cout/cerr iostreams are not thread-safe
757 : :
758 : :
759 : 49 : void reportable_exception::report(ostream& o) const {
760 [ + - + - ]: 49 : obatched(o) << message << endl;
761 : 49 : }
762 : :
763 : :
764 : : ////////////////////////////////////////////////////////////////////////
765 : :
766 : :
767 : : // RAII style sqlite prepared-statement holder that matches { } block lifetime
768 : :
769 : : struct sqlite_ps
770 : : {
771 : : private:
772 : : sqlite3* db;
773 : : const string nickname;
774 : : const string sql;
775 : : sqlite3_stmt *pp;
776 : :
777 : : sqlite_ps(const sqlite_ps&); // make uncopyable
778 : : sqlite_ps& operator=(const sqlite_ps &); // make unassignable
779 : :
780 : : public:
781 [ + - ]: 288 : sqlite_ps (sqlite3* d, const string& n, const string& s): db(d), nickname(n), sql(s) {
782 : : // tmp_ms_metric tick("sqlite3","prep",nickname);
783 [ - + ]: 288 : if (verbose > 4)
784 [ # # # # : 0 : obatched(clog) << nickname << " prep " << sql << endl;
# # # # #
# ]
785 [ + - ]: 288 : int rc = sqlite3_prepare_v2 (db, sql.c_str(), -1 /* to \0 */, & this->pp, NULL);
786 [ - + ]: 288 : if (rc != SQLITE_OK)
787 [ # # # # ]: 0 : throw sqlite_exception(rc, "prepare " + sql);
788 : 288 : }
789 : :
790 : 2631 : sqlite_ps& reset()
791 : : {
792 [ + - + - : 7893 : tmp_ms_metric tick("sqlite3","reset",nickname);
+ - ]
793 [ + - ]: 2631 : sqlite3_reset(this->pp);
794 : 5262 : return *this;
795 : : }
796 : :
797 : 3381 : sqlite_ps& bind(int parameter, const string& str)
798 : : {
799 [ - + ]: 3381 : if (verbose > 4)
800 [ # # # # : 0 : obatched(clog) << nickname << " bind " << parameter << "=" << str << endl;
# # # # #
# # # ]
801 : 3381 : int rc = sqlite3_bind_text (this->pp, parameter, str.c_str(), -1, SQLITE_TRANSIENT);
802 [ - + ]: 3381 : if (rc != SQLITE_OK)
803 [ # # # # ]: 0 : throw sqlite_exception(rc, "sqlite3 bind");
804 : 3381 : return *this;
805 : : }
806 : :
807 : 1184 : sqlite_ps& bind(int parameter, int64_t value)
808 : : {
809 [ - + ]: 1184 : if (verbose > 4)
810 [ # # # # : 0 : obatched(clog) << nickname << " bind " << parameter << "=" << value << endl;
# # # # #
# # # ]
811 : 1184 : int rc = sqlite3_bind_int64 (this->pp, parameter, value);
812 [ - + ]: 1184 : if (rc != SQLITE_OK)
813 [ # # # # ]: 0 : throw sqlite_exception(rc, "sqlite3 bind");
814 : 1184 : return *this;
815 : : }
816 : :
817 : : sqlite_ps& bind(int parameter)
818 : : {
819 : : if (verbose > 4)
820 : : obatched(clog) << nickname << " bind " << parameter << "=" << "NULL" << endl;
821 : : int rc = sqlite3_bind_null (this->pp, parameter);
822 : : if (rc != SQLITE_OK)
823 : : throw sqlite_exception(rc, "sqlite3 bind");
824 : : return *this;
825 : : }
826 : :
827 : :
828 : 1941 : void step_ok_done() {
829 [ + - + - : 7764 : tmp_ms_metric tick("sqlite3","step_done",nickname);
+ - ]
830 [ + - ]: 1941 : int rc = sqlite3_step (this->pp);
831 [ - + ]: 1941 : if (verbose > 4)
832 [ # # # # : 0 : obatched(clog) << nickname << " step-ok-done(" << sqlite3_errstr(rc) << ") " << sql << endl;
# # # # #
# # # # #
# # ]
833 [ + - + + : 1941 : if (rc != SQLITE_OK && rc != SQLITE_DONE && rc != SQLITE_ROW)
+ + ]
834 [ + - + - ]: 1 : throw sqlite_exception(rc, "sqlite3 step");
835 [ + - ]: 1940 : (void) sqlite3_reset (this->pp);
836 : 1940 : }
837 : :
838 : :
839 : 610 : int step() {
840 [ + - + - : 1830 : tmp_ms_metric tick("sqlite3","step",nickname);
+ - ]
841 [ + - ]: 610 : int rc = sqlite3_step (this->pp);
842 [ - + ]: 610 : if (verbose > 4)
843 [ # # # # : 0 : obatched(clog) << nickname << " step(" << sqlite3_errstr(rc) << ") " << sql << endl;
# # # # #
# # # # #
# # ]
844 : 1220 : return rc;
845 : : }
846 : :
847 : 288 : ~sqlite_ps () { sqlite3_finalize (this->pp); }
848 : 1016 : operator sqlite3_stmt* () { return this->pp; }
849 : : };
850 : :
851 : :
852 : : ////////////////////////////////////////////////////////////////////////
853 : :
854 : : // RAII style templated autocloser
855 : :
856 : : template <class Payload, class Ignore>
857 : : struct defer_dtor
858 : : {
859 : : public:
860 : : typedef Ignore (*dtor_fn) (Payload);
861 : :
862 : : private:
863 : : Payload p;
864 : : dtor_fn fn;
865 : :
866 : : public:
867 : 342 : defer_dtor(Payload _p, dtor_fn _fn): p(_p), fn(_fn) {}
868 : 342 : ~defer_dtor() { (void) (*fn)(p); }
869 : :
870 : : private:
871 : : defer_dtor(const defer_dtor<Payload,Ignore>&); // make uncopyable
872 : : defer_dtor& operator=(const defer_dtor<Payload,Ignore> &); // make unassignable
873 : : };
874 : :
875 : :
876 : :
877 : : ////////////////////////////////////////////////////////////////////////
878 : :
879 : :
880 : : static string
881 : 310 : header_censor(const string& str)
882 : : {
883 : 310 : string y;
884 [ + + ]: 3861 : for (auto&& x : str)
885 : : {
886 [ + + + + : 3551 : if (isalnum(x) || x == '/' || x == '.' || x == ',' || x == '_' || x == ':')
+ + + + +
- - + ]
887 [ + - ]: 3550 : y += x;
888 : : }
889 : 310 : return y;
890 : : }
891 : :
892 : :
893 : : static string
894 : 155 : conninfo (struct MHD_Connection * conn)
895 : : {
896 : : char hostname[256]; // RFC1035
897 : : char servname[256];
898 : 155 : int sts = -1;
899 : :
900 [ - + ]: 155 : if (conn == 0)
901 [ # # ]: 0 : return "internal";
902 : :
903 : : /* Look up client address data. */
904 [ + - ]: 155 : const union MHD_ConnectionInfo *u = MHD_get_connection_info (conn,
905 : : MHD_CONNECTION_INFO_CLIENT_ADDRESS);
906 [ + - ]: 155 : struct sockaddr *so = u ? u->client_addr : 0;
907 : :
908 [ + - + - ]: 155 : if (so && so->sa_family == AF_INET) {
909 [ + - ]: 155 : sts = getnameinfo (so, sizeof (struct sockaddr_in), hostname, sizeof (hostname), servname,
910 : : sizeof (servname), NI_NUMERICHOST | NI_NUMERICSERV);
911 [ # # # # ]: 0 : } else if (so && so->sa_family == AF_INET6) {
912 [ # # ]: 0 : sts = getnameinfo (so, sizeof (struct sockaddr_in6), hostname, sizeof (hostname),
913 : : servname, sizeof (servname), NI_NUMERICHOST | NI_NUMERICSERV);
914 : : }
915 [ - + ]: 155 : if (sts != 0) {
916 : 0 : hostname[0] = servname[0] = '\0';
917 : : }
918 : :
919 : : // extract headers relevant to administration
920 [ + - + - ]: 155 : const char* user_agent = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "User-Agent") ?: "";
921 [ + - + + ]: 155 : const char* x_forwarded_for = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "X-Forwarded-For") ?: "";
922 : : // NB: these are untrustworthy, beware if machine-processing log files
923 : :
924 [ + - + - : 465 : return string(hostname) + string(":") + string(servname) +
+ - + - +
- ]
925 [ + - + - : 620 : string(" UA:") + header_censor(string(user_agent)) +
+ - + - +
- ]
926 [ + - + - : 465 : string(" XFF:") + header_censor(string(x_forwarded_for));
+ - + - +
- ]
927 : : }
928 : :
929 : :
930 : :
931 : : ////////////////////////////////////////////////////////////////////////
932 : :
933 : :
934 : : static void
935 : 64 : add_mhd_last_modified (struct MHD_Response *resp, time_t mtime)
936 : : {
937 : 64 : struct tm *now = gmtime (&mtime);
938 [ + - ]: 64 : if (now != NULL)
939 : : {
940 : : char datebuf[80];
941 : 64 : size_t rc = strftime (datebuf, sizeof (datebuf), "%a, %d %b %Y %T GMT", now);
942 [ + - + - ]: 64 : if (rc > 0 && rc < sizeof (datebuf))
943 [ + - ]: 64 : (void) MHD_add_response_header (resp, "Last-Modified", datebuf);
944 : : }
945 : :
946 : 64 : (void) MHD_add_response_header (resp, "Cache-Control", "public");
947 : 64 : }
948 : :
949 : :
950 : :
951 : : static struct MHD_Response*
952 : 18 : handle_buildid_f_match (bool internal_req_t,
953 : : int64_t b_mtime,
954 : : const string& b_source0,
955 : : int *result_fd)
956 : : {
957 : : (void) internal_req_t; // ignored
958 [ + - ]: 18 : int fd = open(b_source0.c_str(), O_RDONLY);
959 [ - + ]: 18 : if (fd < 0)
960 [ # # # # : 0 : throw libc_exception (errno, string("open ") + b_source0);
# # ]
961 : :
962 : : // NB: use manual close(2) in error case instead of defer_dtor, because
963 : : // in the normal case, we want to hand the fd over to libmicrohttpd for
964 : : // file transfer.
965 : :
966 : : struct stat s;
967 : 18 : int rc = fstat(fd, &s);
968 [ - + ]: 18 : if (rc < 0)
969 : : {
970 [ # # ]: 0 : close(fd);
971 [ # # # # : 0 : throw libc_exception (errno, string("fstat ") + b_source0);
# # ]
972 : : }
973 : :
974 [ - + ]: 18 : if ((int64_t) s.st_mtime != b_mtime)
975 : : {
976 [ # # ]: 0 : if (verbose)
977 [ # # # # : 0 : obatched(clog) << "mtime mismatch for " << b_source0 << endl;
# # # # ]
978 [ # # ]: 0 : close(fd);
979 : 0 : return 0;
980 : : }
981 : :
982 [ + - + - : 18 : inc_metric ("http_responses_total","result","file");
+ - + - ]
983 [ + - ]: 18 : struct MHD_Response* r = MHD_create_response_from_fd ((uint64_t) s.st_size, fd);
984 [ - + ]: 18 : if (r == 0)
985 : : {
986 [ # # ]: 0 : if (verbose)
987 [ # # # # : 0 : obatched(clog) << "cannot create fd-response for " << b_source0 << endl;
# # # # ]
988 [ # # ]: 0 : close(fd);
989 : : }
990 : : else
991 : : {
992 [ + - ]: 18 : MHD_add_response_header (r, "Content-Type", "application/octet-stream");
993 [ + - ]: 18 : add_mhd_last_modified (r, s.st_mtime);
994 [ + - ]: 18 : if (verbose > 1)
995 [ + - + - : 18 : obatched(clog) << "serving file " << b_source0 << endl;
+ - + - ]
996 : : /* libmicrohttpd will close it. */
997 [ + - ]: 18 : if (result_fd)
998 : 18 : *result_fd = fd;
999 : : }
1000 : :
1001 : 18 : return r;
1002 : : }
1003 : :
1004 : :
1005 : : // quote all questionable characters of str for safe passage through a sh -c expansion.
1006 : : static string
1007 : 6 : shell_escape(const string& str)
1008 : : {
1009 : 6 : string y;
1010 [ + + ]: 508 : for (auto&& x : str)
1011 : : {
1012 [ + + + + ]: 502 : if (! isalnum(x) && x != '/')
1013 [ + - ]: 52 : y += "\\";
1014 [ + - ]: 502 : y += x;
1015 : : }
1016 : 6 : return y;
1017 : : }
1018 : :
1019 : :
1020 : : // PR25548: Perform POSIX / RFC3986 style path canonicalization on the input string.
1021 : : //
1022 : : // Namely:
1023 : : // // -> /
1024 : : // /foo/../ -> /
1025 : : // /./ -> /
1026 : : //
1027 : : // This mapping is done on dwarf-side source path names, which may
1028 : : // include these constructs, so we can deal with debuginfod clients
1029 : : // that accidentally canonicalize the paths.
1030 : : //
1031 : : // realpath(3) is close but not quite right, because it also resolves
1032 : : // symbolic links. Symlinks at the debuginfod server have nothing to
1033 : : // do with the build-time symlinks, thus they must not be considered.
1034 : : //
1035 : : // see also curl Curl_dedotdotify() aka RFC3986, which we mostly follow here
1036 : : // see also libc __realpath()
1037 : : // see also llvm llvm::sys::path::remove_dots()
1038 : : static string
1039 : 537 : canon_pathname (const string& input)
1040 : : {
1041 [ + - ]: 1074 : string i = input; // 5.2.4 (1)
1042 : 537 : string o;
1043 : :
1044 [ + + ]: 3416 : while (i.size() != 0)
1045 : : {
1046 : : // 5.2.4 (2) A
1047 [ + - - + ]: 2879 : if (i.substr(0,3) == "../")
1048 [ # # ]: 0 : i = i.substr(3);
1049 [ + - - + ]: 2879 : else if(i.substr(0,2) == "./")
1050 [ # # ]: 0 : i = i.substr(2);
1051 : :
1052 : : // 5.2.4 (2) B
1053 [ + - + + ]: 2879 : else if (i.substr(0,3) == "/./")
1054 [ + - ]: 55 : i = i.substr(2);
1055 [ - + ]: 2824 : else if (i == "/.")
1056 [ # # ]: 0 : i = ""; // no need to handle "/." complete-path-segment case; we're dealing with file names
1057 : :
1058 : : // 5.2.4 (2) C
1059 [ + - + + ]: 2824 : else if (i.substr(0,4) == "/../") {
1060 [ + - ]: 81 : i = i.substr(3);
1061 [ + - ]: 81 : string::size_type sl = o.rfind("/");
1062 [ + - ]: 81 : if (sl != string::npos)
1063 [ + - ]: 81 : o = o.substr(0, sl);
1064 : : else
1065 [ # # ]: 0 : o = "";
1066 [ - + ]: 2743 : } else if (i == "/..")
1067 [ # # ]: 0 : i = ""; // no need to handle "/.." complete-path-segment case; we're dealing with file names
1068 : :
1069 : : // 5.2.4 (2) D
1070 : : // no need to handle these cases; we're dealing with file names
1071 [ - + ]: 2743 : else if (i == ".")
1072 [ # # ]: 0 : i = "";
1073 [ - + ]: 2743 : else if (i == "..")
1074 [ # # ]: 0 : i = "";
1075 : :
1076 : : // POSIX special: map // to /
1077 [ + - + + ]: 2743 : else if (i.substr(0,2) == "//")
1078 [ + - ]: 14 : i = i.substr(1);
1079 : :
1080 : : // 5.2.4 (2) E
1081 : : else {
1082 [ + - + - ]: 2729 : string::size_type next_slash = i.find("/", (i[0]=='/' ? 1 : 0)); // skip first slash
1083 [ + - + - ]: 2729 : o += i.substr(0, next_slash);
1084 [ + + ]: 2729 : if (next_slash == string::npos)
1085 [ + - ]: 537 : i = "";
1086 : : else
1087 [ + - ]: 2192 : i = i.substr(next_slash);
1088 : : }
1089 : : }
1090 : :
1091 : 1074 : return o;
1092 : : }
1093 : :
1094 : :
1095 : : // Estimate available free space for a given filesystem via statfs(2).
1096 : : // Return true if the free fraction is known to be smaller than the
1097 : : // given minimum percentage. Also update a related metric.
1098 : 449 : bool statfs_free_enough_p(const string& path, const string& label, long minfree = 0)
1099 : : {
1100 : : struct statfs sfs;
1101 : 449 : int rc = statfs(path.c_str(), &sfs);
1102 [ + + ]: 449 : if (rc == 0)
1103 : : {
1104 : 448 : double s = (double) sfs.f_bavail / (double) sfs.f_blocks;
1105 [ + - + - : 448 : set_metric("filesys_free_ratio","purpose",label, s);
+ - ]
1106 : 448 : return ((s * 100.0) < minfree);
1107 : : }
1108 : 1 : return false;
1109 : : }
1110 : :
1111 : :
1112 : :
1113 : : // A map-like class that owns a cache of file descriptors (indexed by
1114 : : // file / content names).
1115 : : //
1116 : : // If only it could use fd's instead of file names ... but we can't
1117 : : // dup(2) to create independent descriptors for the same unlinked
1118 : : // files, so would have to use some goofy linux /proc/self/fd/%d
1119 : : // hack such as the following
1120 : :
1121 : : #if 0
1122 : : int superdup(int fd)
1123 : : {
1124 : : #ifdef __linux__
1125 : : char *fdpath = NULL;
1126 : : int rc = asprintf(& fdpath, "/proc/self/fd/%d", fd);
1127 : : int newfd;
1128 : : if (rc >= 0)
1129 : : newfd = open(fdpath, O_RDONLY);
1130 : : else
1131 : : newfd = -1;
1132 : : free (fdpath);
1133 : : return newfd;
1134 : : #else
1135 : : return -1;
1136 : : #endif
1137 : : }
1138 : : #endif
1139 : :
1140 : : class libarchive_fdcache
1141 : : {
1142 : : private:
1143 : : mutex fdcache_lock;
1144 : :
1145 : : struct fdcache_entry
1146 : : {
1147 : : string archive;
1148 : : string entry;
1149 : : string fd;
1150 : : double fd_size_mb; // slightly rounded up megabytes
1151 : : };
1152 : : deque<fdcache_entry> lru; // @head: most recently used
1153 : : long max_fds;
1154 : : long max_mbs;
1155 : :
1156 : : public:
1157 : 111 : void set_metrics()
1158 : : {
1159 : 111 : double total_mb = 0.0;
1160 [ + + ]: 280 : for (auto i = lru.begin(); i < lru.end(); i++)
1161 : 169 : total_mb += i->fd_size_mb;
1162 [ + - + - ]: 111 : set_metric("fdcache_bytes", (int64_t)(total_mb*1024.0*1024.0));
1163 [ + - + - ]: 111 : set_metric("fdcache_count", lru.size());
1164 : 111 : }
1165 : :
1166 : 53 : void intern(const string& a, const string& b, string fd, off_t sz, bool front_p)
1167 : : {
1168 : : {
1169 [ + - ]: 106 : unique_lock<mutex> lock(fdcache_lock);
1170 [ + + ]: 120 : for (auto i = lru.begin(); i < lru.end(); i++) // nuke preexisting copy
1171 : : {
1172 [ + + - + : 67 : if (i->archive == a && i->entry == b)
- + ]
1173 : : {
1174 : 0 : unlink (i->fd.c_str());
1175 [ # # ]: 0 : lru.erase(i);
1176 [ # # # # : 0 : inc_metric("fdcache_op_count","op","dequeue");
# # # # ]
1177 : 0 : break; // must not continue iterating
1178 : : }
1179 : : }
1180 : 53 : double mb = (sz+65535)/1048576.0; // round up to 64K block
1181 [ + - + - : 106 : fdcache_entry n = { a, b, fd, mb };
+ - ]
1182 [ + + ]: 53 : if (front_p)
1183 : : {
1184 [ + - + - : 34 : inc_metric("fdcache_op_count","op","enqueue_front");
+ - + - ]
1185 [ + - ]: 34 : lru.push_front(n);
1186 : : }
1187 : : else
1188 : : {
1189 [ + - + - : 19 : inc_metric("fdcache_op_count","op","enqueue_back");
+ - + - ]
1190 [ + - ]: 19 : lru.push_back(n);
1191 : : }
1192 [ + + ]: 53 : if (verbose > 3)
1193 [ + - + - ]: 96 : obatched(clog) << "fdcache interned a=" << a << " b=" << b
1194 [ + - + - : 48 : << " fd=" << fd << " mb=" << mb << " front=" << front_p << endl;
+ - + - +
- + - + -
+ - + - +
- ]
1195 : : }
1196 : 53 : set_metrics();
1197 : :
1198 : : // NB: we age the cache at lookup time too
1199 [ + - + - : 53 : if (statfs_free_enough_p(tmpdir, "tmpdir", fdcache_mintmp))
- + ]
1200 : : {
1201 [ # # # # : 0 : inc_metric("fdcache_op_count","op","emerg-flush");
# # # # ]
1202 [ # # # # ]: 0 : obatched(clog) << "fdcache emergency flush for filling tmpdir" << endl;
1203 : 0 : this->limit(0, 0); // emergency flush
1204 : : }
1205 [ + + ]: 53 : else if (front_p)
1206 : 34 : this->limit(max_fds, max_mbs); // age cache if required
1207 : 53 : }
1208 : :
1209 : 45 : int lookup(const string& a, const string& b)
1210 : : {
1211 : 45 : int fd = -1;
1212 : : {
1213 [ + - ]: 90 : unique_lock<mutex> lock(fdcache_lock);
1214 [ + + ]: 95 : for (auto i = lru.begin(); i < lru.end(); i++)
1215 : : {
1216 [ + + + + : 61 : if (i->archive == a && i->entry == b)
+ + ]
1217 : : { // found it; move it to head of lru
1218 [ + - ]: 22 : fdcache_entry n = *i;
1219 [ + - ]: 11 : lru.erase(i); // invalidates i, so no more iteration!
1220 [ + - ]: 11 : lru.push_front(n);
1221 [ + - + - : 11 : inc_metric("fdcache_op_count","op","requeue_front");
+ - + - ]
1222 [ + - ]: 11 : fd = open(n.fd.c_str(), O_RDONLY); // NB: no problem if dup() fails; looks like cache miss
1223 : 11 : break;
1224 : : }
1225 : : }
1226 : : }
1227 : :
1228 [ + - + - : 45 : if (statfs_free_enough_p(tmpdir, "tmpdir", fdcache_mintmp))
- + ]
1229 : : {
1230 [ # # # # : 0 : inc_metric("fdcache_op_count","op","emerg-flush");
# # # # ]
1231 [ # # ]: 0 : obatched(clog) << "fdcache emergency flush for filling tmpdir";
1232 : 0 : this->limit(0, 0); // emergency flush
1233 : : }
1234 [ + + ]: 45 : else if (fd >= 0)
1235 : 11 : this->limit(max_fds, max_mbs); // age cache if required
1236 : :
1237 : 45 : return fd;
1238 : : }
1239 : :
1240 : 53 : int probe(const string& a, const string& b) // just a cache residency check - don't modify LRU state, don't open
1241 : : {
1242 [ + - ]: 106 : unique_lock<mutex> lock(fdcache_lock);
1243 [ + + ]: 120 : for (auto i = lru.begin(); i < lru.end(); i++)
1244 : : {
1245 [ + + - + : 67 : if (i->archive == a && i->entry == b)
- + ]
1246 : : {
1247 [ # # # # : 0 : inc_metric("fdcache_op_count","op","probe_hit");
# # # # ]
1248 : 0 : return true;
1249 : : }
1250 : : }
1251 [ + - + - : 53 : inc_metric("fdcache_op_count","op","probe_miss");
+ - + - ]
1252 : 53 : return false;
1253 : : }
1254 : :
1255 : 0 : void clear(const string& a, const string& b)
1256 : : {
1257 [ # # ]: 0 : unique_lock<mutex> lock(fdcache_lock);
1258 [ # # ]: 0 : for (auto i = lru.begin(); i < lru.end(); i++)
1259 : : {
1260 [ # # # # : 0 : if (i->archive == a && i->entry == b)
# # ]
1261 : : { // found it; move it to head of lru
1262 [ # # ]: 0 : fdcache_entry n = *i;
1263 [ # # ]: 0 : lru.erase(i); // invalidates i, so no more iteration!
1264 [ # # # # : 0 : inc_metric("fdcache_op_count","op","clear");
# # # # ]
1265 : 0 : unlink (n.fd.c_str());
1266 [ # # ]: 0 : set_metrics();
1267 : 0 : return;
1268 : : }
1269 : : }
1270 : : }
1271 : :
1272 : :
1273 : 61 : void limit(long maxfds, long maxmbs, bool metrics_p = true)
1274 : : {
1275 [ + + + + : 61 : if (verbose > 3 && (this->max_fds != maxfds || this->max_mbs != maxmbs))
- + ]
1276 [ + - + - : 8 : obatched(clog) << "fdcache limited to maxfds=" << maxfds << " maxmbs=" << maxmbs << endl;
+ - + - +
- + - ]
1277 : :
1278 [ + - ]: 122 : unique_lock<mutex> lock(fdcache_lock);
1279 : 61 : this->max_fds = maxfds;
1280 : 61 : this->max_mbs = maxmbs;
1281 : :
1282 : 61 : long total_fd = 0;
1283 : 61 : double total_mb = 0.0;
1284 [ + + ]: 110 : for (auto i = lru.begin(); i < lru.end(); i++)
1285 : : {
1286 : : // accumulate totals from most recently used one going backward
1287 : 89 : total_fd ++;
1288 : 89 : total_mb += i->fd_size_mb;
1289 [ + + - + ]: 89 : if (total_fd > max_fds || total_mb > max_mbs)
1290 : : {
1291 : : // found the cut here point!
1292 : :
1293 [ + + ]: 93 : for (auto j = i; j < lru.end(); j++) // close all the fds from here on in
1294 : : {
1295 [ + + ]: 53 : if (verbose > 3)
1296 [ + - + - ]: 96 : obatched(clog) << "fdcache evicted a=" << j->archive << " b=" << j->entry
1297 [ + - + - : 48 : << " fd=" << j->fd << " mb=" << j->fd_size_mb << endl;
+ - + - +
- + - + -
+ - ]
1298 [ + + ]: 53 : if (metrics_p)
1299 [ + - + - : 47 : inc_metric("fdcache_op_count","op","evict");
+ - + - ]
1300 : 53 : unlink (j->fd.c_str());
1301 : : }
1302 : :
1303 [ + - ]: 40 : lru.erase(i, lru.end()); // erase the nodes generally
1304 : 40 : break;
1305 : : }
1306 : : }
1307 [ + + + - ]: 61 : if (metrics_p) set_metrics();
1308 : 61 : }
1309 : :
1310 : :
1311 : 3 : ~libarchive_fdcache()
1312 : 3 : {
1313 : : // unlink any fdcache entries in $TMPDIR
1314 : : // don't update metrics; those globals may be already destroyed
1315 : 3 : limit(0, 0, false);
1316 : 3 : }
1317 : : };
1318 : : static libarchive_fdcache fdcache;
1319 : :
1320 : :
1321 : : // For security/portability reasons, many distro-package archives have
1322 : : // a "./" in front of path names; others have nothing, others have
1323 : : // "/". Canonicalize them all to a single leading "/", with the
1324 : : // assumption that this matches the dwarf-derived file names too.
1325 : 137 : string canonicalized_archive_entry_pathname(struct archive_entry *e)
1326 : : {
1327 [ + - + - ]: 274 : string fn = archive_entry_pathname(e);
1328 [ - + ]: 137 : if (fn.size() == 0)
1329 : 0 : return fn;
1330 [ + - - + ]: 137 : if (fn[0] == '/')
1331 : 0 : return fn;
1332 [ + - + + ]: 137 : if (fn[0] == '.')
1333 [ + - ]: 117 : return fn.substr(1);
1334 : : else
1335 [ + - + - ]: 40 : return string("/")+fn;
1336 : : }
1337 : :
1338 : :
1339 : :
1340 : : static struct MHD_Response*
1341 : 75 : handle_buildid_r_match (bool internal_req_p,
1342 : : int64_t b_mtime,
1343 : : const string& b_source0,
1344 : : const string& b_source1,
1345 : : int *result_fd)
1346 : : {
1347 : : struct stat fs;
1348 : 75 : int rc = stat (b_source0.c_str(), &fs);
1349 [ + + ]: 75 : if (rc != 0)
1350 [ + - + - : 30 : throw libc_exception (errno, string("stat ") + b_source0);
+ - ]
1351 : :
1352 [ - + ]: 45 : if ((int64_t) fs.st_mtime != b_mtime)
1353 : : {
1354 [ # # ]: 0 : if (verbose)
1355 [ # # # # : 0 : obatched(clog) << "mtime mismatch for " << b_source0 << endl;
# # # # ]
1356 : 0 : return 0;
1357 : : }
1358 : :
1359 : : // check for a match in the fdcache first
1360 [ + - ]: 45 : int fd = fdcache.lookup(b_source0, b_source1);
1361 [ + + ]: 45 : while (fd >= 0) // got one!; NB: this is really an if() with a possible branch out to the end
1362 : : {
1363 : 11 : rc = fstat(fd, &fs);
1364 [ - + ]: 11 : if (rc < 0) // disappeared?
1365 : : {
1366 [ # # ]: 0 : if (verbose)
1367 [ # # # # : 0 : obatched(clog) << "cannot fstat fdcache " << b_source0 << endl;
# # # # ]
1368 [ # # ]: 0 : close(fd);
1369 [ # # ]: 0 : fdcache.clear(b_source0, b_source1);
1370 : 0 : break; // branch out of if "loop", to try new libarchive fetch attempt
1371 : : }
1372 : :
1373 [ + - ]: 11 : struct MHD_Response* r = MHD_create_response_from_fd (fs.st_size, fd);
1374 [ - + ]: 11 : if (r == 0)
1375 : : {
1376 [ # # ]: 0 : if (verbose)
1377 [ # # # # : 0 : obatched(clog) << "cannot create fd-response for " << b_source0 << endl;
# # # # ]
1378 [ # # ]: 0 : close(fd);
1379 : 0 : break; // branch out of if "loop", to try new libarchive fetch attempt
1380 : : }
1381 : :
1382 [ + - + - : 11 : inc_metric ("http_responses_total","result","archive fdcache");
+ - + - ]
1383 : :
1384 [ + - ]: 11 : MHD_add_response_header (r, "Content-Type", "application/octet-stream");
1385 [ + - ]: 11 : add_mhd_last_modified (r, fs.st_mtime);
1386 [ + - ]: 11 : if (verbose > 1)
1387 [ + - + - : 11 : obatched(clog) << "serving fdcache archive " << b_source0 << " file " << b_source1 << endl;
+ - + - +
- + - ]
1388 : : /* libmicrohttpd will close it. */
1389 [ + - ]: 11 : if (result_fd)
1390 : 11 : *result_fd = fd;
1391 : 11 : return r;
1392 : : // NB: see, we never go around the 'loop' more than once
1393 : : }
1394 : :
1395 : : // no match ... grumble, must process the archive
1396 [ + - ]: 68 : string archive_decoder = "/dev/null";
1397 [ + - ]: 68 : string archive_extension = "";
1398 [ + + ]: 136 : for (auto&& arch : scan_archives)
1399 [ + - + + ]: 102 : if (string_endswith(b_source0, arch.first))
1400 : : {
1401 [ + - ]: 34 : archive_extension = arch.first;
1402 [ + - ]: 34 : archive_decoder = arch.second;
1403 : : }
1404 : : FILE* fp;
1405 : : defer_dtor<FILE*,int>::dtor_fn dfn;
1406 [ + + ]: 34 : if (archive_decoder != "cat")
1407 : : {
1408 [ + - + - : 9 : string popen_cmd = archive_decoder + " " + shell_escape(b_source0);
+ - ]
1409 [ + - ]: 3 : fp = popen (popen_cmd.c_str(), "r"); // "e" O_CLOEXEC?
1410 : 3 : dfn = pclose;
1411 [ - + ]: 3 : if (fp == NULL)
1412 [ # # # # : 0 : throw libc_exception (errno, string("popen ") + popen_cmd);
# # ]
1413 : : }
1414 : : else
1415 : : {
1416 [ + - ]: 31 : fp = fopen (b_source0.c_str(), "r");
1417 : 31 : dfn = fclose;
1418 [ - + ]: 31 : if (fp == NULL)
1419 [ # # # # : 0 : throw libc_exception (errno, string("fopen ") + b_source0);
# # ]
1420 : : }
1421 : 68 : defer_dtor<FILE*,int> fp_closer (fp, dfn);
1422 : :
1423 : : struct archive *a;
1424 [ + - ]: 34 : a = archive_read_new();
1425 [ - + ]: 34 : if (a == NULL)
1426 [ # # # # ]: 0 : throw archive_exception("cannot create archive reader");
1427 : 34 : defer_dtor<struct archive*,int> archive_closer (a, archive_read_free);
1428 : :
1429 [ + - ]: 34 : rc = archive_read_support_format_all(a);
1430 [ - + ]: 34 : if (rc != ARCHIVE_OK)
1431 [ # # # # ]: 0 : throw archive_exception(a, "cannot select all format");
1432 [ + - ]: 34 : rc = archive_read_support_filter_all(a);
1433 [ - + ]: 34 : if (rc != ARCHIVE_OK)
1434 [ # # # # ]: 0 : throw archive_exception(a, "cannot select all filters");
1435 : :
1436 [ + - ]: 34 : rc = archive_read_open_FILE (a, fp);
1437 [ - + ]: 34 : if (rc != ARCHIVE_OK)
1438 [ # # # # ]: 0 : throw archive_exception(a, "cannot open archive from pipe");
1439 : :
1440 : : // archive traversal is in three stages, no, four stages:
1441 : : // 1) skip entries whose names do not match the requested one
1442 : : // 2) extract the matching entry name (set r = result)
1443 : : // 3) extract some number of prefetched entries (just into fdcache)
1444 : : // 4) abort any further processing
1445 : 34 : struct MHD_Response* r = 0; // will set in stage 2
1446 [ + + ]: 34 : unsigned prefetch_count =
1447 : : internal_req_p ? 0 : fdcache_prefetch; // will decrement in stage 3
1448 : :
1449 [ + + + + ]: 340 : while(r == 0 || prefetch_count > 0) // stage 1, 2, or 3
1450 : : {
1451 [ - + ]: 336 : if (interrupted)
1452 : 30 : break;
1453 : :
1454 : : struct archive_entry *e;
1455 [ + - ]: 336 : rc = archive_read_next_header (a, &e);
1456 [ + + ]: 336 : if (rc != ARCHIVE_OK)
1457 : 30 : break;
1458 : :
1459 [ + - + + ]: 306 : if (! S_ISREG(archive_entry_mode (e))) // skip non-files completely
1460 : 306 : continue;
1461 : :
1462 [ + - ]: 68 : string fn = canonicalized_archive_entry_pathname (e);
1463 [ + + + + : 68 : if ((r == 0) && (fn != b_source1)) // stage 1
+ + ]
1464 : 15 : continue;
1465 : :
1466 [ + - - + ]: 53 : if (fdcache.probe (b_source0, fn)) // skip if already interned
1467 : 0 : continue;
1468 : :
1469 : : // extract this file to a temporary file
1470 : 53 : char* tmppath = NULL;
1471 : 53 : rc = asprintf (&tmppath, "%s/debuginfod.XXXXXX", tmpdir.c_str());
1472 [ - + ]: 53 : if (rc < 0)
1473 [ # # # # ]: 0 : throw libc_exception (ENOMEM, "cannot allocate tmppath");
1474 : 53 : defer_dtor<void*,void> tmmpath_freer (tmppath, free);
1475 [ + - ]: 53 : fd = mkstemp (tmppath);
1476 [ - + ]: 53 : if (fd < 0)
1477 [ # # # # ]: 0 : throw libc_exception (errno, "cannot create temporary file");
1478 : : // NB: don't unlink (tmppath), as fdcache will take charge of it.
1479 : :
1480 : : // NB: this can take many uninterruptible seconds for a huge file
1481 [ + - ]: 53 : rc = archive_read_data_into_fd (a, fd);
1482 [ - + ]: 53 : if (rc != ARCHIVE_OK) // e.g. ENOSPC!
1483 : : {
1484 [ # # ]: 0 : close (fd);
1485 : 0 : unlink (tmppath);
1486 [ # # # # ]: 0 : throw archive_exception(a, "cannot extract file");
1487 : : }
1488 : :
1489 : : // Set the mtime so the fdcache file mtimes, even prefetched ones,
1490 : : // propagate to future webapi clients.
1491 : : struct timeval tvs[2];
1492 [ + - ]: 53 : tvs[0].tv_sec = tvs[1].tv_sec = archive_entry_mtime(e);
1493 : 53 : tvs[0].tv_usec = tvs[1].tv_usec = 0;
1494 : 53 : (void) futimes (fd, tvs); /* best effort */
1495 : :
1496 [ + + ]: 53 : if (r != 0) // stage 3
1497 : : {
1498 : : // NB: now we know we have a complete reusable file; make fdcache
1499 : : // responsible for unlinking it later.
1500 [ + - + - ]: 19 : fdcache.intern(b_source0, fn,
1501 : : tmppath, archive_entry_size(e),
1502 [ + - ]: 38 : false); // prefetched ones go to back of lru
1503 : 19 : prefetch_count --;
1504 [ + - ]: 19 : close (fd); // we're not saving this fd to make a mhd-response from!
1505 : 19 : continue;
1506 : : }
1507 : :
1508 : : // NB: now we know we have a complete reusable file; make fdcache
1509 : : // responsible for unlinking it later.
1510 [ + - + - ]: 34 : fdcache.intern(b_source0, b_source1,
1511 : : tmppath, archive_entry_size(e),
1512 [ + - ]: 68 : true); // requested ones go to the front of lru
1513 : :
1514 [ + - + - : 34 : inc_metric ("http_responses_total","result",archive_extension + " archive");
+ - + - ]
1515 [ + - + - ]: 34 : r = MHD_create_response_from_fd (archive_entry_size(e), fd);
1516 [ - + ]: 34 : if (r == 0)
1517 : : {
1518 [ # # ]: 0 : if (verbose)
1519 [ # # # # : 0 : obatched(clog) << "cannot create fd-response for " << b_source0 << endl;
# # # # ]
1520 [ # # ]: 0 : close(fd);
1521 : 0 : break; // assume no chance of better luck around another iteration; no other copies of same file
1522 : : }
1523 : : else
1524 : : {
1525 [ + - ]: 34 : MHD_add_response_header (r, "Content-Type", "application/octet-stream");
1526 [ + - + - ]: 34 : add_mhd_last_modified (r, archive_entry_mtime(e));
1527 [ + - ]: 34 : if (verbose > 1)
1528 [ + - + - : 34 : obatched(clog) << "serving archive " << b_source0 << " file " << b_source1 << endl;
+ - + - +
- + - ]
1529 : : /* libmicrohttpd will close it. */
1530 [ + - ]: 34 : if (result_fd)
1531 : 34 : *result_fd = fd;
1532 : 34 : continue;
1533 : : }
1534 : : }
1535 : :
1536 : : // XXX: rpm/file not found: delete this R entry?
1537 : 34 : return r;
1538 : : }
1539 : :
1540 : :
1541 : : static struct MHD_Response*
1542 : 93 : handle_buildid_match (bool internal_req_p,
1543 : : int64_t b_mtime,
1544 : : const string& b_stype,
1545 : : const string& b_source0,
1546 : : const string& b_source1,
1547 : : int *result_fd)
1548 : : {
1549 : : try
1550 : : {
1551 [ + + ]: 93 : if (b_stype == "F")
1552 [ + - ]: 18 : return handle_buildid_f_match(internal_req_p, b_mtime, b_source0, result_fd);
1553 [ + - ]: 75 : else if (b_stype == "R")
1554 [ + + ]: 75 : return handle_buildid_r_match(internal_req_p, b_mtime, b_source0, b_source1, result_fd);
1555 : : }
1556 [ - + ]: 60 : catch (const reportable_exception &e)
1557 : : {
1558 [ + - ]: 30 : e.report(clog);
1559 : : // Report but swallow libc etc. errors here; let the caller
1560 : : // iterate to other matches of the content.
1561 : : }
1562 : :
1563 : 30 : return 0;
1564 : : }
1565 : :
1566 : :
1567 : : static int
1568 : 7 : debuginfod_find_progress (debuginfod_client *, long a, long b)
1569 : : {
1570 [ - + ]: 7 : if (verbose > 4)
1571 [ # # # # : 0 : obatched(clog) << "federated debuginfod progress=" << a << "/" << b << endl;
# # # # #
# ]
1572 : :
1573 : 7 : return interrupted;
1574 : : }
1575 : :
1576 : :
1577 : : // a little lru pool of debuginfod_client*s for reuse between query threads
1578 : :
1579 : : mutex dc_pool_lock;
1580 : : deque<debuginfod_client*> dc_pool;
1581 : :
1582 : 16 : debuginfod_client* debuginfod_pool_begin()
1583 : : {
1584 [ + - ]: 32 : unique_lock<mutex> lock(dc_pool_lock);
1585 [ + + ]: 16 : if (dc_pool.size() > 0)
1586 : : {
1587 [ + - + - : 11 : inc_metric("dc_pool_op_count","op","begin-reuse");
+ - + - ]
1588 : 11 : debuginfod_client *c = dc_pool.front();
1589 : 11 : dc_pool.pop_front();
1590 : 11 : return c;
1591 : : }
1592 [ + - + - : 5 : inc_metric("dc_pool_op_count","op","begin-new");
+ - + - ]
1593 [ + - ]: 5 : return debuginfod_begin();
1594 : : }
1595 : :
1596 : :
1597 : 5 : void debuginfod_pool_groom()
1598 : : {
1599 [ + - ]: 10 : unique_lock<mutex> lock(dc_pool_lock);
1600 [ + + ]: 7 : while (dc_pool.size() > 0)
1601 : : {
1602 [ + - + - : 2 : inc_metric("dc_pool_op_count","op","end");
+ - + - ]
1603 [ + - ]: 2 : debuginfod_end(dc_pool.front());
1604 : 2 : dc_pool.pop_front();
1605 : : }
1606 : 5 : }
1607 : :
1608 : :
1609 : 16 : void debuginfod_pool_end(debuginfod_client* c)
1610 : : {
1611 [ + - ]: 32 : unique_lock<mutex> lock(dc_pool_lock);
1612 [ + - + - : 16 : inc_metric("dc_pool_op_count","op","end-save");
+ - + - ]
1613 [ + - ]: 16 : dc_pool.push_front(c); // accelerate reuse, vs. push_back
1614 : 16 : }
1615 : :
1616 : :
1617 : : static struct MHD_Response*
1618 : 81 : handle_buildid (MHD_Connection* conn,
1619 : : const string& buildid /* unsafe */,
1620 : : const string& artifacttype /* unsafe */,
1621 : : const string& suffix /* unsafe */,
1622 : : int *result_fd)
1623 : : {
1624 : : // validate artifacttype
1625 : 162 : string atype_code;
1626 [ + + + - ]: 81 : if (artifacttype == "debuginfo") atype_code = "D";
1627 [ + + + - ]: 32 : else if (artifacttype == "executable") atype_code = "E";
1628 [ + + + - ]: 15 : else if (artifacttype == "source") atype_code = "S";
1629 [ + - + - ]: 1 : else throw reportable_exception("invalid artifacttype");
1630 : :
1631 [ + - + - : 80 : inc_metric("http_requests_total", "type", artifacttype);
+ - ]
1632 : :
1633 [ + + - + : 80 : if (atype_code == "S" && suffix == "")
- + ]
1634 [ # # # # ]: 0 : throw reportable_exception("invalid source suffix");
1635 : :
1636 : : // validate buildid
1637 : 80 : if ((buildid.size() < 2) || // not empty
1638 [ + - + - : 160 : (buildid.size() % 2) || // even number
- + - + ]
1639 : 80 : (buildid.find_first_not_of("0123456789abcdef") != string::npos)) // pure tasty lowercase hex
1640 [ # # # # ]: 0 : throw reportable_exception("invalid buildid");
1641 : :
1642 [ + - ]: 80 : if (verbose > 1)
1643 [ + - + - ]: 160 : obatched(clog) << "searching for buildid=" << buildid << " artifacttype=" << artifacttype
1644 [ + - + - : 80 : << " suffix=" << suffix << endl;
+ - + - +
- + - ]
1645 : :
1646 : : // If invoked from the scanner threads, use the scanners' read-write
1647 : : // connection. Otherwise use the web query threads' read-only connection.
1648 [ + + ]: 80 : sqlite3 *thisdb = (conn == 0) ? db : dbq;
1649 : :
1650 : 80 : sqlite_ps *pp = 0;
1651 : :
1652 [ + + ]: 80 : if (atype_code == "D")
1653 : : {
1654 : 49 : pp = new sqlite_ps (thisdb, "mhd-query-d",
1655 : : "select mtime, sourcetype, source0, source1 from " BUILDIDS "_query_d where buildid = ? "
1656 [ + - + - : 49 : "order by mtime desc");
+ - + - ]
1657 [ + - ]: 49 : pp->reset();
1658 [ + - ]: 49 : pp->bind(1, buildid);
1659 : : }
1660 [ + + ]: 31 : else if (atype_code == "E")
1661 : : {
1662 : 17 : pp = new sqlite_ps (thisdb, "mhd-query-e",
1663 : : "select mtime, sourcetype, source0, source1 from " BUILDIDS "_query_e where buildid = ? "
1664 [ + - + - : 17 : "order by mtime desc");
+ - + - ]
1665 [ + - ]: 17 : pp->reset();
1666 [ + - ]: 17 : pp->bind(1, buildid);
1667 : : }
1668 [ + - ]: 14 : else if (atype_code == "S")
1669 : : {
1670 : : // PR25548
1671 : : // Incoming source queries may come in with either dwarf-level OR canonicalized paths.
1672 : : // We let the query pass with either one.
1673 : :
1674 : 14 : pp = new sqlite_ps (thisdb, "mhd-query-s",
1675 : : "select mtime, sourcetype, source0, source1 from " BUILDIDS "_query_s where buildid = ? and artifactsrc in (?,?) "
1676 [ + - + - : 14 : "order by sharedprefix(source0,source0ref) desc, mtime desc");
+ - + - ]
1677 [ + - ]: 14 : pp->reset();
1678 [ + - ]: 14 : pp->bind(1, buildid);
1679 : : // NB: we don't store the non-canonicalized path names any more, but old databases
1680 : : // might have them (and no canon ones), so we keep searching for both.
1681 [ + - ]: 14 : pp->bind(2, suffix);
1682 [ + - + - ]: 14 : pp->bind(3, canon_pathname(suffix));
1683 : : }
1684 : 160 : unique_ptr<sqlite_ps> ps_closer(pp); // release pp if exception or return
1685 : :
1686 : : // consume all the rows
1687 : : while (1)
1688 : : {
1689 [ + - ]: 110 : int rc = pp->step();
1690 [ + + ]: 110 : if (rc == SQLITE_DONE) break;
1691 [ + + ]: 94 : if (rc != SQLITE_ROW)
1692 [ + - + - ]: 1 : throw sqlite_exception(rc, "step");
1693 : :
1694 [ + - ]: 93 : int64_t b_mtime = sqlite3_column_int64 (*pp, 0);
1695 [ + - + - : 93 : string b_stype = string((const char*) sqlite3_column_text (*pp, 1) ?: ""); /* by DDL may not be NULL */
+ - ]
1696 [ + - + - : 93 : string b_source0 = string((const char*) sqlite3_column_text (*pp, 2) ?: ""); /* may be NULL */
+ - ]
1697 [ + - + + : 93 : string b_source1 = string((const char*) sqlite3_column_text (*pp, 3) ?: ""); /* may be NULL */
+ - ]
1698 : :
1699 [ + - ]: 93 : if (verbose > 1)
1700 [ + - + - : 186 : obatched(clog) << "found mtime=" << b_mtime << " stype=" << b_stype
+ - ]
1701 [ + - + - : 93 : << " source0=" << b_source0 << " source1=" << b_source1 << endl;
+ - + - +
- + - +
- ]
1702 : :
1703 : : // Try accessing the located match.
1704 : : // XXX: in case of multiple matches, attempt them in parallel?
1705 [ + - ]: 93 : auto r = handle_buildid_match (conn ? false : true,
1706 : : b_mtime, b_stype, b_source0, b_source1, result_fd);
1707 [ + + ]: 93 : if (r)
1708 : 63 : return r;
1709 : 30 : }
1710 [ + - ]: 16 : pp->reset();
1711 : :
1712 : : // We couldn't find it in the database. Last ditch effort
1713 : : // is to defer to other debuginfo servers.
1714 : :
1715 : 16 : int fd = -1;
1716 [ + - ]: 16 : debuginfod_client *client = debuginfod_pool_begin ();
1717 [ + - ]: 16 : if (client != NULL)
1718 : : {
1719 [ + - ]: 16 : debuginfod_set_progressfn (client, & debuginfod_find_progress);
1720 : :
1721 [ + - ]: 16 : if (conn)
1722 : : {
1723 : : // Transcribe incoming User-Agent:
1724 [ + - + - : 32 : string ua = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "User-Agent") ?: "";
+ - ]
1725 [ + - + - ]: 48 : string ua_complete = string("User-Agent: ") + ua;
1726 [ + - ]: 16 : debuginfod_add_http_header (client, ua_complete.c_str());
1727 : :
1728 : : // Compute larger XFF:, for avoiding info loss during
1729 : : // federation, and for future cyclicity detection.
1730 [ + - + + : 32 : string xff = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "X-Forwarded-For") ?: "";
+ - ]
1731 [ + + ]: 16 : if (xff != "")
1732 [ + - + - ]: 3 : xff += string(", "); // comma separated list
1733 : :
1734 : : // Compute the client's numeric IP address only - so can't merge with conninfo()
1735 [ + - ]: 16 : const union MHD_ConnectionInfo *u = MHD_get_connection_info (conn,
1736 : : MHD_CONNECTION_INFO_CLIENT_ADDRESS);
1737 [ + - ]: 16 : struct sockaddr *so = u ? u->client_addr : 0;
1738 : 16 : char hostname[256] = ""; // RFC1035
1739 [ + - + - ]: 16 : if (so && so->sa_family == AF_INET)
1740 [ + - ]: 16 : (void) getnameinfo (so, sizeof (struct sockaddr_in), hostname, sizeof (hostname), NULL, 0,
1741 : : NI_NUMERICHOST);
1742 [ # # # # ]: 0 : else if (so && so->sa_family == AF_INET6)
1743 [ # # ]: 0 : (void) getnameinfo (so, sizeof (struct sockaddr_in6), hostname, sizeof (hostname), NULL, 0,
1744 : : NI_NUMERICHOST);
1745 : :
1746 [ + - + - : 64 : string xff_complete = string("X-Forwarded-For: ")+xff+string(hostname);
+ - + - ]
1747 [ + - ]: 16 : debuginfod_add_http_header (client, xff_complete.c_str());
1748 : : }
1749 : :
1750 [ + + ]: 16 : if (artifacttype == "debuginfo")
1751 [ + - ]: 15 : fd = debuginfod_find_debuginfo (client,
1752 : 15 : (const unsigned char*) buildid.c_str(),
1753 : : 0, NULL);
1754 [ + - ]: 1 : else if (artifacttype == "executable")
1755 [ + - ]: 1 : fd = debuginfod_find_executable (client,
1756 : 1 : (const unsigned char*) buildid.c_str(),
1757 : : 0, NULL);
1758 [ # # ]: 0 : else if (artifacttype == "source")
1759 [ # # ]: 0 : fd = debuginfod_find_source (client,
1760 : 0 : (const unsigned char*) buildid.c_str(),
1761 : : 0, suffix.c_str(), NULL);
1762 : : }
1763 : : else
1764 : 0 : fd = -errno; /* Set by debuginfod_begin. */
1765 [ + - ]: 16 : debuginfod_pool_end (client);
1766 : :
1767 [ + + ]: 16 : if (fd >= 0)
1768 : : {
1769 [ + - + - : 1 : inc_metric ("http_responses_total","result","upstream");
+ - + - ]
1770 : : struct stat s;
1771 : 1 : int rc = fstat (fd, &s);
1772 [ + - ]: 1 : if (rc == 0)
1773 : : {
1774 [ + - ]: 1 : auto r = MHD_create_response_from_fd ((uint64_t) s.st_size, fd);
1775 [ + - ]: 1 : if (r)
1776 : : {
1777 [ + - ]: 1 : MHD_add_response_header (r, "Content-Type", "application/octet-stream");
1778 [ + - ]: 1 : add_mhd_last_modified (r, s.st_mtime);
1779 [ + - ]: 1 : if (verbose > 1)
1780 [ + - + - : 1 : obatched(clog) << "serving file from upstream debuginfod/cache" << endl;
+ - ]
1781 [ + - ]: 1 : if (result_fd)
1782 : 1 : *result_fd = fd;
1783 : 1 : return r; // NB: don't close fd; libmicrohttpd will
1784 : : }
1785 : : }
1786 [ # # ]: 0 : close (fd);
1787 : : }
1788 : : else
1789 [ + + - ]: 15 : switch(fd)
1790 : : {
1791 : 13 : case -ENOSYS:
1792 : 13 : break;
1793 : 2 : case -ENOENT:
1794 : 2 : break;
1795 : 0 : default: // some more tricky error
1796 [ # # # # ]: 0 : throw libc_exception(-fd, "upstream debuginfod query failed");
1797 : : }
1798 : :
1799 [ + - + - ]: 15 : throw reportable_exception(MHD_HTTP_NOT_FOUND, "not found");
1800 : : }
1801 : :
1802 : :
1803 : : ////////////////////////////////////////////////////////////////////////
1804 : :
1805 : : static map<string,double> metrics; // arbitrary data for /metrics query
1806 : : // NB: store int64_t since all our metrics are integers; prometheus accepts double
1807 : : static mutex metrics_lock;
1808 : : // NB: these objects get released during the process exit via global dtors
1809 : : // do not call them from within other global dtors
1810 : :
1811 : : // utility function for assembling prometheus-compatible
1812 : : // name="escaped-value" strings
1813 : : // https://prometheus.io/docs/instrumenting/exposition_formats/
1814 : : static string
1815 : 14502 : metric_label(const string& name, const string& value)
1816 : : {
1817 : 14502 : string x = name + "=\"";
1818 [ + + ]: 206773 : for (auto&& c : value)
1819 [ - - - + ]: 192279 : switch(c)
1820 : : {
1821 [ # # ]: 0 : case '\\': x += "\\\\"; break;
1822 [ # # ]: 0 : case '\"': x += "\\\""; break;
1823 [ # # ]: 0 : case '\n': x += "\\n"; break;
1824 [ + - ]: 192279 : default: x += c; break;
1825 : : }
1826 [ + - ]: 14458 : x += "\"";
1827 : 14508 : return x;
1828 : : }
1829 : :
1830 : :
1831 : : // add prometheus-format metric name + label tuple (if any) + value
1832 : :
1833 : : static void
1834 : 228 : set_metric(const string& metric, double value)
1835 : : {
1836 [ + - ]: 228 : unique_lock<mutex> lock(metrics_lock);
1837 [ + - ]: 228 : metrics[metric] = value;
1838 : 228 : }
1839 : : #if 0 /* unused */
1840 : : static void
1841 : : inc_metric(const string& metric)
1842 : : {
1843 : : unique_lock<mutex> lock(metrics_lock);
1844 : : metrics[metric] ++;
1845 : : }
1846 : : #endif
1847 : : static void
1848 : 885 : set_metric(const string& metric,
1849 : : const string& lname, const string& lvalue,
1850 : : double value)
1851 : : {
1852 [ + - + - : 2657 : string key = (metric + "{" + metric_label(lname, lvalue) + "}");
+ - + - ]
1853 [ + - ]: 886 : unique_lock<mutex> lock(metrics_lock);
1854 [ + - ]: 886 : metrics[key] = value;
1855 : 886 : }
1856 : :
1857 : : static void
1858 : 7224 : inc_metric(const string& metric,
1859 : : const string& lname, const string& lvalue)
1860 : : {
1861 [ + - + - : 21670 : string key = (metric + "{" + metric_label(lname, lvalue) + "}");
+ - + - ]
1862 [ + - ]: 7224 : unique_lock<mutex> lock(metrics_lock);
1863 [ + - ]: 7224 : metrics[key] ++;
1864 : 7224 : }
1865 : : static void
1866 : 6398 : add_metric(const string& metric,
1867 : : const string& lname, const string& lvalue,
1868 : : double value)
1869 : : {
1870 [ + - + - : 19195 : string key = (metric + "{" + metric_label(lname, lvalue) + "}");
+ - + - ]
1871 [ + - ]: 6399 : unique_lock<mutex> lock(metrics_lock);
1872 [ + - ]: 6399 : metrics[key] += value;
1873 : 6399 : }
1874 : : #if 0
1875 : : static void
1876 : : add_metric(const string& metric,
1877 : : double value)
1878 : : {
1879 : : unique_lock<mutex> lock(metrics_lock);
1880 : : metrics[metric] += value;
1881 : : }
1882 : : #endif
1883 : :
1884 : :
1885 : : // and more for higher arity labels if needed
1886 : :
1887 : :
1888 : : static struct MHD_Response*
1889 : 80 : handle_metrics (off_t* size)
1890 : : {
1891 [ + - ]: 160 : stringstream o;
1892 : : {
1893 [ + - ]: 160 : unique_lock<mutex> lock(metrics_lock);
1894 [ + + ]: 11038 : for (auto&& i : metrics)
1895 : 10958 : o << i.first
1896 : : << " "
1897 [ + - + - : 10958 : << std::setprecision(std::numeric_limits<double>::digits10 + 1)
+ - ]
1898 [ + - ]: 10958 : << i.second
1899 [ + - ]: 10958 : << endl;
1900 : : }
1901 [ + - ]: 80 : const string& os = o.str();
1902 [ + - ]: 160 : MHD_Response* r = MHD_create_response_from_buffer (os.size(),
1903 : 80 : (void*) os.c_str(),
1904 : : MHD_RESPMEM_MUST_COPY);
1905 : 80 : *size = os.size();
1906 [ + - ]: 80 : MHD_add_response_header (r, "Content-Type", "text/plain");
1907 : 160 : return r;
1908 : : }
1909 : :
1910 : : static struct MHD_Response*
1911 : 0 : handle_root (off_t* size)
1912 : : {
1913 [ # # # # : 0 : static string version = "debuginfod (" + string (PACKAGE_NAME) + ") "
# # ]
1914 [ # # # # : 0 : + string (PACKAGE_VERSION);
# # # # #
# ]
1915 : 0 : MHD_Response* r = MHD_create_response_from_buffer (version.size (),
1916 : 0 : (void *) version.c_str (),
1917 : : MHD_RESPMEM_PERSISTENT);
1918 : 0 : *size = version.size ();
1919 : 0 : MHD_add_response_header (r, "Content-Type", "text/plain");
1920 : 0 : return r;
1921 : : }
1922 : :
1923 : :
1924 : : ////////////////////////////////////////////////////////////////////////
1925 : :
1926 : :
1927 : : /* libmicrohttpd callback */
1928 : : static MHD_RESULT
1929 : 310 : handler_cb (void * /*cls*/,
1930 : : struct MHD_Connection *connection,
1931 : : const char *url,
1932 : : const char *method,
1933 : : const char * /*version*/,
1934 : : const char * /*upload_data*/,
1935 : : size_t * /*upload_data_size*/,
1936 : : void ** ptr)
1937 : : {
1938 : 310 : struct MHD_Response *r = NULL;
1939 [ + - ]: 620 : string url_copy = url;
1940 : :
1941 : : /* libmicrohttpd always makes (at least) two callbacks: once just
1942 : : past the headers, and one after the request body is finished
1943 : : being received. If we process things early (first callback) and
1944 : : queue a response, libmicrohttpd would suppress http keep-alive
1945 : : (via connection->read_closed = true). */
1946 : : static int aptr; /* just some random object to use as a flag */
1947 [ + + ]: 310 : if (&aptr != *ptr)
1948 : : {
1949 : : /* do never respond on first call */
1950 : 155 : *ptr = &aptr;
1951 : 155 : return MHD_YES;
1952 : : }
1953 : 155 : *ptr = NULL; /* reset when done */
1954 : :
1955 : : #if MHD_VERSION >= 0x00097002
1956 : : enum MHD_Result rc;
1957 : : #else
1958 : : int rc = MHD_NO; // mhd
1959 : : #endif
1960 : 155 : int http_code = 500;
1961 : 155 : off_t http_size = -1;
1962 : : struct timespec ts_start, ts_end;
1963 : 155 : clock_gettime (CLOCK_MONOTONIC, &ts_start);
1964 : :
1965 : : try
1966 : : {
1967 [ + - - + ]: 155 : if (string(method) != "GET")
1968 [ # # # # ]: 0 : throw reportable_exception(400, "we support GET only");
1969 : :
1970 : : /* Start decoding the URL. */
1971 : 155 : size_t slash1 = url_copy.find('/', 1);
1972 [ + - ]: 310 : string url1 = url_copy.substr(0, slash1); // ok even if slash1 not found
1973 : :
1974 [ + + + - : 155 : if (slash1 != string::npos && url1 == "/buildid")
+ + ]
1975 : : {
1976 [ + - + - : 365 : tmp_inc_metric m ("thread_busy", "role", "http-buildid");
+ - + - ]
1977 : 73 : size_t slash2 = url_copy.find('/', slash1+1);
1978 [ - + ]: 73 : if (slash2 == string::npos)
1979 [ # # # # ]: 0 : throw reportable_exception("/buildid/ webapi error, need buildid");
1980 : :
1981 [ + - ]: 146 : string buildid = url_copy.substr(slash1+1, slash2-slash1-1);
1982 : :
1983 : 73 : size_t slash3 = url_copy.find('/', slash2+1);
1984 : 163 : string artifacttype, suffix;
1985 [ + + ]: 73 : if (slash3 == string::npos)
1986 : : {
1987 [ + - ]: 59 : artifacttype = url_copy.substr(slash2+1);
1988 [ + - ]: 59 : suffix = "";
1989 : : }
1990 : : else
1991 : : {
1992 [ + - ]: 14 : artifacttype = url_copy.substr(slash2+1, slash3-slash2-1);
1993 [ + - ]: 14 : suffix = url_copy.substr(slash3); // include the slash in the suffix
1994 : : }
1995 : :
1996 : : // get the resulting fd so we can report its size
1997 : : int fd;
1998 [ + + ]: 73 : r = handle_buildid(connection, buildid, artifacttype, suffix, &fd);
1999 [ + - ]: 56 : if (r)
2000 : : {
2001 : : struct stat fs;
2002 [ + - ]: 56 : if (fstat(fd, &fs) == 0)
2003 : 56 : http_size = fs.st_size;
2004 : : // libmicrohttpd will close (fd);
2005 : : }
2006 : : }
2007 [ + + ]: 82 : else if (url1 == "/metrics")
2008 : : {
2009 [ + - + - : 320 : tmp_inc_metric m ("thread_busy", "role", "http-metrics");
+ - + - ]
2010 [ + - + - : 80 : inc_metric("http_requests_total", "type", "metrics");
+ - + - ]
2011 [ + - ]: 80 : r = handle_metrics(& http_size);
2012 : : }
2013 [ - + ]: 2 : else if (url1 == "/")
2014 : : {
2015 [ # # # # : 0 : inc_metric("http_requests_total", "type", "/");
# # # # ]
2016 [ # # ]: 0 : r = handle_root(& http_size);
2017 : : }
2018 : : else
2019 [ + - + - : 2 : throw reportable_exception("webapi error, unrecognized '" + url1 + "'");
+ - ]
2020 : :
2021 [ - + ]: 136 : if (r == 0)
2022 [ # # # # ]: 0 : throw reportable_exception("internal error, missing response");
2023 : :
2024 [ + - ]: 136 : rc = MHD_queue_response (connection, MHD_HTTP_OK, r);
2025 : 136 : http_code = MHD_HTTP_OK;
2026 [ + - ]: 136 : MHD_destroy_response (r);
2027 : : }
2028 [ - + ]: 19 : catch (const reportable_exception& e)
2029 : : {
2030 [ + - + - : 19 : inc_metric("http_responses_total","result","error");
+ - + - ]
2031 [ + - ]: 19 : e.report(clog);
2032 : 19 : http_code = e.code;
2033 : 19 : http_size = e.message.size();
2034 [ + - ]: 19 : rc = e.mhd_send_response (connection);
2035 : : }
2036 : :
2037 : 155 : clock_gettime (CLOCK_MONOTONIC, &ts_end);
2038 : 155 : double deltas = (ts_end.tv_sec - ts_start.tv_sec) + (ts_end.tv_nsec - ts_start.tv_nsec)/1.e9;
2039 [ + - + - : 310 : obatched(clog) << conninfo(connection)
+ - ]
2040 : : << ' ' << method << ' ' << url
2041 [ + - + - : 155 : << ' ' << http_code << ' ' << http_size
+ - + - +
- + - +
- ]
2042 [ + - + - : 155 : << ' ' << (int)(deltas*1000) << "ms"
+ - + - ]
2043 [ + - ]: 155 : << endl;
2044 : :
2045 : : // related prometheus metrics
2046 [ + - ]: 155 : string http_code_str = to_string(http_code);
2047 [ + - ]: 155 : if (http_size >= 0)
2048 [ + - ]: 155 : add_metric("http_responses_transfer_bytes_sum","code",http_code_str,
2049 [ + - + - ]: 310 : http_size);
2050 [ + - + - : 155 : inc_metric("http_responses_transfer_bytes_count","code",http_code_str);
+ - ]
2051 : :
2052 [ + - ]: 155 : add_metric("http_responses_duration_milliseconds_sum","code",http_code_str,
2053 [ + - + - ]: 310 : deltas*1000); // prometheus prefers _seconds and floating point
2054 [ + - + - : 155 : inc_metric("http_responses_duration_milliseconds_count","code",http_code_str);
+ - ]
2055 : :
2056 : 155 : return rc;
2057 : : }
2058 : :
2059 : :
2060 : : ////////////////////////////////////////////////////////////////////////
2061 : : // borrowed originally from src/nm.c get_local_names()
2062 : :
2063 : : static void
2064 : 23 : dwarf_extract_source_paths (Elf *elf, set<string>& debug_sourcefiles)
2065 : : noexcept // no exceptions - so we can simplify the altdbg resource release at end
2066 : : {
2067 : 23 : Dwarf* dbg = dwarf_begin_elf (elf, DWARF_C_READ, NULL);
2068 [ - + ]: 23 : if (dbg == NULL)
2069 : 0 : return;
2070 : :
2071 : 23 : Dwarf* altdbg = NULL;
2072 : 23 : int altdbg_fd = -1;
2073 : :
2074 : : // DWZ handling: if we have an unsatisfied debug-alt-link, add an
2075 : : // empty string into the outgoing sourcefiles set, so the caller
2076 : : // should know that our data is incomplete.
2077 : : const char *alt_name_p;
2078 : : const void *alt_build_id; // elfutils-owned memory
2079 : 23 : ssize_t sz = dwelf_dwarf_gnu_debugaltlink (dbg, &alt_name_p, &alt_build_id);
2080 [ + + ]: 23 : if (sz > 0) // got one!
2081 : : {
2082 : 16 : string buildid;
2083 : 8 : unsigned char* build_id_bytes = (unsigned char*) alt_build_id;
2084 [ + + ]: 168 : for (ssize_t idx=0; idx<sz; idx++)
2085 : : {
2086 : 160 : buildid += "0123456789abcdef"[build_id_bytes[idx] >> 4];
2087 : 160 : buildid += "0123456789abcdef"[build_id_bytes[idx] & 0xf];
2088 : : }
2089 : :
2090 [ + - ]: 8 : if (verbose > 3)
2091 : 8 : obatched(clog) << "Need altdebug buildid=" << buildid << endl;
2092 : :
2093 : : // but is it unsatisfied the normal elfutils ways?
2094 : 8 : Dwarf* alt = dwarf_getalt (dbg);
2095 [ + - ]: 8 : if (alt == NULL)
2096 : : {
2097 : : // Yup, unsatisfied the normal way. Maybe we can satisfy it
2098 : : // from our own debuginfod database.
2099 : : int alt_fd;
2100 : 8 : struct MHD_Response *r = 0;
2101 : : try
2102 : : {
2103 [ + - + - : 8 : r = handle_buildid (0, buildid, "debuginfo", "", &alt_fd);
+ - ]
2104 : : }
2105 [ - - ]: 0 : catch (const reportable_exception& e)
2106 : : {
2107 : : // swallow exceptions
2108 : : }
2109 : :
2110 : : // NB: this is not actually recursive! This invokes the web-query
2111 : : // path, which cannot get back into the scan code paths.
2112 [ + - ]: 8 : if (r)
2113 : : {
2114 : : // Found it!
2115 : 8 : altdbg_fd = dup(alt_fd); // ok if this fails, downstream failures ok
2116 : 8 : alt = altdbg = dwarf_begin (altdbg_fd, DWARF_C_READ);
2117 : : // NB: must close this dwarf and this fd at the bottom of the function!
2118 : 8 : MHD_destroy_response (r); // will close alt_fd
2119 [ + - ]: 8 : if (alt)
2120 : 8 : dwarf_setalt (dbg, alt);
2121 : : }
2122 : : }
2123 : : else
2124 : : {
2125 : : // NB: dwarf_setalt(alt) inappropriate - already done!
2126 : : // NB: altdbg will stay 0 so nothing tries to redundantly dealloc.
2127 : : }
2128 : :
2129 [ + - ]: 8 : if (alt)
2130 : : {
2131 [ + - ]: 8 : if (verbose > 3)
2132 : 8 : obatched(clog) << "Resolved altdebug buildid=" << buildid << endl;
2133 : : }
2134 : : else // (alt == NULL) - signal possible presence of poor debuginfo
2135 : : {
2136 : 0 : debug_sourcefiles.insert("");
2137 [ # # ]: 0 : if (verbose > 3)
2138 : 0 : obatched(clog) << "Unresolved altdebug buildid=" << buildid << endl;
2139 : : }
2140 : : }
2141 : :
2142 : 23 : Dwarf_Off offset = 0;
2143 : : Dwarf_Off old_offset;
2144 : : size_t hsize;
2145 : :
2146 [ + + ]: 350 : while (dwarf_nextcu (dbg, old_offset = offset, &offset, &hsize, NULL, NULL, NULL) == 0)
2147 : : {
2148 : : Dwarf_Die cudie_mem;
2149 : 327 : Dwarf_Die *cudie = dwarf_offdie (dbg, old_offset + hsize, &cudie_mem);
2150 : :
2151 [ - + ]: 327 : if (cudie == NULL)
2152 : 4 : continue;
2153 [ + + ]: 327 : if (dwarf_tag (cudie) != DW_TAG_compile_unit)
2154 : 4 : continue;
2155 : :
2156 [ + - ]: 323 : const char *cuname = dwarf_diename(cudie) ?: "unknown";
2157 : :
2158 : : Dwarf_Files *files;
2159 : : size_t nfiles;
2160 [ - + ]: 323 : if (dwarf_getsrcfiles (cudie, &files, &nfiles) != 0)
2161 : 0 : continue;
2162 : :
2163 : : // extract DW_AT_comp_dir to resolve relative file names
2164 : 323 : const char *comp_dir = "";
2165 : : const char *const *dirs;
2166 : : size_t ndirs;
2167 [ + - + - ]: 646 : if (dwarf_getsrcdirs (files, &dirs, &ndirs) == 0 &&
2168 [ + - ]: 323 : dirs[0] != NULL)
2169 : 323 : comp_dir = dirs[0];
2170 [ - + ]: 323 : if (comp_dir == NULL)
2171 : 0 : comp_dir = "";
2172 : :
2173 [ + + ]: 323 : if (verbose > 3)
2174 : 34 : obatched(clog) << "searching for sources for cu=" << cuname << " comp_dir=" << comp_dir
2175 : 17 : << " #files=" << nfiles << " #dirs=" << ndirs << endl;
2176 : :
2177 [ - + - - ]: 323 : if (comp_dir[0] == '\0' && cuname[0] != '/')
2178 : : {
2179 : : // This is a common symptom for dwz-compressed debug files,
2180 : : // where the altdebug file cannot be resolved.
2181 [ # # ]: 0 : if (verbose > 3)
2182 : 0 : obatched(clog) << "skipping cu=" << cuname << " due to empty comp_dir" << endl;
2183 : 0 : continue;
2184 : : }
2185 : :
2186 [ + + ]: 5079 : for (size_t f = 1; f < nfiles; f++)
2187 : : {
2188 : 4756 : const char *hat = dwarf_filesrc (files, f, NULL, NULL);
2189 [ - + ]: 4756 : if (hat == NULL)
2190 : 0 : continue;
2191 : :
2192 [ - + ]: 4756 : if (string(hat) == "<built-in>") // gcc intrinsics, don't bother record
2193 : 0 : continue;
2194 : :
2195 : 4756 : string waldo;
2196 [ + + ]: 4756 : if (hat[0] == '/') // absolute
2197 : 2934 : waldo = (string (hat));
2198 [ + - ]: 1822 : else if (comp_dir[0] != '\0') // comp_dir relative
2199 : 1822 : waldo = (string (comp_dir) + string("/") + string (hat));
2200 : : else
2201 : : {
2202 [ # # ]: 0 : if (verbose > 3)
2203 : 0 : obatched(clog) << "skipping hat=" << hat << " due to empty comp_dir" << endl;
2204 : 0 : continue;
2205 : : }
2206 : :
2207 : : // NB: this is the 'waldo' that a dbginfo client will have
2208 : : // to supply for us to give them the file The comp_dir
2209 : : // prefixing is a definite complication. Otherwise we'd
2210 : : // have to return a setof comp_dirs (one per CU!) with
2211 : : // corresponding filesrc[] names, instead of one absolute
2212 : : // resoved set. Maybe we'll have to do that anyway. XXX
2213 : :
2214 [ - + ]: 4756 : if (verbose > 4)
2215 : 0 : obatched(clog) << waldo
2216 [ # # ]: 0 : << (debug_sourcefiles.find(waldo)==debug_sourcefiles.end() ? " new" : " dup") << endl;
2217 : :
2218 : 4756 : debug_sourcefiles.insert (waldo);
2219 : : }
2220 : : }
2221 : :
2222 : 23 : dwarf_end(dbg);
2223 [ + + ]: 23 : if (altdbg)
2224 : 8 : dwarf_end(altdbg);
2225 [ + + ]: 23 : if (altdbg_fd >= 0)
2226 : 8 : close(altdbg_fd);
2227 : : }
2228 : :
2229 : :
2230 : :
2231 : : static void
2232 : 113 : elf_classify (int fd, bool &executable_p, bool &debuginfo_p, string &buildid, set<string>& debug_sourcefiles)
2233 : : {
2234 : 113 : Elf *elf = elf_begin (fd, ELF_C_READ_MMAP_PRIVATE, NULL);
2235 [ - + ]: 114 : if (elf == NULL)
2236 : 0 : return;
2237 : :
2238 : : try // catch our types of errors and clean up the Elf* object
2239 : : {
2240 [ + - + + ]: 114 : if (elf_kind (elf) != ELF_K_ELF)
2241 : : {
2242 [ + - ]: 72 : elf_end (elf);
2243 : 72 : return;
2244 : : }
2245 : :
2246 : : GElf_Ehdr ehdr_storage;
2247 [ + - ]: 42 : GElf_Ehdr *ehdr = gelf_getehdr (elf, &ehdr_storage);
2248 [ - + ]: 42 : if (ehdr == NULL)
2249 : : {
2250 [ # # ]: 0 : elf_end (elf);
2251 : 0 : return;
2252 : : }
2253 : 42 : auto elf_type = ehdr->e_type;
2254 : :
2255 : : const void *build_id; // elfutils-owned memory
2256 [ + - ]: 42 : ssize_t sz = dwelf_elf_gnu_build_id (elf, & build_id);
2257 [ - + ]: 42 : if (sz <= 0)
2258 : : {
2259 : : // It's not a diagnostic-worthy error for an elf file to lack build-id.
2260 : : // It might just be very old.
2261 [ # # ]: 0 : elf_end (elf);
2262 : 0 : return;
2263 : : }
2264 : :
2265 : : // build_id is a raw byte array; convert to hexadecimal *lowercase*
2266 : 42 : unsigned char* build_id_bytes = (unsigned char*) build_id;
2267 [ + + ]: 874 : for (ssize_t idx=0; idx<sz; idx++)
2268 : : {
2269 [ + - ]: 832 : buildid += "0123456789abcdef"[build_id_bytes[idx] >> 4];
2270 [ + - ]: 826 : buildid += "0123456789abcdef"[build_id_bytes[idx] & 0xf];
2271 : : }
2272 : :
2273 : : // now decide whether it's an executable - namely, any allocatable section has
2274 : : // PROGBITS;
2275 [ + + + + ]: 42 : if (elf_type == ET_EXEC || elf_type == ET_DYN)
2276 : : {
2277 : : size_t shnum;
2278 [ + - ]: 36 : int rc = elf_getshdrnum (elf, &shnum);
2279 [ - + ]: 36 : if (rc < 0)
2280 [ # # # # ]: 0 : throw elfutils_exception(rc, "getshdrnum");
2281 : :
2282 : 36 : executable_p = false;
2283 [ + + ]: 680 : for (size_t sc = 0; sc < shnum; sc++)
2284 : : {
2285 [ + - ]: 663 : Elf_Scn *scn = elf_getscn (elf, sc);
2286 [ - + ]: 663 : if (scn == NULL)
2287 : 0 : continue;
2288 : :
2289 : : GElf_Shdr shdr_mem;
2290 [ + - ]: 663 : GElf_Shdr *shdr = gelf_getshdr (scn, &shdr_mem);
2291 [ - + ]: 663 : if (shdr == NULL)
2292 : 0 : continue;
2293 : :
2294 : : // allocated (loadable / vm-addr-assigned) section with available content?
2295 [ + + + + ]: 663 : if ((shdr->sh_type == SHT_PROGBITS) && (shdr->sh_flags & SHF_ALLOC))
2296 : : {
2297 [ - + ]: 19 : if (verbose > 4)
2298 [ # # # # : 0 : obatched(clog) << "executable due to SHF_ALLOC SHT_PROGBITS sc=" << sc << endl;
# # # # ]
2299 : 19 : executable_p = true;
2300 : 19 : break; // no need to keep looking for others
2301 : : }
2302 : : } // iterate over sections
2303 : : } // executable_p classification
2304 : :
2305 : : // now decide whether it's a debuginfo - namely, if it has any .debug* or .zdebug* sections
2306 : : // logic mostly stolen from fweimer@redhat.com's elfclassify drafts
2307 : : size_t shstrndx;
2308 [ + - ]: 42 : int rc = elf_getshdrstrndx (elf, &shstrndx);
2309 [ - + ]: 42 : if (rc < 0)
2310 [ # # # # ]: 0 : throw elfutils_exception(rc, "getshdrstrndx");
2311 : :
2312 : 42 : Elf_Scn *scn = NULL;
2313 : 42 : bool symtab_p = false;
2314 : 42 : bool bits_alloc_p = false;
2315 : : while (true)
2316 : : {
2317 [ + - ]: 1074 : scn = elf_nextscn (elf, scn);
2318 [ + + ]: 1076 : if (scn == NULL)
2319 : 19 : break;
2320 : : GElf_Shdr shdr_storage;
2321 [ + - ]: 1057 : GElf_Shdr *shdr = gelf_getshdr (scn, &shdr_storage);
2322 [ - + ]: 1054 : if (shdr == NULL)
2323 : 0 : break;
2324 [ + - ]: 1054 : const char *section_name = elf_strptr (elf, shstrndx, shdr->sh_name);
2325 [ - + ]: 1056 : if (section_name == NULL)
2326 : 0 : break;
2327 [ + + - + : 2088 : if (startswith (section_name, ".debug_line") ||
+ + ]
2328 : 1034 : startswith (section_name, ".zdebug_line"))
2329 : : {
2330 : 23 : debuginfo_p = true;
2331 : 23 : dwarf_extract_source_paths (elf, debug_sourcefiles);
2332 : 23 : break; // expecting only one .*debug_line, so no need to look for others
2333 : : }
2334 [ + + + + : 1996 : else if (startswith (section_name, ".debug_") ||
+ + ]
2335 : 965 : startswith (section_name, ".zdebug_"))
2336 : : {
2337 : 69 : debuginfo_p = true;
2338 : : // NB: don't break; need to parse .debug_line for sources
2339 : : }
2340 [ + + ]: 963 : else if (shdr->sh_type == SHT_SYMTAB)
2341 : : {
2342 : 1 : symtab_p = true;
2343 : : }
2344 [ + + ]: 962 : else if (shdr->sh_type != SHT_NOBITS
2345 [ + + ]: 562 : && shdr->sh_type != SHT_NOTE
2346 [ + + ]: 477 : && (shdr->sh_flags & SHF_ALLOC) != 0)
2347 : : {
2348 : 408 : bits_alloc_p = true;
2349 : : }
2350 : 1032 : }
2351 : :
2352 : : // For more expansive elf/split-debuginfo classification, we
2353 : : // want to identify as debuginfo "strip -s"-produced files
2354 : : // without .debug_info* (like libicudata), but we don't want to
2355 : : // identify "strip -g" executables (with .symtab left there).
2356 [ + + - + ]: 42 : if (symtab_p && !bits_alloc_p)
2357 : 0 : debuginfo_p = true;
2358 : : }
2359 [ # # ]: 0 : catch (const reportable_exception& e)
2360 : : {
2361 [ # # ]: 0 : e.report(clog);
2362 : : }
2363 : 42 : elf_end (elf);
2364 : : }
2365 : :
2366 : :
2367 : : static void
2368 : 173 : scan_source_file (const string& rps, const stat_t& st,
2369 : : sqlite_ps& ps_upsert_buildids,
2370 : : sqlite_ps& ps_upsert_files,
2371 : : sqlite_ps& ps_upsert_de,
2372 : : sqlite_ps& ps_upsert_s,
2373 : : sqlite_ps& ps_query,
2374 : : sqlite_ps& ps_scan_done,
2375 : : unsigned& fts_cached,
2376 : : unsigned& fts_executable,
2377 : : unsigned& fts_debuginfo,
2378 : : unsigned& fts_sourcefiles)
2379 : : {
2380 : : /* See if we know of it already. */
2381 : : int rc = ps_query
2382 [ + - ]: 173 : .reset()
2383 [ + - ]: 173 : .bind(1, rps)
2384 [ + - ]: 173 : .bind(2, st.st_mtime)
2385 [ + - ]: 173 : .step();
2386 [ + - ]: 173 : ps_query.reset();
2387 [ + + ]: 173 : if (rc == SQLITE_ROW) // i.e., a result, as opposed to DONE (no results)
2388 : : // no need to recheck a file/version we already know
2389 : : // specifically, no need to elf-begin a file we already determined is non-elf
2390 : : // (so is stored with buildid=NULL)
2391 : : {
2392 : 128 : fts_cached++;
2393 : 128 : return;
2394 : : }
2395 : :
2396 : 45 : bool executable_p = false, debuginfo_p = false; // E and/or D
2397 : 90 : string buildid;
2398 : 90 : set<string> sourcefiles;
2399 : :
2400 [ + - ]: 45 : int fd = open (rps.c_str(), O_RDONLY);
2401 : : try
2402 : : {
2403 [ + - ]: 45 : if (fd >= 0)
2404 [ + - ]: 45 : elf_classify (fd, executable_p, debuginfo_p, buildid, sourcefiles);
2405 : : else
2406 [ # # # # : 0 : throw libc_exception(errno, string("open ") + rps);
# # ]
2407 : 45 : add_metric ("scanned_bytes_total","source","file",
2408 [ + - + - : 45 : st.st_size);
+ - + - ]
2409 [ + - + - : 45 : inc_metric ("scanned_files_total","source","file");
+ - + - ]
2410 : : }
2411 : : // NB: we catch exceptions here too, so that we can
2412 : : // cache the corrupt-elf case (!executable_p &&
2413 : : // !debuginfo_p) just below, just as if we had an
2414 : : // EPERM error from open(2).
2415 [ - - ]: 0 : catch (const reportable_exception& e)
2416 : : {
2417 [ - - ]: 0 : e.report(clog);
2418 : : }
2419 : :
2420 [ + - ]: 45 : if (fd >= 0)
2421 [ + - ]: 45 : close (fd);
2422 : :
2423 : : // register this file name in the interning table
2424 : : ps_upsert_files
2425 [ + - ]: 45 : .reset()
2426 [ + - ]: 45 : .bind(1, rps)
2427 [ + - ]: 45 : .step_ok_done();
2428 : :
2429 [ + + ]: 45 : if (buildid == "")
2430 : : {
2431 : : // no point storing an elf file without buildid
2432 : 41 : executable_p = false;
2433 : 41 : debuginfo_p = false;
2434 : : }
2435 : : else
2436 : : {
2437 : : // register this build-id in the interning table
2438 : : ps_upsert_buildids
2439 [ + - ]: 4 : .reset()
2440 [ + - ]: 4 : .bind(1, buildid)
2441 [ + - ]: 4 : .step_ok_done();
2442 : : }
2443 : :
2444 [ + + ]: 45 : if (executable_p)
2445 : 3 : fts_executable ++;
2446 [ + + ]: 45 : if (debuginfo_p)
2447 : 3 : fts_debuginfo ++;
2448 [ + + + + ]: 45 : if (executable_p || debuginfo_p)
2449 : : {
2450 : : ps_upsert_de
2451 [ + - ]: 4 : .reset()
2452 [ + - ]: 4 : .bind(1, buildid)
2453 [ + + + - ]: 4 : .bind(2, debuginfo_p ? 1 : 0)
2454 [ + + + - ]: 4 : .bind(3, executable_p ? 1 : 0)
2455 [ + - ]: 4 : .bind(4, rps)
2456 [ + - ]: 4 : .bind(5, st.st_mtime)
2457 [ + - ]: 4 : .step_ok_done();
2458 : : }
2459 [ + + ]: 45 : if (executable_p)
2460 [ + - + - : 3 : inc_metric("found_executable_total","source","files");
+ - + - ]
2461 [ + + ]: 45 : if (debuginfo_p)
2462 [ + - + - : 3 : inc_metric("found_debuginfo_total","source","files");
+ - + - ]
2463 : :
2464 [ + + + - : 45 : if (sourcefiles.size() && buildid != "")
+ + ]
2465 : : {
2466 : 3 : fts_sourcefiles += sourcefiles.size();
2467 : :
2468 [ + + ]: 487 : for (auto&& dwarfsrc : sourcefiles)
2469 : : {
2470 : 484 : char *srp = realpath(dwarfsrc.c_str(), NULL);
2471 [ + + ]: 484 : if (srp == NULL) // also if DWZ unresolved dwarfsrc=""
2472 : 6 : continue; // unresolvable files are not a serious problem
2473 : : // throw libc_exception(errno, "fts/file realpath " + srcpath);
2474 [ + - ]: 478 : string srps = string(srp);
2475 : 478 : free (srp);
2476 : :
2477 : : struct stat sfs;
2478 : 478 : rc = stat(srps.c_str(), &sfs);
2479 [ - + ]: 478 : if (rc != 0)
2480 : 0 : continue;
2481 : :
2482 [ + - ]: 478 : if (verbose > 2)
2483 [ + - + - ]: 956 : obatched(clog) << "recorded buildid=" << buildid << " file=" << srps
2484 [ + - + - : 478 : << " mtime=" << sfs.st_mtime
+ - + - +
- ]
2485 [ + - + - : 478 : << " as source " << dwarfsrc << endl;
+ - ]
2486 : :
2487 : : ps_upsert_files
2488 [ + - ]: 478 : .reset()
2489 [ + - ]: 478 : .bind(1, srps)
2490 [ + - ]: 478 : .step_ok_done();
2491 : :
2492 : : // PR25548: store canonicalized dwarfsrc path
2493 [ + - ]: 956 : string dwarfsrc_canon = canon_pathname (dwarfsrc);
2494 [ + + ]: 478 : if (dwarfsrc_canon != dwarfsrc)
2495 : : {
2496 [ + + ]: 88 : if (verbose > 3)
2497 [ + - + - : 1 : obatched(clog) << "canonicalized src=" << dwarfsrc << " alias=" << dwarfsrc_canon << endl;
+ - + - +
- + - ]
2498 : : }
2499 : :
2500 : : ps_upsert_files
2501 [ + - ]: 478 : .reset()
2502 [ + - ]: 478 : .bind(1, dwarfsrc_canon)
2503 [ + - ]: 478 : .step_ok_done();
2504 : :
2505 : : ps_upsert_s
2506 [ + - ]: 478 : .reset()
2507 [ + - ]: 478 : .bind(1, buildid)
2508 [ + - ]: 478 : .bind(2, dwarfsrc_canon)
2509 [ + - ]: 478 : .bind(3, srps)
2510 [ + - ]: 478 : .bind(4, sfs.st_mtime)
2511 [ + - ]: 478 : .step_ok_done();
2512 : :
2513 [ + - + - : 478 : inc_metric("found_sourcerefs_total","source","files");
+ - + - ]
2514 : : }
2515 : : }
2516 : :
2517 : : ps_scan_done
2518 [ + - ]: 45 : .reset()
2519 [ + - ]: 45 : .bind(1, rps)
2520 [ + - ]: 45 : .bind(2, st.st_mtime)
2521 [ + - ]: 45 : .bind(3, st.st_size)
2522 [ + - ]: 45 : .step_ok_done();
2523 : :
2524 [ + - ]: 45 : if (verbose > 2)
2525 [ + - + - ]: 90 : obatched(clog) << "recorded buildid=" << buildid << " file=" << rps
2526 [ + - + - : 45 : << " mtime=" << st.st_mtime << " atype="
+ - + - +
- ]
2527 : : << (executable_p ? "E" : "")
2528 [ + - + + : 45 : << (debuginfo_p ? "D" : "") << endl;
+ - + + +
- + - ]
2529 : : }
2530 : :
2531 : :
2532 : :
2533 : :
2534 : :
2535 : : // Analyze given archive file of given age; record buildids / exec/debuginfo-ness of its
2536 : : // constituent files with given upsert statements.
2537 : : static void
2538 : 36 : archive_classify (const string& rps, string& archive_extension,
2539 : : sqlite_ps& ps_upsert_buildids, sqlite_ps& ps_upsert_files,
2540 : : sqlite_ps& ps_upsert_de, sqlite_ps& ps_upsert_sref, sqlite_ps& ps_upsert_sdef,
2541 : : time_t mtime,
2542 : : unsigned& fts_executable, unsigned& fts_debuginfo, unsigned& fts_sref, unsigned& fts_sdef,
2543 : : bool& fts_sref_complete_p)
2544 : : {
2545 [ + - ]: 72 : string archive_decoder = "/dev/null";
2546 [ + + ]: 144 : for (auto&& arch : scan_archives)
2547 [ + - + + ]: 108 : if (string_endswith(rps, arch.first))
2548 : : {
2549 [ + - ]: 36 : archive_extension = arch.first;
2550 [ + - ]: 36 : archive_decoder = arch.second;
2551 : : }
2552 : :
2553 : : FILE* fp;
2554 : : defer_dtor<FILE*,int>::dtor_fn dfn;
2555 [ + + ]: 36 : if (archive_decoder != "cat")
2556 : : {
2557 [ + - + - : 9 : string popen_cmd = archive_decoder + " " + shell_escape(rps);
+ - ]
2558 [ + - ]: 3 : fp = popen (popen_cmd.c_str(), "r"); // "e" O_CLOEXEC?
2559 : 3 : dfn = pclose;
2560 [ - + ]: 3 : if (fp == NULL)
2561 [ # # # # : 0 : throw libc_exception (errno, string("popen ") + popen_cmd);
# # ]
2562 : : }
2563 : : else
2564 : : {
2565 [ + - ]: 33 : fp = fopen (rps.c_str(), "r");
2566 : 33 : dfn = fclose;
2567 [ - + ]: 33 : if (fp == NULL)
2568 [ # # # # : 0 : throw libc_exception (errno, string("fopen ") + rps);
# # ]
2569 : : }
2570 : 72 : defer_dtor<FILE*,int> fp_closer (fp, dfn);
2571 : :
2572 : : struct archive *a;
2573 [ + - ]: 36 : a = archive_read_new();
2574 [ - + ]: 36 : if (a == NULL)
2575 [ # # # # ]: 0 : throw archive_exception("cannot create archive reader");
2576 : 72 : defer_dtor<struct archive*,int> archive_closer (a, archive_read_free);
2577 : :
2578 [ + - ]: 36 : int rc = archive_read_support_format_all(a);
2579 [ - + ]: 36 : if (rc != ARCHIVE_OK)
2580 [ # # # # ]: 0 : throw archive_exception(a, "cannot select all formats");
2581 [ + - ]: 36 : rc = archive_read_support_filter_all(a);
2582 [ - + ]: 36 : if (rc != ARCHIVE_OK)
2583 [ # # # # ]: 0 : throw archive_exception(a, "cannot select all filters");
2584 : :
2585 [ + - ]: 36 : rc = archive_read_open_FILE (a, fp);
2586 [ - + ]: 36 : if (rc != ARCHIVE_OK)
2587 [ # # # # ]: 0 : throw archive_exception(a, "cannot open archive from pipe");
2588 : :
2589 [ + + ]: 36 : if (verbose > 3)
2590 [ + - + - : 34 : obatched(clog) << "libarchive scanning " << rps << endl;
+ - + - ]
2591 : :
2592 : : while(1) // parse archive entries
2593 : : {
2594 [ - + ]: 255 : if (interrupted)
2595 : 0 : break;
2596 : :
2597 : : try
2598 : : {
2599 : : struct archive_entry *e;
2600 [ + - ]: 255 : rc = archive_read_next_header (a, &e);
2601 [ + + ]: 254 : if (rc != ARCHIVE_OK)
2602 : 36 : break;
2603 : :
2604 [ + - + + ]: 218 : if (! S_ISREG(archive_entry_mode (e))) // skip non-files completely
2605 : 150 : continue;
2606 : :
2607 [ + - ]: 138 : string fn = canonicalized_archive_entry_pathname (e);
2608 : :
2609 [ + + ]: 69 : if (verbose > 3)
2610 [ + - + - : 64 : obatched(clog) << "libarchive checking " << fn << endl;
+ - + - ]
2611 : :
2612 : : // extract this file to a temporary file
2613 : 69 : char* tmppath = NULL;
2614 : 69 : rc = asprintf (&tmppath, "%s/debuginfod.XXXXXX", tmpdir.c_str());
2615 [ - + ]: 69 : if (rc < 0)
2616 [ # # # # ]: 0 : throw libc_exception (ENOMEM, "cannot allocate tmppath");
2617 : 138 : defer_dtor<void*,void> tmmpath_freer (tmppath, free);
2618 [ + - ]: 69 : int fd = mkstemp (tmppath);
2619 [ - + ]: 69 : if (fd < 0)
2620 [ # # # # ]: 0 : throw libc_exception (errno, "cannot create temporary file");
2621 : 69 : unlink (tmppath); // unlink now so OS will release the file as soon as we close the fd
2622 : 138 : defer_dtor<int,int> minifd_closer (fd, close);
2623 : :
2624 [ + - ]: 69 : rc = archive_read_data_into_fd (a, fd);
2625 [ - + ]: 69 : if (rc != ARCHIVE_OK)
2626 [ # # # # ]: 0 : throw archive_exception(a, "cannot extract file");
2627 : :
2628 : : // finally ... time to run elf_classify on this bad boy and update the database
2629 : 69 : bool executable_p = false, debuginfo_p = false;
2630 : 138 : string buildid;
2631 : 138 : set<string> sourcefiles;
2632 [ + - ]: 68 : elf_classify (fd, executable_p, debuginfo_p, buildid, sourcefiles);
2633 : : // NB: might throw
2634 : :
2635 [ + + ]: 69 : if (buildid != "") // intern buildid
2636 : : {
2637 : : ps_upsert_buildids
2638 [ + - ]: 38 : .reset()
2639 [ + - ]: 38 : .bind(1, buildid)
2640 [ + - ]: 38 : .step_ok_done();
2641 : : }
2642 : :
2643 : : ps_upsert_files // register this rpm constituent file name in interning table
2644 [ + - ]: 69 : .reset()
2645 [ + - ]: 69 : .bind(1, fn)
2646 [ + - ]: 69 : .step_ok_done();
2647 : :
2648 [ + + ]: 69 : if (sourcefiles.size() > 0) // sref records needed
2649 : : {
2650 : : // NB: we intern each source file once. Once raw, as it
2651 : : // appears in the DWARF file list coming back from
2652 : : // elf_classify() - because it'll end up in the
2653 : : // _norm.artifactsrc column. We don't also put another
2654 : : // version with a '.' at the front, even though that's
2655 : : // how rpm/cpio packs names, because we hide that from
2656 : : // the database for storage efficiency.
2657 : :
2658 [ + + ]: 61 : for (auto&& s : sourcefiles)
2659 : : {
2660 [ - + ]: 45 : if (s == "")
2661 : : {
2662 : 0 : fts_sref_complete_p = false;
2663 : 0 : continue;
2664 : : }
2665 : :
2666 : : // PR25548: store canonicalized source path
2667 : 45 : const string& dwarfsrc = s;
2668 [ + - ]: 45 : string dwarfsrc_canon = canon_pathname (dwarfsrc);
2669 [ + + ]: 45 : if (dwarfsrc_canon != dwarfsrc)
2670 : : {
2671 [ + - ]: 2 : if (verbose > 3)
2672 [ + - + - : 2 : obatched(clog) << "canonicalized src=" << dwarfsrc << " alias=" << dwarfsrc_canon << endl;
+ - + - +
- + - ]
2673 : : }
2674 : :
2675 : : ps_upsert_files
2676 [ + - ]: 45 : .reset()
2677 [ + - ]: 45 : .bind(1, dwarfsrc_canon)
2678 [ + - ]: 45 : .step_ok_done();
2679 : :
2680 : : ps_upsert_sref
2681 [ + - ]: 45 : .reset()
2682 [ + - ]: 45 : .bind(1, buildid)
2683 [ + - ]: 45 : .bind(2, dwarfsrc_canon)
2684 [ + - ]: 45 : .step_ok_done();
2685 : :
2686 : 45 : fts_sref ++;
2687 : : }
2688 : : }
2689 : :
2690 [ + + ]: 69 : if (executable_p)
2691 : 16 : fts_executable ++;
2692 [ + + ]: 69 : if (debuginfo_p)
2693 : 22 : fts_debuginfo ++;
2694 : :
2695 [ + + + + ]: 69 : if (executable_p || debuginfo_p)
2696 : : {
2697 : : ps_upsert_de
2698 [ + - ]: 38 : .reset()
2699 [ + - ]: 38 : .bind(1, buildid)
2700 [ + + + - ]: 38 : .bind(2, debuginfo_p ? 1 : 0)
2701 [ + + + - ]: 38 : .bind(3, executable_p ? 1 : 0)
2702 [ + - ]: 38 : .bind(4, rps)
2703 [ + - ]: 38 : .bind(5, mtime)
2704 [ + - ]: 38 : .bind(6, fn)
2705 [ + - ]: 38 : .step_ok_done();
2706 : : }
2707 : : else // potential source - sdef record
2708 : : {
2709 : 31 : fts_sdef ++;
2710 : : ps_upsert_sdef
2711 [ + - ]: 31 : .reset()
2712 [ + - ]: 31 : .bind(1, rps)
2713 [ + - ]: 31 : .bind(2, mtime)
2714 [ + - ]: 31 : .bind(3, fn)
2715 [ + - ]: 31 : .step_ok_done();
2716 : : }
2717 : :
2718 [ + - + + : 69 : if ((verbose > 2) && (executable_p || debuginfo_p))
+ + ]
2719 [ + - + - ]: 76 : obatched(clog) << "recorded buildid=" << buildid << " rpm=" << rps << " file=" << fn
2720 [ + - + - : 38 : << " mtime=" << mtime << " atype="
+ - + - +
- + - +
- ]
2721 : : << (executable_p ? "E" : "")
2722 : : << (debuginfo_p ? "D" : "")
2723 [ + - + + : 38 : << " sourcefiles=" << sourcefiles.size() << endl;
+ - + + +
- + - + -
+ - ]
2724 : :
2725 : : }
2726 [ - - ]: 0 : catch (const reportable_exception& e)
2727 : : {
2728 [ - - ]: 0 : e.report(clog);
2729 : : }
2730 : 219 : }
2731 : 36 : }
2732 : :
2733 : :
2734 : :
2735 : : // scan for archive files such as .rpm
2736 : : static void
2737 : 118 : scan_archive_file (const string& rps, const stat_t& st,
2738 : : sqlite_ps& ps_upsert_buildids,
2739 : : sqlite_ps& ps_upsert_files,
2740 : : sqlite_ps& ps_upsert_de,
2741 : : sqlite_ps& ps_upsert_sref,
2742 : : sqlite_ps& ps_upsert_sdef,
2743 : : sqlite_ps& ps_query,
2744 : : sqlite_ps& ps_scan_done,
2745 : : unsigned& fts_cached,
2746 : : unsigned& fts_executable,
2747 : : unsigned& fts_debuginfo,
2748 : : unsigned& fts_sref,
2749 : : unsigned& fts_sdef)
2750 : : {
2751 : : /* See if we know of it already. */
2752 : : int rc = ps_query
2753 [ + - ]: 118 : .reset()
2754 [ + - ]: 118 : .bind(1, rps)
2755 [ + - ]: 118 : .bind(2, st.st_mtime)
2756 [ + - ]: 118 : .step();
2757 [ + - ]: 118 : ps_query.reset();
2758 [ + + ]: 118 : if (rc == SQLITE_ROW) // i.e., a result, as opposed to DONE (no results)
2759 : : // no need to recheck a file/version we already know
2760 : : // specifically, no need to parse this archive again, since we already have
2761 : : // it as a D or E or S record,
2762 : : // (so is stored with buildid=NULL)
2763 : : {
2764 : 82 : fts_cached ++;
2765 : 82 : return;
2766 : : }
2767 : :
2768 : : // intern the archive file name
2769 : : ps_upsert_files
2770 [ + - ]: 36 : .reset()
2771 [ + - ]: 36 : .bind(1, rps)
2772 [ + - ]: 36 : .step_ok_done();
2773 : :
2774 : : // extract the archive contents
2775 : 36 : unsigned my_fts_executable = 0, my_fts_debuginfo = 0, my_fts_sref = 0, my_fts_sdef = 0;
2776 : 36 : bool my_fts_sref_complete_p = true;
2777 : : try
2778 : : {
2779 : 72 : string archive_extension;
2780 : 36 : archive_classify (rps, archive_extension,
2781 : : ps_upsert_buildids, ps_upsert_files,
2782 : : ps_upsert_de, ps_upsert_sref, ps_upsert_sdef, // dalt
2783 [ + - ]: 36 : st.st_mtime,
2784 : : my_fts_executable, my_fts_debuginfo, my_fts_sref, my_fts_sdef,
2785 : : my_fts_sref_complete_p);
2786 [ + - ]: 72 : add_metric ("scanned_bytes_total","source",archive_extension + " archive",
2787 [ + - + - : 108 : st.st_size);
+ - ]
2788 [ + - + - : 36 : inc_metric ("scanned_files_total","source",archive_extension + " archive");
+ - + - ]
2789 [ + - + - ]: 36 : add_metric("found_debuginfo_total","source",archive_extension + " archive",
2790 [ + - + - ]: 72 : my_fts_debuginfo);
2791 [ + - + - ]: 36 : add_metric("found_executable_total","source",archive_extension + " archive",
2792 [ + - + - ]: 72 : my_fts_executable);
2793 [ + - + - ]: 36 : add_metric("found_sourcerefs_total","source",archive_extension + " archive",
2794 [ + - + - ]: 72 : my_fts_sref);
2795 : : }
2796 [ - - ]: 0 : catch (const reportable_exception& e)
2797 : : {
2798 [ - - ]: 0 : e.report(clog);
2799 : : }
2800 : :
2801 [ + - ]: 36 : if (verbose > 2)
2802 [ + - + - ]: 72 : obatched(clog) << "scanned archive=" << rps
2803 [ + - + - : 36 : << " mtime=" << st.st_mtime
+ - ]
2804 [ + - ]: 36 : << " executables=" << my_fts_executable
2805 [ + - + - ]: 36 : << " debuginfos=" << my_fts_debuginfo
2806 [ + - + - ]: 36 : << " srefs=" << my_fts_sref
2807 [ + - + - ]: 36 : << " sdefs=" << my_fts_sdef
2808 [ + - + - ]: 36 : << endl;
2809 : :
2810 : 36 : fts_executable += my_fts_executable;
2811 : 36 : fts_debuginfo += my_fts_debuginfo;
2812 : 36 : fts_sref += my_fts_sref;
2813 : 36 : fts_sdef += my_fts_sdef;
2814 : :
2815 [ + - ]: 36 : if (my_fts_sref_complete_p) // leave incomplete?
2816 : : ps_scan_done
2817 [ + - ]: 36 : .reset()
2818 [ + - ]: 36 : .bind(1, rps)
2819 [ + - ]: 36 : .bind(2, st.st_mtime)
2820 [ + - ]: 36 : .bind(3, st.st_size)
2821 [ + - ]: 36 : .step_ok_done();
2822 : : }
2823 : :
2824 : :
2825 : :
2826 : : ////////////////////////////////////////////////////////////////////////
2827 : :
2828 : :
2829 : :
2830 : : // The thread that consumes file names off of the scanq. We hold
2831 : : // the persistent sqlite_ps's at this level and delegate file/archive
2832 : : // scanning to other functions.
2833 : : static void*
2834 : 12 : thread_main_scanner (void* arg)
2835 : : {
2836 : : (void) arg;
2837 : :
2838 : : // all the prepared statements fit to use, the _f_ set:
2839 [ + - + - : 48 : sqlite_ps ps_f_upsert_buildids (db, "file-buildids-intern", "insert or ignore into " BUILDIDS "_buildids VALUES (NULL, ?);");
+ - ]
2840 [ + - + - : 48 : sqlite_ps ps_f_upsert_files (db, "file-files-intern", "insert or ignore into " BUILDIDS "_files VALUES (NULL, ?);");
+ - ]
2841 : : sqlite_ps ps_f_upsert_de (db, "file-de-upsert",
2842 : : "insert or ignore into " BUILDIDS "_f_de "
2843 : : "(buildid, debuginfo_p, executable_p, file, mtime) "
2844 : : "values ((select id from " BUILDIDS "_buildids where hex = ?),"
2845 : : " ?,?,"
2846 [ + - + - : 48 : " (select id from " BUILDIDS "_files where name = ?), ?);");
+ - ]
2847 : : sqlite_ps ps_f_upsert_s (db, "file-s-upsert",
2848 : : "insert or ignore into " BUILDIDS "_f_s "
2849 : : "(buildid, artifactsrc, file, mtime) "
2850 : : "values ((select id from " BUILDIDS "_buildids where hex = ?),"
2851 : : " (select id from " BUILDIDS "_files where name = ?),"
2852 : : " (select id from " BUILDIDS "_files where name = ?),"
2853 [ + - + - : 48 : " ?);");
+ - ]
2854 : : sqlite_ps ps_f_query (db, "file-negativehit-find",
2855 : : "select 1 from " BUILDIDS "_file_mtime_scanned where sourcetype = 'F' "
2856 [ + - + - : 48 : "and file = (select id from " BUILDIDS "_files where name = ?) and mtime = ?;");
+ - ]
2857 : : sqlite_ps ps_f_scan_done (db, "file-scanned",
2858 : : "insert or ignore into " BUILDIDS "_file_mtime_scanned (sourcetype, file, mtime, size)"
2859 [ + - + - : 48 : "values ('F', (select id from " BUILDIDS "_files where name = ?), ?, ?);");
+ - ]
2860 : :
2861 : : // and now for the _r_ set
2862 [ + - + - : 48 : sqlite_ps ps_r_upsert_buildids (db, "rpm-buildid-intern", "insert or ignore into " BUILDIDS "_buildids VALUES (NULL, ?);");
+ - ]
2863 [ + - + - : 48 : sqlite_ps ps_r_upsert_files (db, "rpm-file-intern", "insert or ignore into " BUILDIDS "_files VALUES (NULL, ?);");
+ - ]
2864 : : sqlite_ps ps_r_upsert_de (db, "rpm-de-insert",
2865 : : "insert or ignore into " BUILDIDS "_r_de (buildid, debuginfo_p, executable_p, file, mtime, content) values ("
2866 : : "(select id from " BUILDIDS "_buildids where hex = ?), ?, ?, "
2867 : : "(select id from " BUILDIDS "_files where name = ?), ?, "
2868 [ + - + - : 48 : "(select id from " BUILDIDS "_files where name = ?));");
+ - ]
2869 : : sqlite_ps ps_r_upsert_sref (db, "rpm-sref-insert",
2870 : : "insert or ignore into " BUILDIDS "_r_sref (buildid, artifactsrc) values ("
2871 : : "(select id from " BUILDIDS "_buildids where hex = ?), "
2872 [ + - + - : 48 : "(select id from " BUILDIDS "_files where name = ?));");
+ - ]
2873 : : sqlite_ps ps_r_upsert_sdef (db, "rpm-sdef-insert",
2874 : : "insert or ignore into " BUILDIDS "_r_sdef (file, mtime, content) values ("
2875 : : "(select id from " BUILDIDS "_files where name = ?), ?,"
2876 [ + - + - : 48 : "(select id from " BUILDIDS "_files where name = ?));");
+ - ]
2877 : : sqlite_ps ps_r_query (db, "rpm-negativehit-query",
2878 : : "select 1 from " BUILDIDS "_file_mtime_scanned where "
2879 [ + - + - : 48 : "sourcetype = 'R' and file = (select id from " BUILDIDS "_files where name = ?) and mtime = ?;");
+ - ]
2880 : : sqlite_ps ps_r_scan_done (db, "rpm-scanned",
2881 : : "insert or ignore into " BUILDIDS "_file_mtime_scanned (sourcetype, file, mtime, size)"
2882 [ + - + - : 36 : "values ('R', (select id from " BUILDIDS "_files where name = ?), ?, ?);");
+ - ]
2883 : :
2884 : :
2885 : 12 : unsigned fts_cached = 0, fts_executable = 0, fts_debuginfo = 0, fts_sourcefiles = 0;
2886 : 12 : unsigned fts_sref = 0, fts_sdef = 0;
2887 : :
2888 [ + - + - : 12 : add_metric("thread_count", "role", "scan", 1);
+ - + - ]
2889 [ + - + - : 12 : add_metric("thread_busy", "role", "scan", 1);
+ - + - ]
2890 [ + + ]: 197 : while (! interrupted)
2891 : : {
2892 : 185 : scan_payload p;
2893 : :
2894 [ + - + - : 185 : add_metric("thread_busy", "role", "scan", -1);
+ - + - ]
2895 [ + - ]: 185 : bool gotone = scanq.wait_front(p);
2896 [ + - + - : 185 : add_metric("thread_busy", "role", "scan", 1);
+ - + - ]
2897 : :
2898 [ + + ]: 185 : if (! gotone) continue; // go back to waiting
2899 : :
2900 : : try
2901 : : {
2902 : 173 : bool scan_archive = false;
2903 [ + + ]: 692 : for (auto&& arch : scan_archives)
2904 [ + - + + ]: 519 : if (string_endswith(p.first, arch.first))
2905 : 118 : scan_archive = true;
2906 : :
2907 [ + + ]: 173 : if (scan_archive)
2908 [ + - ]: 118 : scan_archive_file (p.first, p.second,
2909 : : ps_r_upsert_buildids,
2910 : : ps_r_upsert_files,
2911 : : ps_r_upsert_de,
2912 : : ps_r_upsert_sref,
2913 : : ps_r_upsert_sdef,
2914 : : ps_r_query,
2915 : : ps_r_scan_done,
2916 : : fts_cached,
2917 : : fts_executable,
2918 : : fts_debuginfo,
2919 : : fts_sref,
2920 : : fts_sdef);
2921 : :
2922 [ + - ]: 173 : if (scan_files) // NB: maybe "else if" ?
2923 [ + - ]: 173 : scan_source_file (p.first, p.second,
2924 : : ps_f_upsert_buildids,
2925 : : ps_f_upsert_files,
2926 : : ps_f_upsert_de,
2927 : : ps_f_upsert_s,
2928 : : ps_f_query,
2929 : : ps_f_scan_done,
2930 : : fts_cached, fts_executable, fts_debuginfo, fts_sourcefiles);
2931 : : }
2932 [ - - ]: 0 : catch (const reportable_exception& e)
2933 : : {
2934 [ - - ]: 0 : e.report(cerr);
2935 : : }
2936 : :
2937 [ + + + + : 173 : if (fts_cached || fts_executable || fts_debuginfo || fts_sourcefiles || fts_sref || fts_sdef)
- + - - -
- ]
2938 : : {} // NB: not just if a successful scan - we might have encountered -ENOSPC & failed
2939 [ + - + - ]: 173 : (void) statfs_free_enough_p(db_path, "database"); // report sqlite filesystem size
2940 [ + - + - ]: 173 : (void) statfs_free_enough_p(tmpdir, "tmpdir"); // this too, in case of fdcache/tmpfile usage
2941 : :
2942 : : // finished a scanning step -- not a "loop", because we just
2943 : : // consume the traversal loop's work, whenever
2944 [ + - + - : 173 : inc_metric("thread_work_total","role","scan");
+ - + - ]
2945 : : }
2946 : :
2947 : :
2948 [ + - + - : 12 : add_metric("thread_busy", "role", "scan", -1);
+ - + - ]
2949 : 24 : return 0;
2950 : : }
2951 : :
2952 : :
2953 : :
2954 : : // The thread that traverses all the source_paths and enqueues all the
2955 : : // matching files into the file/archive scan queue.
2956 : : static void
2957 : 12 : scan_source_paths()
2958 : : {
2959 : : // NB: fedora 31 glibc/fts(3) crashes inside fts_read() on empty
2960 : : // path list.
2961 [ + + ]: 12 : if (source_paths.empty())
2962 : 1 : return;
2963 : :
2964 : : // Turn the source_paths into an fts(3)-compatible char**. Since
2965 : : // source_paths[] does not change after argv processing, the
2966 : : // c_str()'s are safe to keep around awile.
2967 : 22 : vector<const char *> sps;
2968 [ + + ]: 51 : for (auto&& sp: source_paths)
2969 [ + - ]: 40 : sps.push_back(sp.c_str());
2970 [ + - ]: 11 : sps.push_back(NULL);
2971 : :
2972 [ + + + - ]: 11 : FTS *fts = fts_open ((char * const *)sps.data(),
2973 : : (traverse_logical ? FTS_LOGICAL : FTS_PHYSICAL|FTS_XDEV)
2974 : : | FTS_NOCHDIR /* multithreaded */,
2975 : : NULL);
2976 [ - + ]: 11 : if (fts == NULL)
2977 [ # # # # ]: 0 : throw libc_exception(errno, "cannot fts_open");
2978 : 11 : defer_dtor<FTS*,int> fts_cleanup (fts, fts_close);
2979 : :
2980 : : struct timespec ts_start, ts_end;
2981 : 11 : clock_gettime (CLOCK_MONOTONIC, &ts_start);
2982 : 11 : unsigned fts_scanned = 0, fts_regex = 0;
2983 : :
2984 : : FTSENT *f;
2985 [ + - + + ]: 366 : while ((f = fts_read (fts)) != NULL)
2986 : : {
2987 [ - + ]: 355 : if (interrupted) break;
2988 : :
2989 [ - + ]: 355 : if (sigusr2 != forced_groom_count) // stop early if groom triggered
2990 : : {
2991 [ # # ]: 0 : scanq.clear(); // clear previously issued work for scanner threads
2992 : 0 : break;
2993 : : }
2994 : :
2995 : 355 : fts_scanned ++;
2996 : :
2997 [ + - ]: 355 : if (verbose > 2)
2998 [ + - + - : 355 : obatched(clog) << "fts traversing " << f->fts_path << endl;
+ - + - ]
2999 : :
3000 [ + - + + : 355 : switch (f->fts_info)
+ ]
3001 : : {
3002 : 173 : case FTS_F:
3003 : : {
3004 : : /* Found a file. Convert it to an absolute path, so
3005 : : the buildid database does not have relative path
3006 : : names that are unresolvable from a subsequent run
3007 : : in a different cwd. */
3008 : 173 : char *rp = realpath(f->fts_path, NULL);
3009 [ - + ]: 173 : if (rp == NULL)
3010 : 0 : continue; // ignore dangling symlink or such
3011 [ + - ]: 346 : string rps = string(rp);
3012 : 173 : free (rp);
3013 : :
3014 [ + - ]: 173 : bool ri = !regexec (&file_include_regex, rps.c_str(), 0, 0, 0);
3015 [ + - ]: 173 : bool rx = !regexec (&file_exclude_regex, rps.c_str(), 0, 0, 0);
3016 [ + - - + ]: 173 : if (!ri || rx)
3017 : : {
3018 [ # # ]: 0 : if (verbose > 3)
3019 [ # # # # ]: 0 : obatched(clog) << "fts skipped by regex "
3020 [ # # # # : 0 : << (!ri ? "I" : "") << (rx ? "X" : "") << endl;
# # # # #
# ]
3021 : 0 : fts_regex ++;
3022 [ # # ]: 0 : if (!ri)
3023 [ # # # # : 0 : inc_metric("traversed_total","type","file-skipped-I");
# # # # ]
3024 [ # # ]: 0 : if (rx)
3025 [ # # # # : 0 : inc_metric("traversed_total","type","file-skipped-X");
# # # # ]
3026 : : }
3027 : : else
3028 : : {
3029 [ + - + - ]: 173 : scanq.push_back (make_pair(rps, *f->fts_statp));
3030 [ + - + - : 173 : inc_metric("traversed_total","type","file");
+ - + - ]
3031 : : }
3032 : : }
3033 : 173 : break;
3034 : :
3035 : 0 : case FTS_ERR:
3036 : : case FTS_NS:
3037 : : // report on some types of errors because they may reflect fixable misconfiguration
3038 : : {
3039 [ # # # # : 0 : auto x = libc_exception(f->fts_errno, string("fts traversal ") + string(f->fts_path));
# # # # ]
3040 [ # # ]: 0 : x.report(cerr);
3041 : : }
3042 [ # # # # : 0 : inc_metric("traversed_total","type","error");
# # # # ]
3043 : 0 : break;
3044 : :
3045 : 18 : case FTS_SL: // ignore, but count because debuginfod -L would traverse these
3046 [ + - + - : 18 : inc_metric("traversed_total","type","symlink");
+ - + - ]
3047 : 18 : break;
3048 : :
3049 : 82 : case FTS_D: // ignore
3050 [ + - + - : 82 : inc_metric("traversed_total","type","directory");
+ - + - ]
3051 : 82 : break;
3052 : :
3053 : 82 : default: // ignore
3054 [ + - + - : 82 : inc_metric("traversed_total","type","other");
+ - + - ]
3055 : 82 : break;
3056 : : }
3057 : : }
3058 : 11 : clock_gettime (CLOCK_MONOTONIC, &ts_end);
3059 : 11 : double deltas = (ts_end.tv_sec - ts_start.tv_sec) + (ts_end.tv_nsec - ts_start.tv_nsec)/1.e9;
3060 : :
3061 [ + - + - : 22 : obatched(clog) << "fts traversed source paths in " << deltas << "s, scanned=" << fts_scanned
+ - + - +
- ]
3062 [ + - + - : 11 : << ", regex-skipped=" << fts_regex << endl;
+ - ]
3063 : : }
3064 : :
3065 : :
3066 : : static void*
3067 : 3 : thread_main_fts_source_paths (void* arg)
3068 : : {
3069 : : (void) arg; // ignore; we operate on global data
3070 : :
3071 [ + - + - : 3 : set_metric("thread_tid", "role","traverse", tid());
+ - + - ]
3072 [ + - + - : 3 : add_metric("thread_count", "role", "traverse", 1);
+ - + - ]
3073 : :
3074 : 3 : time_t last_rescan = 0;
3075 : :
3076 [ + - ]: 35 : while (! interrupted)
3077 : : {
3078 : 35 : sleep (1);
3079 : 35 : scanq.wait_idle(); // don't start a new traversal while scanners haven't finished the job
3080 : 35 : scanq.done_idle(); // release the hounds
3081 [ + + ]: 35 : if (interrupted) break;
3082 : :
3083 : 32 : time_t now = time(NULL);
3084 : 32 : bool rescan_now = false;
3085 [ + + ]: 32 : if (last_rescan == 0) // at least one initial rescan is documented even for -t0
3086 : 3 : rescan_now = true;
3087 [ + + + + ]: 32 : if (rescan_s > 0 && (long)now > (long)(last_rescan + rescan_s))
3088 : 2 : rescan_now = true;
3089 [ + + ]: 32 : if (sigusr1 != forced_rescan_count)
3090 : : {
3091 : 9 : forced_rescan_count = sigusr1;
3092 : 9 : rescan_now = true;
3093 : : }
3094 [ + + ]: 32 : if (rescan_now)
3095 : : {
3096 [ + - + - : 12 : set_metric("thread_busy", "role","traverse", 1);
+ - + - ]
3097 : : try
3098 : : {
3099 [ + - ]: 12 : scan_source_paths();
3100 : : }
3101 [ - - ]: 0 : catch (const reportable_exception& e)
3102 : : {
3103 [ - - ]: 0 : e.report(cerr);
3104 : : }
3105 : 12 : last_rescan = time(NULL); // NB: now was before scanning
3106 : : // finished a traversal loop
3107 [ + - + - : 12 : inc_metric("thread_work_total", "role","traverse");
+ - + - ]
3108 [ + - + - : 12 : set_metric("thread_busy", "role","traverse", 0);
+ - + - ]
3109 : : }
3110 : : }
3111 : :
3112 : 3 : return 0;
3113 : : }
3114 : :
3115 : :
3116 : :
3117 : : ////////////////////////////////////////////////////////////////////////
3118 : :
3119 : : static void
3120 : 5 : database_stats_report()
3121 : : {
3122 : : sqlite_ps ps_query (db, "database-overview",
3123 [ + - + - : 20 : "select label,quantity from " BUILDIDS "_stats");
+ - ]
3124 : :
3125 [ + - + - : 5 : obatched(clog) << "database record counts:" << endl;
+ - ]
3126 : : while (1)
3127 : : {
3128 [ - + ]: 55 : if (interrupted) break;
3129 [ - + ]: 55 : if (sigusr1 != forced_rescan_count) // stop early if scan triggered
3130 : 0 : break;
3131 : :
3132 [ + - ]: 55 : int rc = ps_query.step();
3133 [ + + ]: 55 : if (rc == SQLITE_DONE) break;
3134 [ - + ]: 50 : if (rc != SQLITE_ROW)
3135 [ # # # # ]: 0 : throw sqlite_exception(rc, "step");
3136 : :
3137 [ + - ]: 100 : obatched(clog)
3138 [ + - + - ]: 100 : << right << setw(20) << ((const char*) sqlite3_column_text(ps_query, 0) ?: (const char*) "NULL")
3139 : : << " "
3140 [ + - + - : 100 : << (sqlite3_column_text(ps_query, 1) ?: (const unsigned char*) "NULL")
+ - + - +
- + - +
- ]
3141 [ + - ]: 50 : << endl;
3142 : :
3143 [ + - + - ]: 50 : set_metric("groom", "statistic",
3144 [ + - ]: 50 : ((const char*) sqlite3_column_text(ps_query, 0) ?: (const char*) "NULL"),
3145 [ + - + - : 150 : (sqlite3_column_double(ps_query, 1)));
+ - + - ]
3146 : 50 : }
3147 : 5 : }
3148 : :
3149 : :
3150 : : // Do a round of database grooming that might take many minutes to run.
3151 : 6 : void groom()
3152 : : {
3153 [ + - + - : 6 : obatched(clog) << "grooming database" << endl;
+ - ]
3154 : :
3155 : : struct timespec ts_start, ts_end;
3156 : 6 : clock_gettime (CLOCK_MONOTONIC, &ts_start);
3157 : :
3158 : : // scan for files that have disappeared
3159 : : sqlite_ps files (db, "check old files", "select s.mtime, s.file, f.name from "
3160 : : BUILDIDS "_file_mtime_scanned s, " BUILDIDS "_files f "
3161 [ + - + - : 19 : "where f.id = s.file");
+ - ]
3162 [ + - + - : 19 : sqlite_ps files_del_f_de (db, "nuke f_de", "delete from " BUILDIDS "_f_de where file = ? and mtime = ?");
+ - ]
3163 [ + - + - : 19 : sqlite_ps files_del_r_de (db, "nuke r_de", "delete from " BUILDIDS "_r_de where file = ? and mtime = ?");
+ - ]
3164 : : sqlite_ps files_del_scan (db, "nuke f_m_s", "delete from " BUILDIDS "_file_mtime_scanned "
3165 [ + - + - : 19 : "where file = ? and mtime = ?");
+ - ]
3166 [ + - ]: 6 : files.reset();
3167 : : while(1)
3168 : : {
3169 [ - + ]: 154 : if (interrupted) break;
3170 : :
3171 [ + - ]: 154 : int rc = files.step();
3172 [ + + ]: 154 : if (rc != SQLITE_ROW)
3173 : 6 : break;
3174 : :
3175 [ + - ]: 148 : int64_t mtime = sqlite3_column_int64 (files, 0);
3176 [ + - ]: 148 : int64_t fileid = sqlite3_column_int64 (files, 1);
3177 [ + - + - ]: 148 : const char* filename = ((const char*) sqlite3_column_text (files, 2) ?: "");
3178 : : struct stat s;
3179 : 148 : rc = stat(filename, &s);
3180 [ + + - + ]: 148 : if (rc < 0 || (mtime != (int64_t) s.st_mtime))
3181 : : {
3182 [ + - ]: 16 : if (verbose > 2)
3183 [ + - + - : 16 : obatched(clog) << "groom: forgetting file=" << filename << " mtime=" << mtime << endl;
+ - + - +
- + - ]
3184 [ + - + - : 16 : files_del_f_de.reset().bind(1,fileid).bind(2,mtime).step_ok_done();
+ - + - ]
3185 [ + - + - : 16 : files_del_r_de.reset().bind(1,fileid).bind(2,mtime).step_ok_done();
+ - + - ]
3186 [ + - + - : 16 : files_del_scan.reset().bind(1,fileid).bind(2,mtime).step_ok_done();
+ - + - ]
3187 [ + - + - : 16 : inc_metric("groomed_total", "decision", "stale");
+ - + - ]
3188 : : }
3189 : : else
3190 [ + - + - : 132 : inc_metric("groomed_total", "decision", "fresh");
+ - + - ]
3191 : :
3192 [ - + ]: 148 : if (sigusr1 != forced_rescan_count) // stop early if scan triggered
3193 : 0 : break;
3194 : 148 : }
3195 [ + - ]: 6 : files.reset();
3196 : :
3197 : : // delete buildids with no references in _r_de or _f_de tables;
3198 : : // cascades to _r_sref & _f_s records
3199 : : sqlite_ps buildids_del (db, "nuke orphan buildids",
3200 : : "delete from " BUILDIDS "_buildids "
3201 : : "where not exists (select 1 from " BUILDIDS "_f_de d where " BUILDIDS "_buildids.id = d.buildid) "
3202 [ + - + - : 19 : "and not exists (select 1 from " BUILDIDS "_r_de d where " BUILDIDS "_buildids.id = d.buildid)");
+ - ]
3203 [ + - + - ]: 6 : buildids_del.reset().step_ok_done();
3204 : :
3205 [ - + ]: 6 : if (interrupted) return;
3206 : :
3207 : : // NB: "vacuum" is too heavy for even daily runs: it rewrites the entire db, so is done as maxigroom -G
3208 [ + - + - : 24 : sqlite_ps g1 (db, "incremental vacuum", "pragma incremental_vacuum");
+ - ]
3209 [ + - + - ]: 6 : g1.reset().step_ok_done();
3210 [ + - + - : 24 : sqlite_ps g2 (db, "optimize", "pragma optimize");
+ - ]
3211 [ + - + + ]: 6 : g2.reset().step_ok_done();
3212 [ + - + - : 15 : sqlite_ps g3 (db, "wal checkpoint", "pragma wal_checkpoint=truncate");
+ - ]
3213 [ + - + - ]: 5 : g3.reset().step_ok_done();
3214 : :
3215 [ + - ]: 5 : database_stats_report();
3216 : :
3217 [ + - + - ]: 5 : (void) statfs_free_enough_p(db_path, "database"); // report sqlite filesystem size
3218 : :
3219 [ + - ]: 5 : sqlite3_db_release_memory(db); // shrink the process if possible
3220 [ + - ]: 5 : sqlite3_db_release_memory(dbq); // ... for both connections
3221 [ + - ]: 5 : debuginfod_pool_groom(); // and release any debuginfod_client objects we've been holding onto
3222 : :
3223 [ + - ]: 5 : fdcache.limit(0,0); // release the fdcache contents
3224 [ + - ]: 5 : fdcache.limit(fdcache_fds,fdcache_mbs); // restore status quo parameters
3225 : :
3226 : 5 : clock_gettime (CLOCK_MONOTONIC, &ts_end);
3227 : 5 : double deltas = (ts_end.tv_sec - ts_start.tv_sec) + (ts_end.tv_nsec - ts_start.tv_nsec)/1.e9;
3228 : :
3229 [ + - + - : 5 : obatched(clog) << "groomed database in " << deltas << "s" << endl;
+ - + - +
- ]
3230 : : }
3231 : :
3232 : :
3233 : : static void*
3234 : 3 : thread_main_groom (void* /*arg*/)
3235 : : {
3236 [ + - + - : 3 : set_metric("thread_tid", "role", "groom", tid());
+ - + - ]
3237 [ + - + - : 3 : add_metric("thread_count", "role", "groom", 1);
+ - + - ]
3238 : :
3239 : 3 : time_t last_groom = 0;
3240 : :
3241 : : while (1)
3242 : : {
3243 : 35 : sleep (1);
3244 : 35 : scanq.wait_idle(); // PR25394: block scanners during grooming!
3245 [ + + ]: 35 : if (interrupted) break;
3246 : :
3247 : 32 : time_t now = time(NULL);
3248 : 32 : bool groom_now = false;
3249 [ + + ]: 32 : if (last_groom == 0) // at least one initial groom is documented even for -g0
3250 : 3 : groom_now = true;
3251 [ + + + + ]: 32 : if (groom_s > 0 && (long)now > (long)(last_groom + groom_s))
3252 : 2 : groom_now = true;
3253 [ + + ]: 32 : if (sigusr2 != forced_groom_count)
3254 : : {
3255 : 3 : forced_groom_count = sigusr2;
3256 : 3 : groom_now = true;
3257 : : }
3258 [ + + ]: 32 : if (groom_now)
3259 : : {
3260 [ + - + - : 6 : set_metric("thread_busy", "role", "groom", 1);
+ - + - ]
3261 : : try
3262 : : {
3263 [ + + ]: 6 : groom ();
3264 : : }
3265 [ - + ]: 1 : catch (const sqlite_exception& e)
3266 : : {
3267 [ + - + - : 1 : obatched(cerr) << e.message << endl;
+ - ]
3268 : : }
3269 : 6 : last_groom = time(NULL); // NB: now was before grooming
3270 : : // finished a grooming loop
3271 [ + - + - : 6 : inc_metric("thread_work_total", "role", "groom");
+ - + - ]
3272 [ + - + - : 6 : set_metric("thread_busy", "role", "groom", 0);
+ - + - ]
3273 : : }
3274 : :
3275 : 32 : scanq.done_idle();
3276 : 32 : }
3277 : :
3278 : 3 : return 0;
3279 : : }
3280 : :
3281 : :
3282 : : ////////////////////////////////////////////////////////////////////////
3283 : :
3284 : :
3285 : : static void
3286 : 3 : signal_handler (int /* sig */)
3287 : : {
3288 : 3 : interrupted ++;
3289 : :
3290 [ + - ]: 3 : if (db)
3291 : 3 : sqlite3_interrupt (db);
3292 [ + - ]: 3 : if (dbq)
3293 : 3 : sqlite3_interrupt (dbq);
3294 : :
3295 : : // NB: don't do anything else in here
3296 : 3 : }
3297 : :
3298 : : static void
3299 : 9 : sigusr1_handler (int /* sig */)
3300 : : {
3301 : 9 : sigusr1 ++;
3302 : : // NB: don't do anything else in here
3303 : 9 : }
3304 : :
3305 : : static void
3306 : 3 : sigusr2_handler (int /* sig */)
3307 : : {
3308 : 3 : sigusr2 ++;
3309 : : // NB: don't do anything else in here
3310 : 3 : }
3311 : :
3312 : :
3313 : :
3314 : :
3315 : :
3316 : : // A user-defined sqlite function, to score the sharedness of the
3317 : : // prefix of two strings. This is used to compare candidate debuginfo
3318 : : // / source-rpm names, so that the closest match
3319 : : // (directory-topology-wise closest) is found. This is important in
3320 : : // case the same sref (source file name) is in many -debuginfo or
3321 : : // -debugsource RPMs, such as when multiple versions/releases of the
3322 : : // same package are in the database.
3323 : :
3324 : 109 : static void sqlite3_sharedprefix_fn (sqlite3_context* c, int argc, sqlite3_value** argv)
3325 : : {
3326 [ - + ]: 109 : if (argc != 2)
3327 : 0 : sqlite3_result_error(c, "expect 2 string arguments", -1);
3328 [ + - + + ]: 218 : else if ((sqlite3_value_type(argv[0]) != SQLITE_TEXT) ||
3329 [ + + ]: 109 : (sqlite3_value_type(argv[1]) != SQLITE_TEXT))
3330 : 3 : sqlite3_result_null(c);
3331 : : else
3332 : : {
3333 : 106 : const unsigned char* a = sqlite3_value_text (argv[0]);
3334 : 106 : const unsigned char* b = sqlite3_value_text (argv[1]);
3335 : 106 : int i = 0;
3336 [ + + ]: 8511 : while (*a++ == *b++)
3337 : 8405 : i++;
3338 : 106 : sqlite3_result_int (c, i);
3339 : : }
3340 : 109 : }
3341 : :
3342 : :
3343 : : int
3344 : 3 : main (int argc, char *argv[])
3345 : : {
3346 : 3 : (void) setlocale (LC_ALL, "");
3347 : 3 : (void) bindtextdomain (PACKAGE_TARNAME, LOCALEDIR);
3348 : 3 : (void) textdomain (PACKAGE_TARNAME);
3349 : :
3350 : : /* Tell the library which version we are expecting. */
3351 [ + - ]: 3 : elf_version (EV_CURRENT);
3352 : :
3353 [ - + + - ]: 3 : tmpdir = string(getenv("TMPDIR") ?: "/tmp");
3354 : :
3355 : : /* Set computed default values. */
3356 [ + - + - : 3 : db_path = string(getenv("HOME") ?: "/") + string("/.debuginfod.sqlite"); /* XDG? */
+ - + - ]
3357 [ + - ]: 3 : int rc = regcomp (& file_include_regex, ".*", REG_EXTENDED|REG_NOSUB); // match everything
3358 [ - + ]: 3 : if (rc != 0)
3359 : : error (EXIT_FAILURE, 0, "regcomp failure: %d", rc);
3360 [ + - ]: 3 : rc = regcomp (& file_exclude_regex, "^$", REG_EXTENDED|REG_NOSUB); // match nothing
3361 [ - + ]: 3 : if (rc != 0)
3362 : : error (EXIT_FAILURE, 0, "regcomp failure: %d", rc);
3363 : :
3364 : : // default parameters for fdcache are computed from system stats
3365 : : struct statfs sfs;
3366 : 3 : rc = statfs(tmpdir.c_str(), &sfs);
3367 [ - + ]: 3 : if (rc < 0)
3368 : 0 : fdcache_mbs = 1024; // 1 gigabyte
3369 : : else
3370 : 3 : fdcache_mbs = sfs.f_bavail * sfs.f_bsize / 1024 / 1024 / 4; // 25% of free space
3371 : 3 : fdcache_mintmp = 25; // emergency flush at 25% remaining (75% full)
3372 : 3 : fdcache_prefetch = 64; // guesstimate storage is this much less costly than re-decompression
3373 : 3 : fdcache_fds = (concurrency + fdcache_prefetch) * 2;
3374 : :
3375 : : /* Parse and process arguments. */
3376 : : int remaining;
3377 : 3 : argp_program_version_hook = print_version; // this works
3378 [ + - ]: 3 : (void) argp_parse (&argp, argc, argv, ARGP_IN_ORDER, &remaining, NULL);
3379 [ - + ]: 3 : if (remaining != argc)
3380 : 0 : error (EXIT_FAILURE, 0,
3381 : 0 : "unexpected argument: %s", argv[remaining]);
3382 : :
3383 [ - + - - : 3 : if (scan_archives.size()==0 && !scan_files && source_paths.size()>0)
- - - + ]
3384 [ # # # # : 0 : obatched(clog) << "warning: without -F -R -U -Z, ignoring PATHs" << endl;
# # ]
3385 : :
3386 [ + - ]: 3 : fdcache.limit(fdcache_fds, fdcache_mbs);
3387 : :
3388 : 3 : (void) signal (SIGPIPE, SIG_IGN); // microhttpd can generate it incidentally, ignore
3389 : 3 : (void) signal (SIGINT, signal_handler); // ^C
3390 : 3 : (void) signal (SIGHUP, signal_handler); // EOF
3391 : 3 : (void) signal (SIGTERM, signal_handler); // systemd
3392 : 3 : (void) signal (SIGUSR1, sigusr1_handler); // end-user
3393 : 3 : (void) signal (SIGUSR2, sigusr2_handler); // end-user
3394 : :
3395 : : /* Get database ready. */
3396 [ + - ]: 3 : rc = sqlite3_open_v2 (db_path.c_str(), &db, (SQLITE_OPEN_READWRITE
3397 : : |SQLITE_OPEN_URI
3398 : : |SQLITE_OPEN_PRIVATECACHE
3399 : : |SQLITE_OPEN_CREATE
3400 : : |SQLITE_OPEN_FULLMUTEX), /* thread-safe */
3401 : : NULL);
3402 [ - + ]: 3 : if (rc == SQLITE_CORRUPT)
3403 : : {
3404 : 0 : (void) unlink (db_path.c_str());
3405 [ # # ]: 0 : error (EXIT_FAILURE, 0,
3406 : : "cannot open %s, deleted database: %s", db_path.c_str(), sqlite3_errmsg(db));
3407 : : }
3408 [ - + ]: 3 : else if (rc)
3409 : : {
3410 [ # # ]: 0 : error (EXIT_FAILURE, 0,
3411 : : "cannot open %s, consider deleting database: %s", db_path.c_str(), sqlite3_errmsg(db));
3412 : : }
3413 : :
3414 : : // open the readonly query variant
3415 : : // NB: PRIVATECACHE allows web queries to operate in parallel with
3416 : : // much other grooming/scanning operation.
3417 [ + - ]: 3 : rc = sqlite3_open_v2 (db_path.c_str(), &dbq, (SQLITE_OPEN_READONLY
3418 : : |SQLITE_OPEN_URI
3419 : : |SQLITE_OPEN_PRIVATECACHE
3420 : : |SQLITE_OPEN_FULLMUTEX), /* thread-safe */
3421 : : NULL);
3422 [ - + ]: 3 : if (rc)
3423 : : {
3424 [ # # ]: 0 : error (EXIT_FAILURE, 0,
3425 : : "cannot open %s, consider deleting database: %s", db_path.c_str(), sqlite3_errmsg(dbq));
3426 : : }
3427 : :
3428 : :
3429 [ + - + - : 3 : obatched(clog) << "opened database " << db_path << endl;
+ - + - ]
3430 [ + - + - : 3 : obatched(clog) << "sqlite version " << sqlite3_version << endl;
+ - + - ]
3431 : :
3432 : : // add special string-prefix-similarity function used in rpm sref/sdef resolution
3433 [ + - ]: 3 : rc = sqlite3_create_function(dbq, "sharedprefix", 2, SQLITE_UTF8, NULL,
3434 : : & sqlite3_sharedprefix_fn, NULL, NULL);
3435 [ - + ]: 3 : if (rc != SQLITE_OK)
3436 [ # # ]: 0 : error (EXIT_FAILURE, 0,
3437 : : "cannot create sharedprefix function: %s", sqlite3_errmsg(dbq));
3438 : :
3439 [ + + ]: 3 : if (verbose > 3)
3440 [ + - + - : 1 : obatched(clog) << "ddl: " << DEBUGINFOD_SQLITE_DDL << endl;
+ - + - ]
3441 [ + - ]: 3 : rc = sqlite3_exec (db, DEBUGINFOD_SQLITE_DDL, NULL, NULL, NULL);
3442 [ - + ]: 3 : if (rc != SQLITE_OK)
3443 : : {
3444 [ # # ]: 0 : error (EXIT_FAILURE, 0,
3445 : : "cannot run database schema ddl: %s", sqlite3_errmsg(db));
3446 : : }
3447 : :
3448 : : // Start httpd server threads. Separate pool for IPv4 and IPv6, in
3449 : : // case the host only has one protocol stack.
3450 [ + - ]: 3 : MHD_Daemon *d4 = MHD_start_daemon (MHD_USE_THREAD_PER_CONNECTION
3451 : : #if MHD_VERSION >= 0x00095300
3452 : : | MHD_USE_INTERNAL_POLLING_THREAD
3453 : : #else
3454 : : | MHD_USE_SELECT_INTERNALLY
3455 : : #endif
3456 : : | MHD_USE_DEBUG, /* report errors to stderr */
3457 : : http_port,
3458 : : NULL, NULL, /* default accept policy */
3459 : : handler_cb, NULL, /* handler callback */
3460 : : MHD_OPTION_END);
3461 [ + - ]: 3 : MHD_Daemon *d6 = MHD_start_daemon (MHD_USE_THREAD_PER_CONNECTION
3462 : : #if MHD_VERSION >= 0x00095300
3463 : : | MHD_USE_INTERNAL_POLLING_THREAD
3464 : : #else
3465 : : | MHD_USE_SELECT_INTERNALLY
3466 : : #endif
3467 : : | MHD_USE_IPv6
3468 : : | MHD_USE_DEBUG, /* report errors to stderr */
3469 : : http_port,
3470 : : NULL, NULL, /* default accept policy */
3471 : : handler_cb, NULL, /* handler callback */
3472 : : MHD_OPTION_END);
3473 : :
3474 [ - + - - ]: 3 : if (d4 == NULL && d6 == NULL) // neither ipv4 nor ipv6? boo
3475 : : {
3476 : 0 : sqlite3 *database = db;
3477 : 0 : sqlite3 *databaseq = dbq;
3478 : 0 : db = dbq = 0; // for signal_handler not to freak
3479 [ # # ]: 0 : sqlite3_close (databaseq);
3480 [ # # ]: 0 : sqlite3_close (database);
3481 : 0 : error (EXIT_FAILURE, 0, "cannot start http server at port %d", http_port);
3482 : : }
3483 : :
3484 [ + - + - ]: 6 : obatched(clog) << "started http server on "
3485 : : << (d4 != NULL ? "IPv4 " : "")
3486 : : << (d6 != NULL ? "IPv6 " : "")
3487 [ + - + - : 3 : << "port=" << http_port << endl;
+ - + - +
- + - +
- ]
3488 : :
3489 : : // add maxigroom sql if -G given
3490 [ - + ]: 3 : if (maxigroom)
3491 : : {
3492 [ # # # # : 0 : obatched(clog) << "maxigrooming database, please wait." << endl;
# # ]
3493 [ # # # # ]: 0 : extra_ddl.push_back("create index if not exists " BUILDIDS "_r_sref_arc on " BUILDIDS "_r_sref(artifactsrc);");
3494 [ # # # # ]: 0 : extra_ddl.push_back("delete from " BUILDIDS "_r_sdef where not exists (select 1 from " BUILDIDS "_r_sref b where " BUILDIDS "_r_sdef.content = b.artifactsrc);");
3495 [ # # # # ]: 0 : extra_ddl.push_back("drop index if exists " BUILDIDS "_r_sref_arc;");
3496 : :
3497 : : // NB: we don't maxigroom the _files interning table. It'd require a temp index on all the
3498 : : // tables that have file foreign-keys, which is a lot.
3499 : :
3500 : : // NB: with =delete, may take up 3x disk space total during vacuum process
3501 : : // vs. =off (only 2x but may corrupt database if program dies mid-vacuum)
3502 : : // vs. =wal (>3x observed, but safe)
3503 [ # # # # ]: 0 : extra_ddl.push_back("pragma journal_mode=delete;");
3504 [ # # # # ]: 0 : extra_ddl.push_back("vacuum;");
3505 [ # # # # ]: 0 : extra_ddl.push_back("pragma journal_mode=wal;");
3506 : : }
3507 : :
3508 : : // run extra -D sql if given
3509 [ - + ]: 3 : for (auto&& i: extra_ddl)
3510 : : {
3511 [ # # ]: 0 : if (verbose > 1)
3512 [ # # # # : 0 : obatched(clog) << "extra ddl:\n" << i << endl;
# # # # ]
3513 [ # # ]: 0 : rc = sqlite3_exec (db, i.c_str(), NULL, NULL, NULL);
3514 [ # # # # : 0 : if (rc != SQLITE_OK && rc != SQLITE_DONE && rc != SQLITE_ROW)
# # ]
3515 [ # # ]: 0 : error (0, 0,
3516 : : "warning: cannot run database extra ddl %s: %s", i.c_str(), sqlite3_errmsg(db));
3517 : : }
3518 : :
3519 [ - + ]: 3 : if (maxigroom)
3520 [ # # # # : 0 : obatched(clog) << "maxigroomed database" << endl;
# # ]
3521 : :
3522 [ + - + - : 3 : obatched(clog) << "search concurrency " << concurrency << endl;
+ - + - ]
3523 [ + - + - : 3 : obatched(clog) << "rescan time " << rescan_s << endl;
+ - + - ]
3524 [ + - + - : 3 : obatched(clog) << "fdcache fds " << fdcache_fds << endl;
+ - + - ]
3525 [ + - + - : 3 : obatched(clog) << "fdcache mbs " << fdcache_mbs << endl;
+ - + - ]
3526 [ + - + - : 3 : obatched(clog) << "fdcache prefetch " << fdcache_prefetch << endl;
+ - + - ]
3527 [ + - + - : 3 : obatched(clog) << "fdcache tmpdir " << tmpdir << endl;
+ - + - ]
3528 [ + - + - : 3 : obatched(clog) << "fdcache tmpdir min% " << fdcache_mintmp << endl;
+ - + - ]
3529 [ + - + - : 3 : obatched(clog) << "groom time " << groom_s << endl;
+ - + - ]
3530 [ + - ]: 3 : if (scan_archives.size()>0)
3531 : : {
3532 [ + - ]: 6 : obatched ob(clog);
3533 [ + - ]: 3 : auto& o = ob << "scanning archive types ";
3534 [ + + ]: 12 : for (auto&& arch : scan_archives)
3535 [ + - + - : 9 : o << arch.first << "(" << arch.second << ") ";
+ - + - ]
3536 [ + - ]: 3 : o << endl;
3537 : : }
3538 : 3 : const char* du = getenv(DEBUGINFOD_URLS_ENV_VAR);
3539 [ + - + + ]: 3 : if (du && du[0] != '\0') // set to non-empty string?
3540 [ + - + - : 2 : obatched(clog) << "upstream debuginfod servers: " << du << endl;
+ - + - ]
3541 : :
3542 : 3 : vector<pthread_t> all_threads;
3543 : :
3544 : : pthread_t pt;
3545 : 3 : rc = pthread_create (& pt, NULL, thread_main_groom, NULL);
3546 [ - + ]: 3 : if (rc)
3547 : : error (EXIT_FAILURE, rc, "cannot spawn thread to groom database\n");
3548 : : else
3549 : : {
3550 : : #ifdef HAVE_PTHREAD_SETNAME_NP
3551 : 3 : (void) pthread_setname_np (pt, "groom");
3552 : : #endif
3553 [ + - ]: 3 : all_threads.push_back(pt);
3554 : : }
3555 : :
3556 [ - + - - : 3 : if (scan_files || scan_archives.size() > 0)
+ - ]
3557 : : {
3558 : 3 : rc = pthread_create (& pt, NULL, thread_main_fts_source_paths, NULL);
3559 [ - + ]: 3 : if (rc)
3560 : : error (EXIT_FAILURE, rc, "cannot spawn thread to traverse source paths\n");
3561 : : #ifdef HAVE_PTHREAD_SETNAME_NP
3562 : 3 : (void) pthread_setname_np (pt, "traverse");
3563 : : #endif
3564 [ + - ]: 3 : all_threads.push_back(pt);
3565 : :
3566 [ + + ]: 15 : for (unsigned i=0; i<concurrency; i++)
3567 : : {
3568 : 12 : rc = pthread_create (& pt, NULL, thread_main_scanner, NULL);
3569 [ - + ]: 12 : if (rc)
3570 : : error (EXIT_FAILURE, rc, "cannot spawn thread to scan source files / archives\n");
3571 : : #ifdef HAVE_PTHREAD_SETNAME_NP
3572 : 12 : (void) pthread_setname_np (pt, "scan");
3573 : : #endif
3574 [ + - ]: 12 : all_threads.push_back(pt);
3575 : : }
3576 : : }
3577 : :
3578 : : /* Trivial main loop! */
3579 [ + - + - ]: 3 : set_metric("ready", 1);
3580 [ + + ]: 18 : while (! interrupted)
3581 [ + - ]: 15 : pause ();
3582 [ + - ]: 3 : scanq.nuke(); // wake up any remaining scanq-related threads, let them die
3583 [ + - + - ]: 3 : set_metric("ready", 0);
3584 : :
3585 [ + - ]: 3 : if (verbose)
3586 [ + - + - : 3 : obatched(clog) << "stopping" << endl;
+ - ]
3587 : :
3588 : : /* Join all our threads. */
3589 [ + + ]: 21 : for (auto&& it : all_threads)
3590 [ + - ]: 18 : pthread_join (it, NULL);
3591 : :
3592 : : /* Stop all the web service threads. */
3593 [ + - + - ]: 3 : if (d4) MHD_stop_daemon (d4);
3594 [ + - + - ]: 3 : if (d6) MHD_stop_daemon (d6);
3595 : :
3596 : : /* With all threads known dead, we can clean up the global resources. */
3597 [ + - ]: 3 : rc = sqlite3_exec (db, DEBUGINFOD_SQLITE_CLEANUP_DDL, NULL, NULL, NULL);
3598 [ - + ]: 3 : if (rc != SQLITE_OK)
3599 : : {
3600 [ # # ]: 0 : error (0, 0,
3601 : : "warning: cannot run database cleanup ddl: %s", sqlite3_errmsg(db));
3602 : : }
3603 : :
3604 : : // NB: no problem with unconditional free here - an earlier failed regcomp would exit program
3605 [ + - ]: 3 : (void) regfree (& file_include_regex);
3606 [ + - ]: 3 : (void) regfree (& file_exclude_regex);
3607 : :
3608 : 3 : sqlite3 *database = db;
3609 : 3 : sqlite3 *databaseq = dbq;
3610 : 3 : db = dbq = 0; // for signal_handler not to freak
3611 [ + - ]: 3 : (void) sqlite3_close (databaseq);
3612 [ + - ]: 3 : (void) sqlite3_close (database);
3613 : :
3614 : 3 : return 0;
3615 : : }
|
